From b55e9e78e36475325f61897cdc993f1982bac486 Mon Sep 17 00:00:00 2001 From: Cameron Sparr Date: Fri, 5 Aug 2016 09:51:20 +0100 Subject: [PATCH 01/28] gopsutil, fix /proc/pid/io naming issue closes #1584 --- CHANGELOG.md | 1 + Godeps | 2 +- plugins/inputs/procstat/spec_processor.go | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 91e933102..b7f49c33b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ - [#1430](https://github.com/influxdata/telegraf/issues/1430): Fix prometheus character sanitizing. Sanitize more win_perf_counters characters. - [#1534](https://github.com/influxdata/telegraf/pull/1534): Add diskio io_time to FreeBSD & report timing metrics as ms (as linux does). - [#1379](https://github.com/influxdata/telegraf/issues/1379): Fix covering Amazon Linux for post remove flow. +- [#1584](https://github.com/influxdata/telegraf/issues/1584): procstat missing fields: read/write bytes & count ## v1.0 beta 3 [2016-07-18] diff --git a/Godeps b/Godeps index 2b4fce555..3cb67336e 100644 --- a/Godeps +++ b/Godeps @@ -44,7 +44,7 @@ github.com/prometheus/client_model fa8ad6fec33561be4280a8f0514318c79d7f6cb6 github.com/prometheus/common e8eabff8812b05acf522b45fdcd725a785188e37 github.com/prometheus/procfs 406e5b7bfd8201a36e2bb5f7bdae0b03380c2ce8 github.com/samuel/go-zookeeper 218e9c81c0dd8b3b18172b2bbfad92cc7d6db55f -github.com/shirou/gopsutil ee66bc560c366dd33b9a4046ba0b644caba46bed +github.com/shirou/gopsutil 4d0c402af66c78735c5ccf820dc2ca7de5e4ff08 github.com/soniah/gosnmp b1b4f885b12c5dcbd021c5cee1c904110de6db7d github.com/sparrc/aerospike-client-go d4bb42d2c2d39dae68e054116f4538af189e05d5 github.com/streadway/amqp b4f3ceab0337f013208d31348b578d83c0064744 diff --git a/plugins/inputs/procstat/spec_processor.go b/plugins/inputs/procstat/spec_processor.go index 3789e99d0..5143d8bcc 100644 --- a/plugins/inputs/procstat/spec_processor.go +++ b/plugins/inputs/procstat/spec_processor.go @@ -71,7 +71,7 @@ func (p *SpecProcessor) pushMetrics() { fields[prefix+"read_count"] = io.ReadCount fields[prefix+"write_count"] = io.WriteCount fields[prefix+"read_bytes"] = io.ReadBytes - fields[prefix+"write_bytes"] = io.WriteCount + fields[prefix+"write_bytes"] = io.WriteBytes } cpu_time, err := p.proc.Times() From 9d3ad6309ed38ea7cefaf14419b5d10d8f99d8ee Mon Sep 17 00:00:00 2001 From: Cameron Sparr Date: Fri, 5 Aug 2016 13:55:02 +0100 Subject: [PATCH 02/28] Remove IF NOT EXISTS from influxdb output --- plugins/outputs/influxdb/influxdb.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/outputs/influxdb/influxdb.go b/plugins/outputs/influxdb/influxdb.go index 2b9fd101c..24065d114 100644 --- a/plugins/outputs/influxdb/influxdb.go +++ b/plugins/outputs/influxdb/influxdb.go @@ -146,7 +146,7 @@ func (i *InfluxDB) Connect() error { func createDatabase(c client.Client, database string) error { // Create Database if it doesn't exist _, err := c.Query(client.Query{ - Command: fmt.Sprintf("CREATE DATABASE IF NOT EXISTS \"%s\"", database), + Command: fmt.Sprintf("CREATE DATABASE \"%s\"", database), }) return err } From f0357b7a12d07c8ce535a6fe2bc6e8d290bccb28 Mon Sep 17 00:00:00 2001 From: Cameron Sparr Date: Fri, 5 Aug 2016 14:51:19 +0100 Subject: [PATCH 03/28] CHANGELOG formatting update put all 1.0 beta releases into a single 1.0 release manifest also add #1586 change --- CHANGELOG.md | 120 +++++++++++++++++---------------------------------- 1 file changed, 40 insertions(+), 80 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b7f49c33b..1812c65fd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,23 +1,5 @@ ## v1.0 [unreleased] -### Features - -- [#1413](https://github.com/influxdata/telegraf/issues/1413): Separate container_version from container_image tag. -- [#1525](https://github.com/influxdata/telegraf/pull/1525): Support setting per-device and total metrics for Docker network and blockio. -- [#1466](https://github.com/influxdata/telegraf/pull/1466): MongoDB input plugin: adding per DB stats from db.stats() - -### Bugfixes - -- [#1519](https://github.com/influxdata/telegraf/pull/1519): Fix error race conditions and partial failures. -- [#1477](https://github.com/influxdata/telegraf/issues/1477): nstat: fix inaccurate config panic. -- [#1481](https://github.com/influxdata/telegraf/issues/1481): jolokia: fix handling multiple multi-dimensional attributes. -- [#1430](https://github.com/influxdata/telegraf/issues/1430): Fix prometheus character sanitizing. Sanitize more win_perf_counters characters. -- [#1534](https://github.com/influxdata/telegraf/pull/1534): Add diskio io_time to FreeBSD & report timing metrics as ms (as linux does). -- [#1379](https://github.com/influxdata/telegraf/issues/1379): Fix covering Amazon Linux for post remove flow. -- [#1584](https://github.com/influxdata/telegraf/issues/1584): procstat missing fields: read/write bytes & count - -## v1.0 beta 3 [2016-07-18] - ### Release Notes **Breaking Change**: Aerospike main server node measurements have been renamed @@ -50,8 +32,15 @@ should now look like: path = "/" ``` +- `flush_jitter` behavior has been changed. The random jitter will now be +evaluated at every flush interval, rather than once at startup. This makes it +consistent with the behavior of `collection_jitter`. + ### Features +- [#1413](https://github.com/influxdata/telegraf/issues/1413): Separate container_version from container_image tag. +- [#1525](https://github.com/influxdata/telegraf/pull/1525): Support setting per-device and total metrics for Docker network and blockio. +- [#1466](https://github.com/influxdata/telegraf/pull/1466): MongoDB input plugin: adding per DB stats from db.stats() - [#1503](https://github.com/influxdata/telegraf/pull/1503): Add tls support for certs to RabbitMQ input plugin - [#1289](https://github.com/influxdata/telegraf/pull/1289): webhooks input plugin. Thanks @francois2metz and @cduez! - [#1247](https://github.com/influxdata/telegraf/pull/1247): rollbar webhook plugin. @@ -66,73 +55,11 @@ should now look like: - [#1434](https://github.com/influxdata/telegraf/pull/1434): Add measurement name arg to logparser plugin. - [#1479](https://github.com/influxdata/telegraf/pull/1479): logparser: change resp_code from a field to a tag. - [#1411](https://github.com/influxdata/telegraf/pull/1411): Implement support for fetching hddtemp data - -### Bugfixes - -- [#1472](https://github.com/influxdata/telegraf/pull/1472): diskio input plugin: set 'skip_serial_number = true' by default to avoid high cardinality. -- [#1426](https://github.com/influxdata/telegraf/pull/1426): nil metrics panic fix. -- [#1384](https://github.com/influxdata/telegraf/pull/1384): Fix datarace in apache input plugin. -- [#1399](https://github.com/influxdata/telegraf/issues/1399): Add `read_repairs` statistics to riak plugin. -- [#1405](https://github.com/influxdata/telegraf/issues/1405): Fix memory/connection leak in prometheus input plugin. -- [#1378](https://github.com/influxdata/telegraf/issues/1378): Trim BOM from config file for Windows support. -- [#1339](https://github.com/influxdata/telegraf/issues/1339): Prometheus client output panic on service reload. -- [#1461](https://github.com/influxdata/telegraf/pull/1461): Prometheus parser, protobuf format header fix. -- [#1334](https://github.com/influxdata/telegraf/issues/1334): Prometheus output, metric refresh and caching fixes. -- [#1432](https://github.com/influxdata/telegraf/issues/1432): Panic fix for multiple graphite outputs under very high load. -- [#1412](https://github.com/influxdata/telegraf/pull/1412): Instrumental output has better reconnect behavior -- [#1460](https://github.com/influxdata/telegraf/issues/1460): Remove PID from procstat plugin to fix cardinality issues. -- [#1427](https://github.com/influxdata/telegraf/issues/1427): Cassandra input: version 2.x "column family" fix. -- [#1463](https://github.com/influxdata/telegraf/issues/1463): Shared WaitGroup in Exec plugin -- [#1436](https://github.com/influxdata/telegraf/issues/1436): logparser: honor modifiers in "pattern" config. -- [#1418](https://github.com/influxdata/telegraf/issues/1418): logparser: error and exit on file permissions/missing errors. -- [#1499](https://github.com/influxdata/telegraf/pull/1499): Make the user able to specify full path for HAproxy stats -- [#1521](https://github.com/influxdata/telegraf/pull/1521): Fix Redis url, an extra "tcp://" was added. - -## v1.0 beta 2 [2016-06-21] - -### Features - - [#1340](https://github.com/influxdata/telegraf/issues/1340): statsd: do not log every dropped metric. - [#1368](https://github.com/influxdata/telegraf/pull/1368): Add precision rounding to all metrics on collection. - [#1390](https://github.com/influxdata/telegraf/pull/1390): Add support for Tengine - [#1320](https://github.com/influxdata/telegraf/pull/1320): Logparser input plugin for parsing grok-style log patterns. - [#1397](https://github.com/influxdata/telegraf/issues/1397): ElasticSearch: now supports connecting to ElasticSearch via SSL - -### Bugfixes - -- [#1330](https://github.com/influxdata/telegraf/issues/1330): Fix exec plugin panic when using single binary. -- [#1336](https://github.com/influxdata/telegraf/issues/1336): Fixed incorrect prometheus metrics source selection. -- [#1112](https://github.com/influxdata/telegraf/issues/1112): Set default Zookeeper chroot to empty string. -- [#1335](https://github.com/influxdata/telegraf/issues/1335): Fix overall ping timeout to be calculated based on per-ping timeout. -- [#1374](https://github.com/influxdata/telegraf/pull/1374): Change "default" retention policy to "". -- [#1377](https://github.com/influxdata/telegraf/issues/1377): Graphite output mangling '%' character. -- [#1396](https://github.com/influxdata/telegraf/pull/1396): Prometheus input plugin now supports x509 certs authentication - -## v1.0 beta 1 [2016-06-07] - -### Release Notes - -- `flush_jitter` behavior has been changed. The random jitter will now be -evaluated at every flush interval, rather than once at startup. This makes it -consistent with the behavior of `collection_jitter`. - -- All AWS plugins now utilize a standard mechanism for evaluating credentials. -This allows all AWS plugins to support environment variables, shared credential -files & profiles, and role assumptions. See the specific plugin README for -details. - -- The AWS CloudWatch input plugin can now declare a wildcard value for a metric -dimension. This causes the plugin to read all metrics that contain the specified -dimension key regardless of value. This is used to export collections of metrics -without having to know the dimension values ahead of time. - -- The AWS CloudWatch input plugin can now be configured with the `cache_ttl` -attribute. This configures the TTL of the internal metric cache. This is useful -in conjunction with wildcard dimension values as it will control the amount of -time before a new metric is included by the plugin. - -### Features - - [#1262](https://github.com/influxdata/telegraf/pull/1261): Add graylog input pluging. - [#1294](https://github.com/influxdata/telegraf/pull/1294): consul input plugin. Thanks @harnash - [#1164](https://github.com/influxdata/telegraf/pull/1164): conntrack input plugin. Thanks @robinpercy! @@ -151,6 +78,38 @@ time before a new metric is included by the plugin. ### Bugfixes +- [#1519](https://github.com/influxdata/telegraf/pull/1519): Fix error race conditions and partial failures. +- [#1477](https://github.com/influxdata/telegraf/issues/1477): nstat: fix inaccurate config panic. +- [#1481](https://github.com/influxdata/telegraf/issues/1481): jolokia: fix handling multiple multi-dimensional attributes. +- [#1430](https://github.com/influxdata/telegraf/issues/1430): Fix prometheus character sanitizing. Sanitize more win_perf_counters characters. +- [#1534](https://github.com/influxdata/telegraf/pull/1534): Add diskio io_time to FreeBSD & report timing metrics as ms (as linux does). +- [#1379](https://github.com/influxdata/telegraf/issues/1379): Fix covering Amazon Linux for post remove flow. +- [#1584](https://github.com/influxdata/telegraf/issues/1584): procstat missing fields: read/write bytes & count +- [#1472](https://github.com/influxdata/telegraf/pull/1472): diskio input plugin: set 'skip_serial_number = true' by default to avoid high cardinality. +- [#1426](https://github.com/influxdata/telegraf/pull/1426): nil metrics panic fix. +- [#1384](https://github.com/influxdata/telegraf/pull/1384): Fix datarace in apache input plugin. +- [#1399](https://github.com/influxdata/telegraf/issues/1399): Add `read_repairs` statistics to riak plugin. +- [#1405](https://github.com/influxdata/telegraf/issues/1405): Fix memory/connection leak in prometheus input plugin. +- [#1378](https://github.com/influxdata/telegraf/issues/1378): Trim BOM from config file for Windows support. +- [#1339](https://github.com/influxdata/telegraf/issues/1339): Prometheus client output panic on service reload. +- [#1461](https://github.com/influxdata/telegraf/pull/1461): Prometheus parser, protobuf format header fix. +- [#1334](https://github.com/influxdata/telegraf/issues/1334): Prometheus output, metric refresh and caching fixes. +- [#1432](https://github.com/influxdata/telegraf/issues/1432): Panic fix for multiple graphite outputs under very high load. +- [#1412](https://github.com/influxdata/telegraf/pull/1412): Instrumental output has better reconnect behavior +- [#1460](https://github.com/influxdata/telegraf/issues/1460): Remove PID from procstat plugin to fix cardinality issues. +- [#1427](https://github.com/influxdata/telegraf/issues/1427): Cassandra input: version 2.x "column family" fix. +- [#1463](https://github.com/influxdata/telegraf/issues/1463): Shared WaitGroup in Exec plugin +- [#1436](https://github.com/influxdata/telegraf/issues/1436): logparser: honor modifiers in "pattern" config. +- [#1418](https://github.com/influxdata/telegraf/issues/1418): logparser: error and exit on file permissions/missing errors. +- [#1499](https://github.com/influxdata/telegraf/pull/1499): Make the user able to specify full path for HAproxy stats +- [#1521](https://github.com/influxdata/telegraf/pull/1521): Fix Redis url, an extra "tcp://" was added. +- [#1330](https://github.com/influxdata/telegraf/issues/1330): Fix exec plugin panic when using single binary. +- [#1336](https://github.com/influxdata/telegraf/issues/1336): Fixed incorrect prometheus metrics source selection. +- [#1112](https://github.com/influxdata/telegraf/issues/1112): Set default Zookeeper chroot to empty string. +- [#1335](https://github.com/influxdata/telegraf/issues/1335): Fix overall ping timeout to be calculated based on per-ping timeout. +- [#1374](https://github.com/influxdata/telegraf/pull/1374): Change "default" retention policy to "". +- [#1377](https://github.com/influxdata/telegraf/issues/1377): Graphite output mangling '%' character. +- [#1396](https://github.com/influxdata/telegraf/pull/1396): Prometheus input plugin now supports x509 certs authentication - [#1252](https://github.com/influxdata/telegraf/pull/1252) & [#1279](https://github.com/influxdata/telegraf/pull/1279): Fix systemd service. Thanks @zbindenren & @PierreF! - [#1221](https://github.com/influxdata/telegraf/pull/1221): Fix influxdb n_shards counter. - [#1258](https://github.com/influxdata/telegraf/pull/1258): Fix potential kernel plugin integer parse error. @@ -160,6 +119,7 @@ time before a new metric is included by the plugin. - [#1316](https://github.com/influxdata/telegraf/pull/1316): Removed leaked "database" tag on redis metrics. Thanks @PierreF! - [#1323](https://github.com/influxdata/telegraf/issues/1323): Processes plugin: fix potential error with /proc/net/stat directory. - [#1322](https://github.com/influxdata/telegraf/issues/1322): Fix rare RHEL 5.2 panic in gopsutil diskio gathering function. +- [#1586](https://github.com/influxdata/telegraf/pull/1586): Remove IF NOT EXISTS from influxdb output database creation. ## v0.13.1 [2016-05-24] From 49988b15a30204ebcf9f453533dc06d65fb11dc4 Mon Sep 17 00:00:00 2001 From: Cameron Sparr Date: Sat, 6 Aug 2016 07:39:59 +0100 Subject: [PATCH 04/28] Default config typo fix --- etc/telegraf.conf | 2 +- internal/config/config.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/etc/telegraf.conf b/etc/telegraf.conf index 338aa1b68..60877af20 100644 --- a/etc/telegraf.conf +++ b/etc/telegraf.conf @@ -55,7 +55,7 @@ ## By default, precision will be set to the same timestamp order as the ## collection interval, with the maximum being 1s. ## Precision will NOT be used for service inputs, such as logparser and statsd. - ## Valid values are "Nns", "Nus" (or "Nµs"), "Nms", "Ns". + ## Valid values are "ns", "us" (or "µs"), "ms", "s". precision = "" ## Run telegraf in debug mode debug = false diff --git a/internal/config/config.go b/internal/config/config.go index 9408d9efd..6823181e1 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -219,7 +219,7 @@ var header = `# Telegraf Configuration ## By default, precision will be set to the same timestamp order as the ## collection interval, with the maximum being 1s. ## Precision will NOT be used for service inputs, such as logparser and statsd. - ## Valid values are "Nns", "Nus" (or "Nµs"), "Nms", "Ns". + ## Valid values are "ns", "us" (or "µs"), "ms", "s". precision = "" ## Run telegraf in debug mode debug = false From d3bb1e70100919b720e4478ee6b9b59201c2e71c Mon Sep 17 00:00:00 2001 From: Cameron Sparr Date: Thu, 28 Jul 2016 12:31:11 +0100 Subject: [PATCH 05/28] Rename internal_models package to models --- agent/accumulator.go | 4 +-- agent/accumulator_test.go | 36 +++++++++++------------ agent/agent.go | 10 +++---- internal/config/config.go | 34 +++++++++++----------- internal/config/config_test.go | 40 +++++++++++++------------- internal/models/filter.go | 2 +- internal/models/filter_test.go | 2 +- internal/models/running_input.go | 2 +- internal/models/running_output.go | 2 +- internal/models/running_output_test.go | 2 +- 10 files changed, 67 insertions(+), 67 deletions(-) diff --git a/agent/accumulator.go b/agent/accumulator.go index d80affe68..f6863b745 100644 --- a/agent/accumulator.go +++ b/agent/accumulator.go @@ -12,7 +12,7 @@ import ( ) func NewAccumulator( - inputConfig *internal_models.InputConfig, + inputConfig *models.InputConfig, metrics chan telegraf.Metric, ) *accumulator { acc := accumulator{} @@ -31,7 +31,7 @@ type accumulator struct { // print every point added to the accumulator trace bool - inputConfig *internal_models.InputConfig + inputConfig *models.InputConfig precision time.Duration diff --git a/agent/accumulator_test.go b/agent/accumulator_test.go index 8618d327d..4dd69985f 100644 --- a/agent/accumulator_test.go +++ b/agent/accumulator_test.go @@ -21,7 +21,7 @@ func TestAdd(t *testing.T) { now := time.Now() a.metrics = make(chan telegraf.Metric, 10) defer close(a.metrics) - a.inputConfig = &internal_models.InputConfig{} + a.inputConfig = &models.InputConfig{} a.Add("acctest", float64(101), map[string]string{}) a.Add("acctest", float64(101), map[string]string{"acc": "test"}) @@ -47,7 +47,7 @@ func TestAddNoPrecisionWithInterval(t *testing.T) { now := time.Date(2006, time.February, 10, 12, 0, 0, 82912748, time.UTC) a.metrics = make(chan telegraf.Metric, 10) defer close(a.metrics) - a.inputConfig = &internal_models.InputConfig{} + a.inputConfig = &models.InputConfig{} a.SetPrecision(0, time.Second) a.Add("acctest", float64(101), map[string]string{}) @@ -74,7 +74,7 @@ func TestAddNoIntervalWithPrecision(t *testing.T) { now := time.Date(2006, time.February, 10, 12, 0, 0, 82912748, time.UTC) a.metrics = make(chan telegraf.Metric, 10) defer close(a.metrics) - a.inputConfig = &internal_models.InputConfig{} + a.inputConfig = &models.InputConfig{} a.SetPrecision(time.Second, time.Millisecond) a.Add("acctest", float64(101), map[string]string{}) @@ -101,7 +101,7 @@ func TestAddDisablePrecision(t *testing.T) { now := time.Date(2006, time.February, 10, 12, 0, 0, 82912748, time.UTC) a.metrics = make(chan telegraf.Metric, 10) defer close(a.metrics) - a.inputConfig = &internal_models.InputConfig{} + a.inputConfig = &models.InputConfig{} a.SetPrecision(time.Second, time.Millisecond) a.DisablePrecision() @@ -129,7 +129,7 @@ func TestDifferentPrecisions(t *testing.T) { now := time.Date(2006, time.February, 10, 12, 0, 0, 82912748, time.UTC) a.metrics = make(chan telegraf.Metric, 10) defer close(a.metrics) - a.inputConfig = &internal_models.InputConfig{} + a.inputConfig = &models.InputConfig{} a.SetPrecision(0, time.Second) a.Add("acctest", float64(101), map[string]string{"acc": "test"}, now) @@ -170,7 +170,7 @@ func TestAddDefaultTags(t *testing.T) { now := time.Now() a.metrics = make(chan telegraf.Metric, 10) defer close(a.metrics) - a.inputConfig = &internal_models.InputConfig{} + a.inputConfig = &models.InputConfig{} a.Add("acctest", float64(101), map[string]string{}) a.Add("acctest", float64(101), map[string]string{"acc": "test"}) @@ -196,7 +196,7 @@ func TestAddFields(t *testing.T) { now := time.Now() a.metrics = make(chan telegraf.Metric, 10) defer close(a.metrics) - a.inputConfig = &internal_models.InputConfig{} + a.inputConfig = &models.InputConfig{} fields := map[string]interface{}{ "usage": float64(99), @@ -229,7 +229,7 @@ func TestAddInfFields(t *testing.T) { now := time.Now() a.metrics = make(chan telegraf.Metric, 10) defer close(a.metrics) - a.inputConfig = &internal_models.InputConfig{} + a.inputConfig = &models.InputConfig{} fields := map[string]interface{}{ "usage": inf, @@ -257,7 +257,7 @@ func TestAddNaNFields(t *testing.T) { now := time.Now() a.metrics = make(chan telegraf.Metric, 10) defer close(a.metrics) - a.inputConfig = &internal_models.InputConfig{} + a.inputConfig = &models.InputConfig{} fields := map[string]interface{}{ "usage": nan, @@ -281,7 +281,7 @@ func TestAddUint64Fields(t *testing.T) { now := time.Now() a.metrics = make(chan telegraf.Metric, 10) defer close(a.metrics) - a.inputConfig = &internal_models.InputConfig{} + a.inputConfig = &models.InputConfig{} fields := map[string]interface{}{ "usage": uint64(99), @@ -310,7 +310,7 @@ func TestAddUint64Overflow(t *testing.T) { now := time.Now() a.metrics = make(chan telegraf.Metric, 10) defer close(a.metrics) - a.inputConfig = &internal_models.InputConfig{} + a.inputConfig = &models.InputConfig{} fields := map[string]interface{}{ "usage": uint64(9223372036854775808), @@ -340,7 +340,7 @@ func TestAddInts(t *testing.T) { now := time.Now() a.metrics = make(chan telegraf.Metric, 10) defer close(a.metrics) - a.inputConfig = &internal_models.InputConfig{} + a.inputConfig = &models.InputConfig{} a.Add("acctest", int(101), map[string]string{}) a.Add("acctest", int32(101), map[string]string{"acc": "test"}) @@ -367,7 +367,7 @@ func TestAddFloats(t *testing.T) { now := time.Now() a.metrics = make(chan telegraf.Metric, 10) defer close(a.metrics) - a.inputConfig = &internal_models.InputConfig{} + a.inputConfig = &models.InputConfig{} a.Add("acctest", float32(101), map[string]string{"acc": "test"}) a.Add("acctest", float64(101), map[string]string{"acc": "test"}, now) @@ -389,7 +389,7 @@ func TestAddStrings(t *testing.T) { now := time.Now() a.metrics = make(chan telegraf.Metric, 10) defer close(a.metrics) - a.inputConfig = &internal_models.InputConfig{} + a.inputConfig = &models.InputConfig{} a.Add("acctest", "test", map[string]string{"acc": "test"}) a.Add("acctest", "foo", map[string]string{"acc": "test"}, now) @@ -411,7 +411,7 @@ func TestAddBools(t *testing.T) { now := time.Now() a.metrics = make(chan telegraf.Metric, 10) defer close(a.metrics) - a.inputConfig = &internal_models.InputConfig{} + a.inputConfig = &models.InputConfig{} a.Add("acctest", true, map[string]string{"acc": "test"}) a.Add("acctest", false, map[string]string{"acc": "test"}, now) @@ -433,11 +433,11 @@ func TestAccFilterTags(t *testing.T) { now := time.Now() a.metrics = make(chan telegraf.Metric, 10) defer close(a.metrics) - filter := internal_models.Filter{ + filter := models.Filter{ TagExclude: []string{"acc"}, } assert.NoError(t, filter.CompileFilter()) - a.inputConfig = &internal_models.InputConfig{} + a.inputConfig = &models.InputConfig{} a.inputConfig.Filter = filter a.Add("acctest", float64(101), map[string]string{}) @@ -465,7 +465,7 @@ func TestAccAddError(t *testing.T) { defer log.SetOutput(os.Stderr) a := accumulator{} - a.inputConfig = &internal_models.InputConfig{} + a.inputConfig = &models.InputConfig{} a.inputConfig.Name = "mock_plugin" a.AddError(fmt.Errorf("foo")) diff --git a/agent/agent.go b/agent/agent.go index 5ee73512b..d86037e79 100644 --- a/agent/agent.go +++ b/agent/agent.go @@ -88,7 +88,7 @@ func (a *Agent) Close() error { return err } -func panicRecover(input *internal_models.RunningInput) { +func panicRecover(input *models.RunningInput) { if err := recover(); err != nil { trace := make([]byte, 2048) runtime.Stack(trace, true) @@ -104,7 +104,7 @@ func panicRecover(input *internal_models.RunningInput) { // reporting interval. func (a *Agent) gatherer( shutdown chan struct{}, - input *internal_models.RunningInput, + input *models.RunningInput, interval time.Duration, metricC chan telegraf.Metric, ) error { @@ -152,7 +152,7 @@ func (a *Agent) gatherer( // over. func gatherWithTimeout( shutdown chan struct{}, - input *internal_models.RunningInput, + input *models.RunningInput, acc *accumulator, timeout time.Duration, ) { @@ -240,7 +240,7 @@ func (a *Agent) flush() { wg.Add(len(a.Config.Outputs)) for _, o := range a.Config.Outputs { - go func(output *internal_models.RunningOutput) { + go func(output *models.RunningOutput) { defer wg.Done() err := output.Write() if err != nil { @@ -351,7 +351,7 @@ func (a *Agent) Run(shutdown chan struct{}) error { if input.Config.Interval != 0 { interval = input.Config.Interval } - go func(in *internal_models.RunningInput, interv time.Duration) { + go func(in *models.RunningInput, interv time.Duration) { defer wg.Done() if err := a.gatherer(shutdown, in, interv, metricC); err != nil { log.Printf(err.Error()) diff --git a/internal/config/config.go b/internal/config/config.go index 6823181e1..0de91277b 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -47,8 +47,8 @@ type Config struct { OutputFilters []string Agent *AgentConfig - Inputs []*internal_models.RunningInput - Outputs []*internal_models.RunningOutput + Inputs []*models.RunningInput + Outputs []*models.RunningOutput } func NewConfig() *Config { @@ -61,8 +61,8 @@ func NewConfig() *Config { }, Tags: make(map[string]string), - Inputs: make([]*internal_models.RunningInput, 0), - Outputs: make([]*internal_models.RunningOutput, 0), + Inputs: make([]*models.RunningInput, 0), + Outputs: make([]*models.RunningOutput, 0), InputFilters: make([]string, 0), OutputFilters: make([]string, 0), } @@ -598,7 +598,7 @@ func (c *Config) addOutput(name string, table *ast.Table) error { return err } - ro := internal_models.NewRunningOutput(name, output, outputConfig, + ro := models.NewRunningOutput(name, output, outputConfig, c.Agent.MetricBatchSize, c.Agent.MetricBufferLimit) c.Outputs = append(c.Outputs, ro) return nil @@ -639,7 +639,7 @@ func (c *Config) addInput(name string, table *ast.Table) error { return err } - rp := &internal_models.RunningInput{ + rp := &models.RunningInput{ Name: name, Input: input, Config: pluginConfig, @@ -650,10 +650,10 @@ func (c *Config) addInput(name string, table *ast.Table) error { // buildFilter builds a Filter // (tagpass/tagdrop/namepass/namedrop/fieldpass/fielddrop) to -// be inserted into the internal_models.OutputConfig/internal_models.InputConfig +// be inserted into the models.OutputConfig/models.InputConfig // to be used for glob filtering on tags and measurements -func buildFilter(tbl *ast.Table) (internal_models.Filter, error) { - f := internal_models.Filter{} +func buildFilter(tbl *ast.Table) (models.Filter, error) { + f := models.Filter{} if node, ok := tbl.Fields["namepass"]; ok { if kv, ok := node.(*ast.KeyValue); ok { @@ -717,7 +717,7 @@ func buildFilter(tbl *ast.Table) (internal_models.Filter, error) { if subtbl, ok := node.(*ast.Table); ok { for name, val := range subtbl.Fields { if kv, ok := val.(*ast.KeyValue); ok { - tagfilter := &internal_models.TagFilter{Name: name} + tagfilter := &models.TagFilter{Name: name} if ary, ok := kv.Value.(*ast.Array); ok { for _, elem := range ary.Value { if str, ok := elem.(*ast.String); ok { @@ -736,7 +736,7 @@ func buildFilter(tbl *ast.Table) (internal_models.Filter, error) { if subtbl, ok := node.(*ast.Table); ok { for name, val := range subtbl.Fields { if kv, ok := val.(*ast.KeyValue); ok { - tagfilter := &internal_models.TagFilter{Name: name} + tagfilter := &models.TagFilter{Name: name} if ary, ok := kv.Value.(*ast.Array); ok { for _, elem := range ary.Value { if str, ok := elem.(*ast.String); ok { @@ -793,9 +793,9 @@ func buildFilter(tbl *ast.Table) (internal_models.Filter, error) { // buildInput parses input specific items from the ast.Table, // builds the filter and returns a -// internal_models.InputConfig to be inserted into internal_models.RunningInput -func buildInput(name string, tbl *ast.Table) (*internal_models.InputConfig, error) { - cp := &internal_models.InputConfig{Name: name} +// models.InputConfig to be inserted into models.RunningInput +func buildInput(name string, tbl *ast.Table) (*models.InputConfig, error) { + cp := &models.InputConfig{Name: name} if node, ok := tbl.Fields["interval"]; ok { if kv, ok := node.(*ast.KeyValue); ok { if str, ok := kv.Value.(*ast.String); ok { @@ -969,14 +969,14 @@ func buildSerializer(name string, tbl *ast.Table) (serializers.Serializer, error // buildOutput parses output specific items from the ast.Table, // builds the filter and returns an -// internal_models.OutputConfig to be inserted into internal_models.RunningInput +// models.OutputConfig to be inserted into models.RunningInput // Note: error exists in the return for future calls that might require error -func buildOutput(name string, tbl *ast.Table) (*internal_models.OutputConfig, error) { +func buildOutput(name string, tbl *ast.Table) (*models.OutputConfig, error) { filter, err := buildFilter(tbl) if err != nil { return nil, err } - oc := &internal_models.OutputConfig{ + oc := &models.OutputConfig{ Name: name, Filter: filter, } diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 1659cd6ec..cb8c9192c 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -26,19 +26,19 @@ func TestConfig_LoadSingleInputWithEnvVars(t *testing.T) { memcached := inputs.Inputs["memcached"]().(*memcached.Memcached) memcached.Servers = []string{"192.168.1.1"} - filter := internal_models.Filter{ + filter := models.Filter{ NameDrop: []string{"metricname2"}, NamePass: []string{"metricname1"}, FieldDrop: []string{"other", "stuff"}, FieldPass: []string{"some", "strings"}, - TagDrop: []internal_models.TagFilter{ - internal_models.TagFilter{ + TagDrop: []models.TagFilter{ + models.TagFilter{ Name: "badtag", Filter: []string{"othertag"}, }, }, - TagPass: []internal_models.TagFilter{ - internal_models.TagFilter{ + TagPass: []models.TagFilter{ + models.TagFilter{ Name: "goodtag", Filter: []string{"mytag"}, }, @@ -46,7 +46,7 @@ func TestConfig_LoadSingleInputWithEnvVars(t *testing.T) { IsActive: true, } assert.NoError(t, filter.CompileFilter()) - mConfig := &internal_models.InputConfig{ + mConfig := &models.InputConfig{ Name: "memcached", Filter: filter, Interval: 10 * time.Second, @@ -66,19 +66,19 @@ func TestConfig_LoadSingleInput(t *testing.T) { memcached := inputs.Inputs["memcached"]().(*memcached.Memcached) memcached.Servers = []string{"localhost"} - filter := internal_models.Filter{ + filter := models.Filter{ NameDrop: []string{"metricname2"}, NamePass: []string{"metricname1"}, FieldDrop: []string{"other", "stuff"}, FieldPass: []string{"some", "strings"}, - TagDrop: []internal_models.TagFilter{ - internal_models.TagFilter{ + TagDrop: []models.TagFilter{ + models.TagFilter{ Name: "badtag", Filter: []string{"othertag"}, }, }, - TagPass: []internal_models.TagFilter{ - internal_models.TagFilter{ + TagPass: []models.TagFilter{ + models.TagFilter{ Name: "goodtag", Filter: []string{"mytag"}, }, @@ -86,7 +86,7 @@ func TestConfig_LoadSingleInput(t *testing.T) { IsActive: true, } assert.NoError(t, filter.CompileFilter()) - mConfig := &internal_models.InputConfig{ + mConfig := &models.InputConfig{ Name: "memcached", Filter: filter, Interval: 5 * time.Second, @@ -113,19 +113,19 @@ func TestConfig_LoadDirectory(t *testing.T) { memcached := inputs.Inputs["memcached"]().(*memcached.Memcached) memcached.Servers = []string{"localhost"} - filter := internal_models.Filter{ + filter := models.Filter{ NameDrop: []string{"metricname2"}, NamePass: []string{"metricname1"}, FieldDrop: []string{"other", "stuff"}, FieldPass: []string{"some", "strings"}, - TagDrop: []internal_models.TagFilter{ - internal_models.TagFilter{ + TagDrop: []models.TagFilter{ + models.TagFilter{ Name: "badtag", Filter: []string{"othertag"}, }, }, - TagPass: []internal_models.TagFilter{ - internal_models.TagFilter{ + TagPass: []models.TagFilter{ + models.TagFilter{ Name: "goodtag", Filter: []string{"mytag"}, }, @@ -133,7 +133,7 @@ func TestConfig_LoadDirectory(t *testing.T) { IsActive: true, } assert.NoError(t, filter.CompileFilter()) - mConfig := &internal_models.InputConfig{ + mConfig := &models.InputConfig{ Name: "memcached", Filter: filter, Interval: 5 * time.Second, @@ -150,7 +150,7 @@ func TestConfig_LoadDirectory(t *testing.T) { assert.NoError(t, err) ex.SetParser(p) ex.Command = "/usr/bin/myothercollector --foo=bar" - eConfig := &internal_models.InputConfig{ + eConfig := &models.InputConfig{ Name: "exec", MeasurementSuffix: "_myothercollector", } @@ -169,7 +169,7 @@ func TestConfig_LoadDirectory(t *testing.T) { pstat := inputs.Inputs["procstat"]().(*procstat.Procstat) pstat.PidFile = "/var/run/grafana-server.pid" - pConfig := &internal_models.InputConfig{Name: "procstat"} + pConfig := &models.InputConfig{Name: "procstat"} pConfig.Tags = make(map[string]string) assert.Equal(t, pstat, c.Inputs[3].Input, diff --git a/internal/models/filter.go b/internal/models/filter.go index ac24ec667..9ad4c0049 100644 --- a/internal/models/filter.go +++ b/internal/models/filter.go @@ -1,4 +1,4 @@ -package internal_models +package models import ( "fmt" diff --git a/internal/models/filter_test.go b/internal/models/filter_test.go index 454f10c45..497d08532 100644 --- a/internal/models/filter_test.go +++ b/internal/models/filter_test.go @@ -1,4 +1,4 @@ -package internal_models +package models import ( "testing" diff --git a/internal/models/running_input.go b/internal/models/running_input.go index cffaf336c..445c5ee96 100644 --- a/internal/models/running_input.go +++ b/internal/models/running_input.go @@ -1,4 +1,4 @@ -package internal_models +package models import ( "time" diff --git a/internal/models/running_output.go b/internal/models/running_output.go index 42025912c..82a6885d5 100644 --- a/internal/models/running_output.go +++ b/internal/models/running_output.go @@ -1,4 +1,4 @@ -package internal_models +package models import ( "log" diff --git a/internal/models/running_output_test.go b/internal/models/running_output_test.go index d9238c5a4..a552629e9 100644 --- a/internal/models/running_output_test.go +++ b/internal/models/running_output_test.go @@ -1,4 +1,4 @@ -package internal_models +package models import ( "fmt" From 22c293de62ac577cad219274daf8a99e6535bb77 Mon Sep 17 00:00:00 2001 From: Jack Zampolin Date: Mon, 8 Aug 2016 15:06:03 -0700 Subject: [PATCH 06/28] Add request for sample queries (#1608) --- CONTRIBUTING.md | 1 + plugins/inputs/EXAMPLE_README.md | 8 ++++++++ plugins/inputs/mock_Plugin.go | 12 ++++++++++++ 3 files changed, 21 insertions(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a639e91f9..8aeb3a614 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -11,6 +11,7 @@ Output plugins READMEs are less structured, but any information you can provide on how the data will look is appreciated. See the [OpenTSDB output](https://github.com/influxdata/telegraf/tree/master/plugins/outputs/opentsdb) for a good example. +1. **Optional:** Help users of your plugin by including example queries for populating dashboards. Include these sample queries in the `README.md` for the plugin. 1. **Optional:** Write a [tickscript](https://docs.influxdata.com/kapacitor/v1.0/tick/syntax/) for your plugin and add it to [Kapacitor](https://github.com/influxdata/kapacitor/tree/master/examples/telegraf). Or mention @jackzampolin in a PR comment with some common queries that you would want to alert on and he will write one for you. ## GoDoc diff --git a/plugins/inputs/EXAMPLE_README.md b/plugins/inputs/EXAMPLE_README.md index 6bebf1e88..d6fcfdb91 100644 --- a/plugins/inputs/EXAMPLE_README.md +++ b/plugins/inputs/EXAMPLE_README.md @@ -27,6 +27,14 @@ The example plugin gathers metrics about example things - tag2 - measurement2 has the following tags: - tag3 + +### Sample Queries: + +These are some useful queries (to generate dashboards or other) to run against data from this plugin: + +``` +SELECT max(field1), mean(field1), min(field1) FROM measurement1 WHERE tag1=bar AND time > now() - 1h GROUP BY tag +``` ### Example Output: diff --git a/plugins/inputs/mock_Plugin.go b/plugins/inputs/mock_Plugin.go index caf30f72f..4dec121bc 100644 --- a/plugins/inputs/mock_Plugin.go +++ b/plugins/inputs/mock_Plugin.go @@ -6,10 +6,22 @@ import ( "github.com/stretchr/testify/mock" ) +// MockPlugin struct should be named the same as the Plugin type MockPlugin struct { mock.Mock } +// Description will appear directly above the plugin definition in the config file +func (m *MockPlugin) Description() string { + return `This is an example plugin` +} + +// SampleConfig will populate the sample configuration portion of the plugin's configuration +func (m *MockPlugin) SampleConfig() string { + return ` sampleVar = 'foo'` +} + +// Gather defines what data the plugin will gather. func (m *MockPlugin) Gather(_a0 telegraf.Accumulator) error { ret := m.Called(_a0) From b0ef506a88ea4410a6588109915cfc74a99ae3dd Mon Sep 17 00:00:00 2001 From: Jack Zampolin Date: Mon, 8 Aug 2016 15:10:07 -0700 Subject: [PATCH 07/28] Add Kafka output readme (#1609) --- plugins/outputs/kafka/README.md | 67 +++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100644 plugins/outputs/kafka/README.md diff --git a/plugins/outputs/kafka/README.md b/plugins/outputs/kafka/README.md new file mode 100644 index 000000000..390407e14 --- /dev/null +++ b/plugins/outputs/kafka/README.md @@ -0,0 +1,67 @@ +# Kafka Producer Output Plugin + +This plugin writes to a [Kafka Broker](http://kafka.apache.org/07/quickstart.html) acting a Kafka Producer. + +``` +[[outputs.kafka]] + ## URLs of kafka brokers + brokers = ["localhost:9092"] + ## Kafka topic for producer messages + topic = "telegraf" + ## Telegraf tag to use as a routing key + ## ie, if this tag exists, it's value will be used as the routing key + routing_tag = "host" + + ## CompressionCodec represents the various compression codecs recognized by + ## Kafka in messages. + ## 0 : No compression + ## 1 : Gzip compression + ## 2 : Snappy compression + compression_codec = 0 + + ## RequiredAcks is used in Produce Requests to tell the broker how many + ## replica acknowledgements it must see before responding + ## 0 : the producer never waits for an acknowledgement from the broker. + ## This option provides the lowest latency but the weakest durability + ## guarantees (some data will be lost when a server fails). + ## 1 : the producer gets an acknowledgement after the leader replica has + ## received the data. This option provides better durability as the + ## client waits until the server acknowledges the request as successful + ## (only messages that were written to the now-dead leader but not yet + ## replicated will be lost). + ## -1: the producer gets an acknowledgement after all in-sync replicas have + ## received the data. This option provides the best durability, we + ## guarantee that no messages will be lost as long as at least one in + ## sync replica remains. + required_acks = -1 + + ## The total number of times to retry sending a message + max_retry = 3 + + ## Optional SSL Config + # ssl_ca = "/etc/telegraf/ca.pem" + # ssl_cert = "/etc/telegraf/cert.pem" + # ssl_key = "/etc/telegraf/key.pem" + ## Use SSL but skip chain & host verification + # insecure_skip_verify = false + + data_format = "influx" +``` + +### Required parameters: + +* `brokers`: List of strings, this is for speaking to a cluster of `kafka` brokers. On each flush interval, Telegraf will randomly choose one of the urls to write to. Each URL should just include host and port e.g. -> `["{host}:{port}","{host2}:{port2}"]` +* `topic`: The `kafka` topic to publish to. + + +### Optional parameters: + +* `routing_tag`: if this tag exists, it's value will be used as the routing key +* `compression_codec`: What level of compression to use: `0` -> no compression, `1` -> gzip compression, `2` -> snappy compression +* `required_acks`: a setting for how may `acks` required from the `kafka` broker cluster. +* `max_retry`: Max number of times to retry failed write +* `ssl_ca`: SSL CA +* `ssl_cert`: SSL CERT +* `ssl_key`: SSL key +* `insecure_skip_verify`: Use SSL but skip chain & host verification (default: false) +* `data_format`: [About Telegraf data formats](https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_OUTPUT.md) From 7b6713b094d92c9d868459ceded7689993479884 Mon Sep 17 00:00:00 2001 From: Dennis Bellinger Date: Fri, 15 Jul 2016 17:00:16 -0400 Subject: [PATCH 08/28] Telegraf support for built-in windows service. Updated windows dependencies Updated the windows dependencies so that the versions matched the dependencies for Mac OS and Linux. Additionally added some that were complained about being missing at compile time. Incorporated kardianos/service for management Incorporated the library github.com/kardianos/service to manage the service on the various platforms (including Windows). This required an alternate main function. The original main function was renamed to reloadLoop (as that is what the main loop in it does) (it also got a couple of parameters). The service management library calls it as the main body of the program. Merged service.go into telegraf.go Due to compilation issues on Windows, moved the code from service.go into telegraf.go and removed service.go entirely. Updated dependencies and fixed Windows service Updated the dependencies so that it builds properly on Windows, additionally, fixed the registered command for starting it as a service (needed to add the config file option). This currently standardizes it as a C:\telegraf\telegraf.conf on Windows. Added dependency for github.com/kardianos/service Removed the common dependencies from _windows file Removed all the common dependencies from the Godeps_windows file and modified Makefile to load Godeps and then Godeps_windows when building for Windows. This should reduce problems caused by the Godeps_windows file being forgotten when updating dependencies. Updated CHANGELOG.md with changes Ran `go fmt ./...` to format code Removed service library on all but Windows The service library [kardianos/service](github.com/kardianos/service) has been disabled on all platforms but windows, as there is already existing infrastructure for other platforms. Removed the dependency line for itself It appears that gdm accidentally added the project itself to the dependency list. This caused the dependency restoration to select an earlier version of the project during build. This only affected windows. This only affected builds after 020b2c70 Updated documentation for Windows Service Removed the documentation about using NSSM and added documentation on installing telegraf directly as a Windows Service. Added license info for kardianos/service Added the license information for github.com/kardianos/service which is licensed under the ZLib license, although that name is never mentioned the license text matches word for word. Changed the Windows Config file default location Updated the default location of the configuration file on Windows from C:\telegraf\telegraf.conf to C:\Program Files\Telegraf\telegraf.conf. With this change includes updating the directions, including directing that the executable be put into that same directory. Additionally, as noted in the instructions, the location of the config file for the service may be changed by specifying the location with the `-config` flag at install time. Fixed bug - Wrong data type: svcConfig svcConfig service.Config => svcConfig *service.Config (It needed to be a pointer) --- CHANGELOG.md | 16 +++++++ Godeps | 2 + Godeps_windows | 63 +++----------------------- Makefile | 1 + cmd/telegraf/telegraf.go | 80 ++++++++++++++++++++++++++++++++- docs/LICENSE_OF_DEPENDENCIES.md | 1 + docs/WINDOWS_SERVICE.md | 57 ++++++++++++----------- 7 files changed, 131 insertions(+), 89 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1812c65fd..febca9c8d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,21 @@ ## v1.0 [unreleased] +### Features + +- [#1413](https://github.com/influxdata/telegraf/issues/1413): Separate container_version from container_image tag. +- [#1525](https://github.com/influxdata/telegraf/pull/1525): Support setting per-device and total metrics for Docker network and blockio. +- [#860](https://github.com/influxdata/telegraf/issues/860): Make Telegraf run as a Windows service + +### Bugfixes + +- [#1519](https://github.com/influxdata/telegraf/pull/1519): Fix error race conditions and partial failures. +- [#1477](https://github.com/influxdata/telegraf/issues/1477): nstat: fix inaccurate config panic. +- [#1481](https://github.com/influxdata/telegraf/issues/1481): jolokia: fix handling multiple multi-dimensional attributes. +- [#1430](https://github.com/influxdata/telegraf/issues/1430): Fix prometheus character sanitizing. Sanitize more win_perf_counters characters. +- [#1534](https://github.com/influxdata/telegraf/pull/1534): Add diskio io_time to FreeBSD & report timing metrics as ms (as linux does). + +## v1.0 beta 3 [2016-07-18] + ### Release Notes **Breaking Change**: Aerospike main server node measurements have been renamed diff --git a/Godeps b/Godeps index 3cb67336e..3c70bcaf8 100644 --- a/Godeps +++ b/Godeps @@ -29,6 +29,8 @@ github.com/hpcloud/tail b2940955ab8b26e19d43a43c4da0475dd81bdb56 github.com/influxdata/config b79f6829346b8d6e78ba73544b1e1038f1f1c9da github.com/influxdata/influxdb e094138084855d444195b252314dfee9eae34cab github.com/influxdata/toml af4df43894b16e3fd2b788d01bd27ad0776ef2d0 +github.com/kardianos/osext 29ae4ffbc9a6fe9fb2bc5029050ce6996ea1d3bc +github.com/kardianos/service 5e335590050d6d00f3aa270217d288dda1c94d0a github.com/klauspost/crc32 19b0b332c9e4516a6370a0456e6182c3b5036720 github.com/lib/pq e182dc4027e2ded4b19396d638610f2653295f36 github.com/matttproud/golang_protobuf_extensions d0c3fe89de86839aecf2e0579c40ba3bb336a453 diff --git a/Godeps_windows b/Godeps_windows index cc3077fd4..b75da1c32 100644 --- a/Godeps_windows +++ b/Godeps_windows @@ -1,59 +1,6 @@ -github.com/Microsoft/go-winio 9f57cbbcbcb41dea496528872a4f0e37a4f7ae98 -github.com/Shopify/sarama 8aadb476e66ca998f2f6bb3c993e9a2daa3666b9 -github.com/Sirupsen/logrus 219c8cb75c258c552e999735be6df753ffc7afdc +github.com/Microsoft/go-winio ce2922f643c8fd76b46cadc7f404a06282678b34 github.com/StackExchange/wmi f3e2bae1e0cb5aef83e319133eabfee30013a4a5 -github.com/amir/raidman 53c1b967405155bfc8758557863bf2e14f814687 -github.com/aws/aws-sdk-go 13a12060f716145019378a10e2806c174356b857 -github.com/beorn7/perks 3ac7bf7a47d159a033b107610db8a1b6575507a4 -github.com/cenkalti/backoff 4dc77674aceaabba2c7e3da25d4c823edfb73f99 -github.com/couchbase/go-couchbase cb664315a324d87d19c879d9cc67fda6be8c2ac1 -github.com/couchbase/gomemcached a5ea6356f648fec6ab89add00edd09151455b4b2 -github.com/couchbase/goutils 5823a0cbaaa9008406021dc5daf80125ea30bba6 -github.com/dancannon/gorethink e7cac92ea2bc52638791a021f212145acfedb1fc -github.com/davecgh/go-spew 5215b55f46b2b919f50a1df0eaa5886afe4e3b3d -github.com/docker/engine-api 8924d6900370b4c7e7984be5adc61f50a80d7537 -github.com/docker/go-connections f549a9393d05688dff0992ef3efd8bbe6c628aeb -github.com/docker/go-units 5d2041e26a699eaca682e2ea41c8f891e1060444 -github.com/eapache/go-resiliency b86b1ec0dd4209a588dc1285cdd471e73525c0b3 -github.com/eapache/queue ded5959c0d4e360646dc9e9908cff48666781367 -github.com/eclipse/paho.mqtt.golang 0f7a459f04f13a41b7ed752d47944528d4bf9a86 -github.com/go-ole/go-ole 50055884d646dd9434f16bbb5c9801749b9bafe4 -github.com/go-sql-driver/mysql 1fca743146605a172a266e1654e01e5cd5669bee -github.com/golang/protobuf 552c7b9542c194800fd493123b3798ef0a832032 -github.com/golang/snappy 427fb6fc07997f43afa32f35e850833760e489a7 -github.com/gonuts/go-shellquote e842a11b24c6abfb3dd27af69a17f482e4b483c2 -github.com/gorilla/context 1ea25387ff6f684839d82767c1733ff4d4d15d0a -github.com/gorilla/mux c9e326e2bdec29039a3761c07bece13133863e1e -github.com/hailocab/go-hostpool e80d13ce29ede4452c43dea11e79b9bc8a15b478 -github.com/influxdata/config b79f6829346b8d6e78ba73544b1e1038f1f1c9da -github.com/influxdata/influxdb e3fef5593c21644f2b43af55d6e17e70910b0e48 -github.com/influxdata/toml af4df43894b16e3fd2b788d01bd27ad0776ef2d0 -github.com/klauspost/crc32 19b0b332c9e4516a6370a0456e6182c3b5036720 -github.com/lib/pq e182dc4027e2ded4b19396d638610f2653295f36 -github.com/lxn/win 9a7734ea4db26bc593d52f6a8a957afdad39c5c1 -github.com/matttproud/golang_protobuf_extensions d0c3fe89de86839aecf2e0579c40ba3bb336a453 -github.com/miekg/dns cce6c130cdb92c752850880fd285bea1d64439dd -github.com/mreiferson/go-snappystream 028eae7ab5c4c9e2d1cb4c4ca1e53259bbe7e504 -github.com/naoina/go-stringutil 6b638e95a32d0c1131db0e7fe83775cbea4a0d0b -github.com/nats-io/nats b13fc9d12b0b123ebc374e6b808c6228ae4234a3 -github.com/nats-io/nuid 4f84f5f3b2786224e336af2e13dba0a0a80b76fa -github.com/nsqio/go-nsq 0b80d6f05e15ca1930e0c5e1d540ed627e299980 -github.com/prometheus/client_golang 18acf9993a863f4c4b40612e19cdd243e7c86831 -github.com/prometheus/client_model fa8ad6fec33561be4280a8f0514318c79d7f6cb6 -github.com/prometheus/common e8eabff8812b05acf522b45fdcd725a785188e37 -github.com/prometheus/procfs 406e5b7bfd8201a36e2bb5f7bdae0b03380c2ce8 -github.com/samuel/go-zookeeper 218e9c81c0dd8b3b18172b2bbfad92cc7d6db55f -github.com/shirou/gopsutil 1f32ce1bb380845be7f5d174ac641a2c592c0c42 -github.com/shirou/w32 ada3ba68f000aa1b58580e45c9d308fe0b7fc5c5 -github.com/soniah/gosnmp b1b4f885b12c5dcbd021c5cee1c904110de6db7d -github.com/streadway/amqp b4f3ceab0337f013208d31348b578d83c0064744 -github.com/stretchr/testify 1f4a1643a57e798696635ea4c126e9127adb7d3c -github.com/wvanbergen/kafka 46f9a1cf3f670edec492029fadded9c2d9e18866 -github.com/wvanbergen/kazoo-go 0f768712ae6f76454f987c3356177e138df258f8 -github.com/zensqlmonitor/go-mssqldb ffe5510c6fa5e15e6d983210ab501c815b56b363 -golang.org/x/net 6acef71eb69611914f7a30939ea9f6e194c78172 -golang.org/x/text a71fd10341b064c10f4a81ceac72bcf70f26ea34 -gopkg.in/dancannon/gorethink.v1 7d1af5be49cb5ecc7b177bf387d232050299d6ef -gopkg.in/fatih/pool.v2 cba550ebf9bce999a02e963296d4bc7a486cb715 -gopkg.in/mgo.v2 d90005c5262a3463800497ea5a89aed5fe22c886 -gopkg.in/yaml.v2 a83829b6f1293c91addabc89d0571c246397bbf4 +github.com/go-ole/go-ole be49f7c07711fcb603cff39e1de7c67926dc0ba7 +github.com/lxn/win 950a0e81e7678e63d8e6cd32412bdecb325ccd88 +github.com/shirou/w32 3c9377fc6748f222729a8270fe2775d149a249ad +golang.org/x/sys a646d33e2ee3172a661fc09bca23bb4889a41bc8 diff --git a/Makefile b/Makefile index ee96e10bd..19eccbb70 100644 --- a/Makefile +++ b/Makefile @@ -37,6 +37,7 @@ prepare: # Use the windows godeps file to prepare dependencies prepare-windows: go get github.com/sparrc/gdm + gdm restore gdm restore -f Godeps_windows # Run all docker containers necessary for unit tests diff --git a/cmd/telegraf/telegraf.go b/cmd/telegraf/telegraf.go index e3398511a..e78207257 100644 --- a/cmd/telegraf/telegraf.go +++ b/cmd/telegraf/telegraf.go @@ -6,6 +6,7 @@ import ( "log" "os" "os/signal" + "runtime" "strings" "syscall" @@ -15,6 +16,7 @@ import ( _ "github.com/influxdata/telegraf/plugins/inputs/all" "github.com/influxdata/telegraf/plugins/outputs" _ "github.com/influxdata/telegraf/plugins/outputs/all" + "github.com/kardianos/service" ) var fDebug = flag.Bool("debug", false, @@ -68,6 +70,7 @@ The flags are: -debug print metrics as they're generated to stdout -quiet run in quiet mode -version print the version to stdout + -service Control the service, ie, 'telegraf -service install (windows only)' In addition to the -config flag, telegraf will also load the config file from an environment variable or default location. Precedence is: @@ -94,7 +97,22 @@ Examples: telegraf -config telegraf.conf -input-filter cpu:mem -output-filter influxdb ` -func main() { +var logger service.Logger + +var stop chan struct{} + +var srvc service.Service +var svcConfig *service.Config + +type program struct{} + +func reloadLoop(stop chan struct{}, s service.Service) { + defer func() { + if service.Interactive() { + os.Exit(0) + } + return + }() reload := make(chan bool, 1) reload <- true for <-reload { @@ -156,6 +174,17 @@ func main() { return } + if *fService != "" && runtime.GOOS == "windows" { + if *fConfig != "" { + (*svcConfig).Arguments = []string{"-config", *fConfig} + } + err := service.Control(s, *fService) + if err != nil { + log.Fatal(err) + } + return + } + // If no other options are specified, load the config file and run. c := config.NewConfig() c.OutputFilters = outputFilters @@ -209,7 +238,8 @@ func main() { signals := make(chan os.Signal) signal.Notify(signals, os.Interrupt, syscall.SIGHUP) go func() { - sig := <-signals + select { + case sig := <-signals: if sig == os.Interrupt { close(shutdown) } @@ -217,6 +247,9 @@ func main() { log.Printf("Reloading Telegraf config\n") <-reload reload <- true + close(shutdown) + } + case <-stop: close(shutdown) } }() @@ -245,3 +278,46 @@ func usageExit(rc int) { fmt.Println(usage) os.Exit(rc) } + +func (p *program) Start(s service.Service) error { + srvc = s + go p.run() + return nil +} +func (p *program) run() { + stop = make(chan struct{}) + reloadLoop(stop, srvc) +} +func (p *program) Stop(s service.Service) error { + close(stop) + return nil +} + +func main() { + if runtime.GOOS == "windows" { + svcConfig = &service.Config{ + Name: "telegraf", + DisplayName: "Telegraf Data Collector Service", + Description: "Collects data using a series of plugins and publishes it to" + + "another series of plugins.", + Arguments: []string{"-config", "C:\\Program Files\\Telegraf\\telegraf.conf"}, + } + + prg := &program{} + s, err := service.New(prg, svcConfig) + if err != nil { + log.Fatal(err) + } + logger, err = s.Logger(nil) + if err != nil { + log.Fatal(err) + } + err = s.Run() + if err != nil { + logger.Error(err) + } + } else { + stop = make(chan struct{}) + reloadLoop(stop, nil) + } +} diff --git a/docs/LICENSE_OF_DEPENDENCIES.md b/docs/LICENSE_OF_DEPENDENCIES.md index d448872f6..5553fda70 100644 --- a/docs/LICENSE_OF_DEPENDENCIES.md +++ b/docs/LICENSE_OF_DEPENDENCIES.md @@ -16,6 +16,7 @@ - github.com/hashicorp/go-msgpack [BSD LICENSE](https://github.com/hashicorp/go-msgpack/blob/master/LICENSE) - github.com/hashicorp/raft [MPL LICENSE](https://github.com/hashicorp/raft/blob/master/LICENSE) - github.com/hashicorp/raft-boltdb [MPL LICENSE](https://github.com/hashicorp/raft-boltdb/blob/master/LICENSE) +- github.com/kardianos/service [ZLIB LICENSE](https://github.com/kardianos/service/blob/master/LICENSE) (License not named but matches word for word with ZLib) - github.com/lib/pq [MIT LICENSE](https://github.com/lib/pq/blob/master/LICENSE.md) - github.com/matttproud/golang_protobuf_extensions [APACHE LICENSE](https://github.com/matttproud/golang_protobuf_extensions/blob/master/LICENSE) - github.com/naoina/go-stringutil [MIT LICENSE](https://github.com/naoina/go-stringutil/blob/master/LICENSE) diff --git a/docs/WINDOWS_SERVICE.md b/docs/WINDOWS_SERVICE.md index 679a41527..646829159 100644 --- a/docs/WINDOWS_SERVICE.md +++ b/docs/WINDOWS_SERVICE.md @@ -1,36 +1,35 @@ # Running Telegraf as a Windows Service -If you have tried to install Go binaries as Windows Services with the **sc.exe** -tool you may have seen that the service errors and stops running after a while. +Telegraf natively supports running as a Windows Service. Outlined below is are +the general steps to set it up. -**NSSM** (the Non-Sucking Service Manager) is a tool that helps you in a -[number of scenarios](http://nssm.cc/scenarios) including running Go binaries -that were not specifically designed to run only in Windows platforms. +1. Obtain the telegraf windows distribution +2. Create the directory `C:\Program Files\Telegraf` (if you install in a different + location simply specify the `-config` parameter with the desired location) +3. Place the executable and the config file into `C:\Program Files\Telegraf` +4. To install the service into the Windows Service Manager, run (as an + administrator): + ```ps + C:\Program Files\Telegraf\telegraf.exe --service install + ``` +5. Edit the configuration file to meet your needs +6. To check that it works, run: + ```ps + C:\Program Files\Telegraf\telegraf.exe --config C:\Program Files\Telegraf\telegraf.conf --test + ``` +7. To start collecting data, run: + ```ps + net start telegraf + ``` -## NSSM Installation via Chocolatey +## Other supported operations -You can install [Chocolatey](https://chocolatey.org/) and [NSSM](http://nssm.cc/) -with these commands +Telegraf can manage its own service through the --service flag: -```powershell -iex ((new-object net.webclient).DownloadString('https://chocolatey.org/install.ps1')) -choco install -y nssm -``` +| Command | Effect | +|------------------------------------|-------------------------------| +| `telegraf.exe --service install` | Install telegraf as a service | +| `telegraf.exe --service uninstall` | Remove the telegraf service | +| `telegraf.exe --service start` | Start the telegraf service | +| `telegraf.exe --service stop` | Stop the telegraf service | -## Installing Telegraf as a Windows Service with NSSM - -You can download the latest Telegraf Windows binaries (still Experimental at -the moment) from [the Telegraf Github repo](https://github.com/influxdata/telegraf). - -Then you can create a C:\telegraf folder, unzip the binary there and modify the -**telegraf.conf** sample to allocate the metrics you want to send to **InfluxDB**. - -Once you have NSSM installed in your system, the process is quite straightforward. -You only need to type this command in your Windows shell - -```powershell -nssm install Telegraf c:\telegraf\telegraf.exe -config c:\telegraf\telegraf.config -``` - -And now your service will be installed in Windows and you will be able to start and -stop it gracefully \ No newline at end of file From 8e2252675689550c20ea4f49ec487cf9f7e8e242 Mon Sep 17 00:00:00 2001 From: Cameron Sparr Date: Mon, 8 Aug 2016 15:55:16 +0100 Subject: [PATCH 09/28] Adding c:\program files\telegraf\telegraf.conf this will now be the default config file location on windows, basically it is the windows equivalent of /etc/telegraf/telegraf.conf also updating the changelog closes #1543 --- CHANGELOG.md | 21 +++++---------------- cmd/telegraf/telegraf.go | 20 ++++++++++---------- docs/WINDOWS_SERVICE.md | 19 ++++++++++++------- internal/config/config.go | 4 ++++ 4 files changed, 31 insertions(+), 33 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index febca9c8d..2af382958 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,23 +1,11 @@ ## v1.0 [unreleased] -### Features - -- [#1413](https://github.com/influxdata/telegraf/issues/1413): Separate container_version from container_image tag. -- [#1525](https://github.com/influxdata/telegraf/pull/1525): Support setting per-device and total metrics for Docker network and blockio. -- [#860](https://github.com/influxdata/telegraf/issues/860): Make Telegraf run as a Windows service - -### Bugfixes - -- [#1519](https://github.com/influxdata/telegraf/pull/1519): Fix error race conditions and partial failures. -- [#1477](https://github.com/influxdata/telegraf/issues/1477): nstat: fix inaccurate config panic. -- [#1481](https://github.com/influxdata/telegraf/issues/1481): jolokia: fix handling multiple multi-dimensional attributes. -- [#1430](https://github.com/influxdata/telegraf/issues/1430): Fix prometheus character sanitizing. Sanitize more win_perf_counters characters. -- [#1534](https://github.com/influxdata/telegraf/pull/1534): Add diskio io_time to FreeBSD & report timing metrics as ms (as linux does). - -## v1.0 beta 3 [2016-07-18] - ### Release Notes +- Telegraf now supports being installed as an official windows service, +which can be installed via +`> C:\Program Files\Telegraf\telegraf.exe --service install` + **Breaking Change**: Aerospike main server node measurements have been renamed aerospike_node. Aerospike namespace measurements have been renamed to aerospike_namespace. They will also now be tagged with the node_name @@ -91,6 +79,7 @@ consistent with the behavior of `collection_jitter`. - [#1278](https://github.com/influxdata/telegraf/pull/1278) & [#1288](https://github.com/influxdata/telegraf/pull/1288) & [#1295](https://github.com/influxdata/telegraf/pull/1295): RabbitMQ/Apache/InfluxDB inputs: made url(s) parameter optional by using reasonable input defaults if not specified - [#1296](https://github.com/influxdata/telegraf/issues/1296): Refactor of flush_jitter argument. - [#1213](https://github.com/influxdata/telegraf/issues/1213): Add inactive & active memory to mem plugin. +- [#1543](https://github.com/influxdata/telegraf/pull/1543): Official Windows service. ### Bugfixes diff --git a/cmd/telegraf/telegraf.go b/cmd/telegraf/telegraf.go index e78207257..f19b127a8 100644 --- a/cmd/telegraf/telegraf.go +++ b/cmd/telegraf/telegraf.go @@ -41,6 +41,8 @@ var fOutputList = flag.Bool("output-list", false, "print available output plugins.") var fUsage = flag.String("usage", "", "print usage for a plugin, ie, 'telegraf -usage mysql'") +var fService = flag.String("service", "", + "operate on the service") // Telegraf version, populated linker. // ie, -ldflags "-X main.version=`git describe --always --tags`" @@ -172,9 +174,7 @@ func reloadLoop(stop chan struct{}, s service.Service) { } } return - } - - if *fService != "" && runtime.GOOS == "windows" { + case *fService != "" && runtime.GOOS == "windows": if *fConfig != "" { (*svcConfig).Arguments = []string{"-config", *fConfig} } @@ -240,13 +240,13 @@ func reloadLoop(stop chan struct{}, s service.Service) { go func() { select { case sig := <-signals: - if sig == os.Interrupt { - close(shutdown) - } - if sig == syscall.SIGHUP { - log.Printf("Reloading Telegraf config\n") - <-reload - reload <- true + if sig == os.Interrupt { + close(shutdown) + } + if sig == syscall.SIGHUP { + log.Printf("Reloading Telegraf config\n") + <-reload + reload <- true close(shutdown) } case <-stop: diff --git a/docs/WINDOWS_SERVICE.md b/docs/WINDOWS_SERVICE.md index 646829159..0ef218350 100644 --- a/docs/WINDOWS_SERVICE.md +++ b/docs/WINDOWS_SERVICE.md @@ -6,20 +6,25 @@ the general steps to set it up. 1. Obtain the telegraf windows distribution 2. Create the directory `C:\Program Files\Telegraf` (if you install in a different location simply specify the `-config` parameter with the desired location) -3. Place the executable and the config file into `C:\Program Files\Telegraf` +3. Place the telegraf.exe and the config file into `C:\Program Files\Telegraf` 4. To install the service into the Windows Service Manager, run (as an administrator): - ```ps - C:\Program Files\Telegraf\telegraf.exe --service install + ``` + > C:\Program Files\Telegraf\telegraf.exe --service install + ``` + 5. Edit the configuration file to meet your needs 6. To check that it works, run: - ```ps - C:\Program Files\Telegraf\telegraf.exe --config C:\Program Files\Telegraf\telegraf.conf --test + ``` + > C:\Program Files\Telegraf\telegraf.exe --config C:\Program Files\Telegraf\telegraf.conf --test + ``` + 7. To start collecting data, run: - ```ps - net start telegraf + + ``` + > net start telegraf ``` ## Other supported operations diff --git a/internal/config/config.go b/internal/config/config.go index 0de91277b..24c1af3fa 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -9,6 +9,7 @@ import ( "os" "path/filepath" "regexp" + "runtime" "sort" "strings" "time" @@ -432,6 +433,9 @@ func getDefaultConfigPath() (string, error) { envfile := os.Getenv("TELEGRAF_CONFIG_PATH") homefile := os.ExpandEnv("${HOME}/.telegraf/telegraf.conf") etcfile := "/etc/telegraf/telegraf.conf" + if runtime.GOOS == "windows" { + etcfile = `C:\Program Files\Telegraf\telegraf.conf` + } for _, path := range []string{envfile, homefile, etcfile} { if _, err := os.Stat(path); err == nil { log.Printf("Using config file: %s", path) From c99c22534b427b14a5d73dde06bcba83f9c0561f Mon Sep 17 00:00:00 2001 From: Cameron Sparr Date: Tue, 9 Aug 2016 07:50:35 +0100 Subject: [PATCH 10/28] influxdb output: config doc update --- plugins/outputs/influxdb/README.md | 40 +++++++++++++++++++++++++++--- 1 file changed, 37 insertions(+), 3 deletions(-) diff --git a/plugins/outputs/influxdb/README.md b/plugins/outputs/influxdb/README.md index b55a2c4c9..11511c3c4 100644 --- a/plugins/outputs/influxdb/README.md +++ b/plugins/outputs/influxdb/README.md @@ -2,6 +2,42 @@ This plugin writes to [InfluxDB](https://www.influxdb.com) via HTTP or UDP. +### Configuration: + +```toml +# Configuration for influxdb server to send metrics to +[[outputs.influxdb]] + ## The full HTTP or UDP endpoint URL for your InfluxDB instance. + ## Multiple urls can be specified as part of the same cluster, + ## this means that only ONE of the urls will be written to each interval. + # urls = ["udp://localhost:8089"] # UDP endpoint example + urls = ["http://localhost:8086"] # required + ## The target database for metrics (telegraf will create it if not exists). + database = "telegraf" # required + + ## Retention policy to write to. Empty string writes to the default rp. + retention_policy = "" + ## Write consistency (clusters only), can be: "any", "one", "quorom", "all" + write_consistency = "any" + + ## Write timeout (for the InfluxDB client), formatted as a string. + ## If not provided, will default to 5s. 0s means no timeout (not recommended). + timeout = "5s" + # username = "telegraf" + # password = "metricsmetricsmetricsmetrics" + ## Set the user agent for HTTP POSTs (can be useful for log differentiation) + # user_agent = "telegraf" + ## Set UDP payload size, defaults to InfluxDB UDP Client default (512 bytes) + # udp_payload = 512 + + ## Optional SSL Config + # ssl_ca = "/etc/telegraf/ca.pem" + # ssl_cert = "/etc/telegraf/cert.pem" + # ssl_key = "/etc/telegraf/key.pem" + ## Use SSL but skip chain & host verification + # insecure_skip_verify = false +``` + ### Required parameters: * `urls`: List of strings, this is for InfluxDB clustering @@ -12,16 +48,14 @@ to write to. Each URL should start with either `http://` or `udp://` ### Optional parameters: +* `write_consistency`: Write consistency (clusters only), can be: "any", "one", "quorom", "all". * `retention_policy`: Retention policy to write to. -* `precision`: Precision of writes, valid values are "ns", "us" (or "µs"), "ms", "s", "m", "h". note: using "s" precision greatly improves InfluxDB compression. * `timeout`: Write timeout (for the InfluxDB client), formatted as a string. If not provided, will default to 5s. 0s means no timeout (not recommended). * `username`: Username for influxdb * `password`: Password for influxdb * `user_agent`: Set the user agent for HTTP POSTs (can be useful for log differentiation) * `udp_payload`: Set UDP payload size, defaults to InfluxDB UDP Client default (512 bytes) - ## Optional SSL Config * `ssl_ca`: SSL CA * `ssl_cert`: SSL CERT * `ssl_key`: SSL key * `insecure_skip_verify`: Use SSL but skip chain & host verification (default: false) -* `write_consistency`: Write consistency for clusters only, can be: "any", "one", "quorom", "all" From 53e31cf1b5fb506ce05e3bceab87f4628dfe8f27 Mon Sep 17 00:00:00 2001 From: Patrick Hemmer Date: Tue, 9 Aug 2016 03:25:59 -0400 Subject: [PATCH 11/28] Fix postgres extensible text (#1601) * convert postgresql_extensible byte slice values to strings * code cleanup in postgresql_extensible --- CHANGELOG.md | 1 + .../postgresql_extensible.go | 34 +++++++++++-------- 2 files changed, 20 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2af382958..02c0d4bbd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -125,6 +125,7 @@ consistent with the behavior of `collection_jitter`. - [#1323](https://github.com/influxdata/telegraf/issues/1323): Processes plugin: fix potential error with /proc/net/stat directory. - [#1322](https://github.com/influxdata/telegraf/issues/1322): Fix rare RHEL 5.2 panic in gopsutil diskio gathering function. - [#1586](https://github.com/influxdata/telegraf/pull/1586): Remove IF NOT EXISTS from influxdb output database creation. +- [#1600](https://github.com/influxdata/telegraf/issues/1600): Fix quoting with text values in postgresql_extensible plugin. ## v0.13.1 [2016-05-24] diff --git a/plugins/inputs/postgresql_extensible/postgresql_extensible.go b/plugins/inputs/postgresql_extensible/postgresql_extensible.go index 75bc6b936..ec281fca2 100644 --- a/plugins/inputs/postgresql_extensible/postgresql_extensible.go +++ b/plugins/inputs/postgresql_extensible/postgresql_extensible.go @@ -266,29 +266,33 @@ func (p *Postgresql) accRow(meas_name string, row scanner, acc telegraf.Accumula tags := map[string]string{} tags["server"] = tagAddress tags["db"] = dbname.String() - var isATag int fields := make(map[string]interface{}) +COLUMN: for col, val := range columnMap { if acc.Debug() { log.Printf("postgresql_extensible: column: %s = %T: %s\n", col, *val, *val) } _, ignore := ignoredColumns[col] - if !ignore && *val != nil { - isATag = 0 - for tag := range p.AdditionalTags { - if col == p.AdditionalTags[tag] { - isATag = 1 - value_type_p := fmt.Sprintf(`%T`, *val) - if value_type_p == "[]uint8" { - tags[col] = fmt.Sprintf(`%s`, *val) - } else if value_type_p == "int64" { - tags[col] = fmt.Sprintf(`%v`, *val) - } - } + if ignore || *val == nil { + continue + } + for _, tag := range p.AdditionalTags { + if col != tag { + continue } - if isATag == 0 { - fields[col] = *val + switch v := (*val).(type) { + case []byte: + tags[col] = string(v) + case int64: + tags[col] = fmt.Sprintf("%d", v) } + continue COLUMN + } + + if v, ok := (*val).([]byte); ok { + fields[col] = string(v) + } else { + fields[col] = *val } } acc.AddFields(meas_name, fields, tags) From 3853d0d0656192d556d78ff91d5e18bd58f0d236 Mon Sep 17 00:00:00 2001 From: Mariusz Brzeski Date: Tue, 9 Aug 2016 09:27:30 +0200 Subject: [PATCH 12/28] Fix problem with metrics when ping return Destination net unreachable ( windows ) (#1561) * Fix problem with metrics when ping return Destination net unreachable Add test case TestUnreachablePingGather Add percent_reply_loss Fix some other tests * Add errors measurment * fir problem with ping reply "TTL expired in transit" ( use regex for more specific condition - TTL in line but it's a not valid replay ) add test case for "TTL expired in transit" - TestTTLExpiredPingGather --- plugins/inputs/ping/README.md | 36 +++++++ plugins/inputs/ping/ping_windows.go | 45 +++++--- plugins/inputs/ping/ping_windows_test.go | 126 +++++++++++++++++++++-- 3 files changed, 183 insertions(+), 24 deletions(-) create mode 100644 plugins/inputs/ping/README.md diff --git a/plugins/inputs/ping/README.md b/plugins/inputs/ping/README.md new file mode 100644 index 000000000..1f087c774 --- /dev/null +++ b/plugins/inputs/ping/README.md @@ -0,0 +1,36 @@ +# Ping input plugin + +This input plugin will measures the round-trip + +## Windows: +### Configration: +``` + ## urls to ping + urls = ["www.google.com"] # required + + ## number of pings to send per collection (ping -n ) + count = 4 # required + + ## Ping timeout, in seconds. 0 means default timeout (ping -w ) + Timeout = 0 +``` +### Measurements & Fields: +- packets_transmitted ( from ping output ) +- reply_received ( increasing only on valid metric from echo replay, eg. 'Destination net unreachable' reply will increment packets_received but not reply_received ) +- packets_received ( from ping output ) +- percent_reply_loss ( compute from packets_transmitted and reply_received ) +- percent_packets_loss ( compute from packets_transmitted and packets_received ) +- errors ( when host can not be found or wrong prameters is passed to application ) +- response time + - average_response_ms ( compute from minimum_response_ms and maximum_response_ms ) + - minimum_response_ms ( from ping output ) + - maximum_response_ms ( from ping output ) + +### Tags: +- server + +### Example Output: +``` +* Plugin: ping, Collection 1 +ping,host=WIN-PBAPLP511R7,url=www.google.com average_response_ms=7i,maximum_response_ms=9i,minimum_response_ms=7i,packets_received=4i,packets_transmitted=4i,percent_packet_loss=0,percent_reply_loss=0,reply_received=4i 1469879119000000000 +``` \ No newline at end of file diff --git a/plugins/inputs/ping/ping_windows.go b/plugins/inputs/ping/ping_windows.go index d36f44526..7fb112810 100644 --- a/plugins/inputs/ping/ping_windows.go +++ b/plugins/inputs/ping/ping_windows.go @@ -65,16 +65,20 @@ func hostPinger(timeout float64, args ...string) (string, error) { // processPingOutput takes in a string output from the ping command // based on linux implementation but using regex ( multilanguage support ) ( shouldn't affect the performance of the program ) -// It returns (, , , , ) -func processPingOutput(out string) (int, int, int, int, int, error) { +// It returns (, , , , , ) +func processPingOutput(out string) (int, int, int, int, int, int, error) { // So find a line contain 3 numbers except reply lines var stats, aproxs []string = nil, nil err := errors.New("Fatal error processing ping output") stat := regexp.MustCompile(`=\W*(\d+)\D*=\W*(\d+)\D*=\W*(\d+)`) aprox := regexp.MustCompile(`=\W*(\d+)\D*ms\D*=\W*(\d+)\D*ms\D*=\W*(\d+)\D*ms`) + tttLine := regexp.MustCompile(`TTL=\d+`) lines := strings.Split(out, "\n") + var receivedReply int = 0 for _, line := range lines { - if !strings.Contains(line, "TTL") { + if tttLine.MatchString(line) { + receivedReply++ + } else { if stats == nil { stats = stat.FindStringSubmatch(line) } @@ -86,35 +90,35 @@ func processPingOutput(out string) (int, int, int, int, int, error) { // stats data should contain 4 members: entireExpression + ( Send, Receive, Lost ) if len(stats) != 4 { - return 0, 0, 0, 0, 0, err + return 0, 0, 0, 0, 0, 0, err } trans, err := strconv.Atoi(stats[1]) if err != nil { - return 0, 0, 0, 0, 0, err + return 0, 0, 0, 0, 0, 0, err } - rec, err := strconv.Atoi(stats[2]) + receivedPacket, err := strconv.Atoi(stats[2]) if err != nil { - return 0, 0, 0, 0, 0, err + return 0, 0, 0, 0, 0, 0, err } // aproxs data should contain 4 members: entireExpression + ( min, max, avg ) if len(aproxs) != 4 { - return trans, rec, 0, 0, 0, err + return trans, receivedReply, receivedPacket, 0, 0, 0, err } min, err := strconv.Atoi(aproxs[1]) if err != nil { - return trans, rec, 0, 0, 0, err + return trans, receivedReply, receivedPacket, 0, 0, 0, err } max, err := strconv.Atoi(aproxs[2]) if err != nil { - return trans, rec, 0, 0, 0, err + return trans, receivedReply, receivedPacket, 0, 0, 0, err } avg, err := strconv.Atoi(aproxs[3]) if err != nil { - return 0, 0, 0, 0, 0, err + return 0, 0, 0, 0, 0, 0, err } - return trans, rec, avg, min, max, err + return trans, receivedReply, receivedPacket, avg, min, max, err } func (p *Ping) timeout() float64 { @@ -159,21 +163,30 @@ func (p *Ping) Gather(acc telegraf.Accumulator) error { pendingError = errors.New(strings.TrimSpace(out) + ", " + err.Error()) } tags := map[string]string{"url": u} - trans, rec, avg, min, max, err := processPingOutput(out) + trans, recReply, receivePacket, avg, min, max, err := processPingOutput(out) if err != nil { // fatal error if pendingError != nil { errorChannel <- pendingError } errorChannel <- err + fields := map[string]interface{}{ + "errors": 100.0, + } + + acc.AddFields("ping", fields, tags) + return } // Calculate packet loss percentage - loss := float64(trans-rec) / float64(trans) * 100.0 + lossReply := float64(trans-recReply) / float64(trans) * 100.0 + lossPackets := float64(trans-receivePacket) / float64(trans) * 100.0 fields := map[string]interface{}{ "packets_transmitted": trans, - "packets_received": rec, - "percent_packet_loss": loss, + "reply_received": recReply, + "packets_received": receivePacket, + "percent_packet_loss": lossPackets, + "percent_reply_loss": lossReply, } if avg > 0 { fields["average_response_ms"] = avg diff --git a/plugins/inputs/ping/ping_windows_test.go b/plugins/inputs/ping/ping_windows_test.go index a4d0609e6..34428b814 100644 --- a/plugins/inputs/ping/ping_windows_test.go +++ b/plugins/inputs/ping/ping_windows_test.go @@ -38,18 +38,20 @@ Approximate round trip times in milli-seconds: ` func TestHost(t *testing.T) { - trans, rec, avg, min, max, err := processPingOutput(winPLPingOutput) + trans, recReply, recPacket, avg, min, max, err := processPingOutput(winPLPingOutput) assert.NoError(t, err) assert.Equal(t, 4, trans, "4 packets were transmitted") - assert.Equal(t, 4, rec, "4 packets were received") + assert.Equal(t, 4, recReply, "4 packets were reply") + assert.Equal(t, 4, recPacket, "4 packets were received") assert.Equal(t, 50, avg, "Average 50") assert.Equal(t, 46, min, "Min 46") assert.Equal(t, 57, max, "max 57") - trans, rec, avg, min, max, err = processPingOutput(winENPingOutput) + trans, recReply, recPacket, avg, min, max, err = processPingOutput(winENPingOutput) assert.NoError(t, err) assert.Equal(t, 4, trans, "4 packets were transmitted") - assert.Equal(t, 4, rec, "4 packets were received") + assert.Equal(t, 4, recReply, "4 packets were reply") + assert.Equal(t, 4, recPacket, "4 packets were received") assert.Equal(t, 50, avg, "Average 50") assert.Equal(t, 50, min, "Min 50") assert.Equal(t, 52, max, "Max 52") @@ -72,7 +74,9 @@ func TestPingGather(t *testing.T) { fields := map[string]interface{}{ "packets_transmitted": 4, "packets_received": 4, + "reply_received": 4, "percent_packet_loss": 0.0, + "percent_reply_loss": 0.0, "average_response_ms": 50, "minimum_response_ms": 50, "maximum_response_ms": 52, @@ -113,7 +117,9 @@ func TestBadPingGather(t *testing.T) { fields := map[string]interface{}{ "packets_transmitted": 4, "packets_received": 0, + "reply_received": 0, "percent_packet_loss": 100.0, + "percent_reply_loss": 100.0, } acc.AssertContainsTaggedFields(t, "ping", fields, tags) } @@ -154,7 +160,9 @@ func TestLossyPingGather(t *testing.T) { fields := map[string]interface{}{ "packets_transmitted": 9, "packets_received": 7, + "reply_received": 7, "percent_packet_loss": 22.22222222222222, + "percent_reply_loss": 22.22222222222222, "average_response_ms": 115, "minimum_response_ms": 114, "maximum_response_ms": 119, @@ -207,12 +215,114 @@ func TestFatalPingGather(t *testing.T) { } p.Gather(&acc) - assert.False(t, acc.HasMeasurement("packets_transmitted"), + assert.True(t, acc.HasFloatField("ping", "errors"), + "Fatal ping should have packet measurements") + assert.False(t, acc.HasIntField("ping", "packets_transmitted"), "Fatal ping should not have packet measurements") - assert.False(t, acc.HasMeasurement("packets_received"), + assert.False(t, acc.HasIntField("ping", "packets_received"), "Fatal ping should not have packet measurements") - assert.False(t, acc.HasMeasurement("percent_packet_loss"), + assert.False(t, acc.HasFloatField("ping", "percent_packet_loss"), "Fatal ping should not have packet measurements") - assert.False(t, acc.HasMeasurement("average_response_ms"), + assert.False(t, acc.HasFloatField("ping", "percent_reply_loss"), + "Fatal ping should not have packet measurements") + assert.False(t, acc.HasIntField("ping", "average_response_ms"), + "Fatal ping should not have packet measurements") + assert.False(t, acc.HasIntField("ping", "maximum_response_ms"), + "Fatal ping should not have packet measurements") + assert.False(t, acc.HasIntField("ping", "minimum_response_ms"), + "Fatal ping should not have packet measurements") +} + +var UnreachablePingOutput = ` +Pinging www.google.pl [8.8.8.8] with 32 bytes of data: +Request timed out. +Request timed out. +Reply from 194.204.175.50: Destination net unreachable. +Request timed out. + +Ping statistics for 8.8.8.8: + Packets: Sent = 4, Received = 1, Lost = 3 (75% loss), +` + +func mockUnreachableHostPinger(timeout float64, args ...string) (string, error) { + return UnreachablePingOutput, errors.New("So very bad") +} + +//Reply from 185.28.251.217: TTL expired in transit. + +// in case 'Destination net unreachable' ping app return receive packet which is not what we need +// it's not contain valid metric so treat it as lost one +func TestUnreachablePingGather(t *testing.T) { + var acc testutil.Accumulator + p := Ping{ + Urls: []string{"www.google.com"}, + pingHost: mockUnreachableHostPinger, + } + + p.Gather(&acc) + + tags := map[string]string{"url": "www.google.com"} + fields := map[string]interface{}{ + "packets_transmitted": 4, + "packets_received": 1, + "reply_received": 0, + "percent_packet_loss": 75.0, + "percent_reply_loss": 100.0, + } + acc.AssertContainsTaggedFields(t, "ping", fields, tags) + + assert.False(t, acc.HasFloatField("ping", "errors"), + "Fatal ping should not have packet measurements") + assert.False(t, acc.HasIntField("ping", "average_response_ms"), + "Fatal ping should not have packet measurements") + assert.False(t, acc.HasIntField("ping", "maximum_response_ms"), + "Fatal ping should not have packet measurements") + assert.False(t, acc.HasIntField("ping", "minimum_response_ms"), + "Fatal ping should not have packet measurements") +} + +var TTLExpiredPingOutput = ` +Pinging www.google.pl [8.8.8.8] with 32 bytes of data: +Request timed out. +Request timed out. +Reply from 185.28.251.217: TTL expired in transit. +Request timed out. + +Ping statistics for 8.8.8.8: + Packets: Sent = 4, Received = 1, Lost = 3 (75% loss), +` + +func mockTTLExpiredPinger(timeout float64, args ...string) (string, error) { + return TTLExpiredPingOutput, errors.New("So very bad") +} + +// in case 'Destination net unreachable' ping app return receive packet which is not what we need +// it's not contain valid metric so treat it as lost one +func TestTTLExpiredPingGather(t *testing.T) { + var acc testutil.Accumulator + p := Ping{ + Urls: []string{"www.google.com"}, + pingHost: mockTTLExpiredPinger, + } + + p.Gather(&acc) + + tags := map[string]string{"url": "www.google.com"} + fields := map[string]interface{}{ + "packets_transmitted": 4, + "packets_received": 1, + "reply_received": 0, + "percent_packet_loss": 75.0, + "percent_reply_loss": 100.0, + } + acc.AssertContainsTaggedFields(t, "ping", fields, tags) + + assert.False(t, acc.HasFloatField("ping", "errors"), + "Fatal ping should not have packet measurements") + assert.False(t, acc.HasIntField("ping", "average_response_ms"), + "Fatal ping should not have packet measurements") + assert.False(t, acc.HasIntField("ping", "maximum_response_ms"), + "Fatal ping should not have packet measurements") + assert.False(t, acc.HasIntField("ping", "minimum_response_ms"), "Fatal ping should not have packet measurements") } From e457b7a8df828403f1cedf44b7d856758bed4963 Mon Sep 17 00:00:00 2001 From: tuier Date: Tue, 9 Aug 2016 08:29:15 +0100 Subject: [PATCH 13/28] Source improvement for librato output (#1416) * Source improvement for librato output Build the source from the list of tag instead of a configuration specified single tag Graphite Serializer: * make buildTags public * make sure not to use empty tags Librato output: * Improve Error handling for librato API base on error or debug flag * Send Metric per Batch (max 300) * use Graphite BuildTag function to generate source The change is made that it should be retro compatible Metric sample: server=127.0.0.1 port=80 state=leader env=test measurement.metric_name value service_n.metric_x Metric before with source tags set as "server": source=127.0.0.1 test.80.127_0_0_1.leader.measurement.metric_name test.80.127_0_0_1.leader.service_n.metric_x Metric now: source=test.80.127.0.0.1.leader measurement.metric_name service_n.metric_x As you can see the source in the "new" version is much more precise That way when filter (only from source) you can filter by env or any other tags * Using template to specify which tagsusing for source, default concat all tags * revert change in graphite serializer * better documentation, change default for template * fmt * test passing with new host as default tags * use host tag in api integration test * Limit 80 char per line, change resolution to be a int in the sample * fmt * remove resolution, doc for template * fmt --- plugins/outputs/librato/librato.go | 176 ++++++++++++++-------- plugins/outputs/librato/librato_test.go | 187 +++++++++++++++--------- 2 files changed, 234 insertions(+), 129 deletions(-) diff --git a/plugins/outputs/librato/librato.go b/plugins/outputs/librato/librato.go index ccb2acd9a..17d0d4c6a 100644 --- a/plugins/outputs/librato/librato.go +++ b/plugins/outputs/librato/librato.go @@ -7,6 +7,7 @@ import ( "io/ioutil" "log" "net/http" + "regexp" "github.com/influxdata/telegraf" "github.com/influxdata/telegraf/internal" @@ -14,19 +15,22 @@ import ( "github.com/influxdata/telegraf/plugins/serializers/graphite" ) +// Librato structure for configuration and client type Librato struct { - ApiUser string - ApiToken string - Debug bool - NameFromTags bool - SourceTag string - Timeout internal.Duration - Template string + APIUser string + APIToken string + Debug bool + SourceTag string // Deprecated, keeping for backward-compatibility + Timeout internal.Duration + Template string - apiUrl string + APIUrl string client *http.Client } +// https://www.librato.com/docs/kb/faq/best_practices/naming_convention_metrics_sources.html#naming-limitations-for-sources-and-metrics +var reUnacceptedChar = regexp.MustCompile("[^.a-zA-Z0-9_-]") + var sampleConfig = ` ## Librator API Docs ## http://dev.librato.com/v1/metrics-authentication @@ -36,20 +40,21 @@ var sampleConfig = ` api_token = "my-secret-token" # required. ## Debug # debug = false - ## Tag Field to populate source attribute (optional) - ## This is typically the _hostname_ from which the metric was obtained. - source_tag = "host" ## Connection timeout. # timeout = "5s" - ## Output Name Template (same as graphite buckets) + ## Output source Template (same as graphite buckets) ## see https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_OUTPUT.md#graphite - template = "host.tags.measurement.field" + ## This template is used in librato's source (not metric's name) + template = "host" + ` +// LMetrics is the default struct for Librato's API fromat type LMetrics struct { Gauges []*Gauge `json:"gauges"` } +// Gauge is the gauge format for Librato's API fromat type Gauge struct { Name string `json:"name"` Value float64 `json:"value"` @@ -57,17 +62,22 @@ type Gauge struct { MeasureTime int64 `json:"measure_time"` } -const librato_api = "https://metrics-api.librato.com/v1/metrics" +const libratoAPI = "https://metrics-api.librato.com/v1/metrics" -func NewLibrato(apiUrl string) *Librato { +// NewLibrato is the main constructor for librato output plugins +func NewLibrato(apiURL string) *Librato { return &Librato{ - apiUrl: apiUrl, + APIUrl: apiURL, + Template: "host", } } +// Connect is the default output plugin connection function who make sure it +// can connect to the endpoint func (l *Librato) Connect() error { - if l.ApiUser == "" || l.ApiToken == "" { - return fmt.Errorf("api_user and api_token are required fields for librato output") + if l.APIUser == "" || l.APIToken == "" { + return fmt.Errorf( + "api_user and api_token are required fields for librato output") } l.client = &http.Client{ Timeout: l.Timeout.Duration, @@ -76,18 +86,23 @@ func (l *Librato) Connect() error { } func (l *Librato) Write(metrics []telegraf.Metric) error { + if len(metrics) == 0 { return nil } - lmetrics := LMetrics{} + if l.Template == "" { + l.Template = "host" + } + if l.SourceTag != "" { + l.Template = l.SourceTag + } + tempGauges := []*Gauge{} - metricCounter := 0 for _, m := range metrics { if gauges, err := l.buildGauges(m); err == nil { for _, gauge := range gauges { tempGauges = append(tempGauges, gauge) - metricCounter++ if l.Debug { log.Printf("[DEBUG] Got a gauge: %v\n", gauge) } @@ -100,81 +115,115 @@ func (l *Librato) Write(metrics []telegraf.Metric) error { } } - lmetrics.Gauges = make([]*Gauge, metricCounter) - copy(lmetrics.Gauges, tempGauges[0:]) - metricsBytes, err := json.Marshal(lmetrics) - if err != nil { - return fmt.Errorf("unable to marshal Metrics, %s\n", err.Error()) - } else { + metricCounter := len(tempGauges) + // make sur we send a batch of maximum 300 + sizeBatch := 300 + for start := 0; start < metricCounter; start += sizeBatch { + lmetrics := LMetrics{} + end := start + sizeBatch + if end > metricCounter { + end = metricCounter + sizeBatch = end - start + } + lmetrics.Gauges = make([]*Gauge, sizeBatch) + copy(lmetrics.Gauges, tempGauges[start:end]) + metricsBytes, err := json.Marshal(lmetrics) + if err != nil { + return fmt.Errorf("unable to marshal Metrics, %s\n", err.Error()) + } + if l.Debug { log.Printf("[DEBUG] Librato request: %v\n", string(metricsBytes)) } - } - req, err := http.NewRequest("POST", l.apiUrl, bytes.NewBuffer(metricsBytes)) - if err != nil { - return fmt.Errorf("unable to create http.Request, %s\n", err.Error()) - } - req.Header.Add("Content-Type", "application/json") - req.SetBasicAuth(l.ApiUser, l.ApiToken) - resp, err := l.client.Do(req) - if err != nil { - if l.Debug { - log.Printf("[DEBUG] Error POSTing metrics: %v\n", err.Error()) + req, err := http.NewRequest( + "POST", + l.APIUrl, + bytes.NewBuffer(metricsBytes)) + if err != nil { + return fmt.Errorf( + "unable to create http.Request, %s\n", + err.Error()) } - return fmt.Errorf("error POSTing metrics, %s\n", err.Error()) - } else { - if l.Debug { + req.Header.Add("Content-Type", "application/json") + req.SetBasicAuth(l.APIUser, l.APIToken) + + resp, err := l.client.Do(req) + if err != nil { + if l.Debug { + log.Printf("[DEBUG] Error POSTing metrics: %v\n", err.Error()) + } + return fmt.Errorf("error POSTing metrics, %s\n", err.Error()) + } + defer resp.Body.Close() + + if resp.StatusCode != 200 || l.Debug { htmlData, err := ioutil.ReadAll(resp.Body) if err != nil { log.Printf("[DEBUG] Couldn't get response! (%v)\n", err) - } else { + } + if resp.StatusCode != 200 { + return fmt.Errorf( + "received bad status code, %d\n %s", + resp.StatusCode, + string(htmlData)) + } + if l.Debug { log.Printf("[DEBUG] Librato response: %v\n", string(htmlData)) } } } - defer resp.Body.Close() - - if resp.StatusCode != 200 { - return fmt.Errorf("received bad status code, %d\n", resp.StatusCode) - } - return nil } +// SampleConfig is function who return the default configuration for this +// output func (l *Librato) SampleConfig() string { return sampleConfig } +// Description is function who return the Description of this output func (l *Librato) Description() string { return "Configuration for Librato API to send metrics to." } func (l *Librato) buildGauges(m telegraf.Metric) ([]*Gauge, error) { + gauges := []*Gauge{} - bucket := graphite.SerializeBucketName(m.Name(), m.Tags(), l.Template, "") + if m.Time().Unix() == 0 { + return gauges, fmt.Errorf( + "Measure time must not be zero\n <%s> \n", + m.String()) + } + metricSource := graphite.InsertField( + graphite.SerializeBucketName("", m.Tags(), l.Template, ""), + "value") + if metricSource == "" { + return gauges, + fmt.Errorf("undeterminable Source type from Field, %s\n", + l.Template) + } for fieldName, value := range m.Fields() { + + metricName := m.Name() + if fieldName != "value" { + metricName = fmt.Sprintf("%s.%s", m.Name(), fieldName) + } + gauge := &Gauge{ - Name: graphite.InsertField(bucket, fieldName), + Source: reUnacceptedChar.ReplaceAllString(metricSource, "-"), + Name: reUnacceptedChar.ReplaceAllString(metricName, "-"), MeasureTime: m.Time().Unix(), } - if !gauge.verifyValue(value) { + if !verifyValue(value) { continue } if err := gauge.setValue(value); err != nil { - return gauges, fmt.Errorf("unable to extract value from Fields, %s\n", + return gauges, fmt.Errorf( + "unable to extract value from Fields, %s\n", err.Error()) } - if l.SourceTag != "" { - if source, ok := m.Tags()[l.SourceTag]; ok { - gauge.Source = source - } else { - return gauges, - fmt.Errorf("undeterminable Source type from Field, %s\n", - l.SourceTag) - } - } gauges = append(gauges, gauge) } if l.Debug { @@ -183,7 +232,7 @@ func (l *Librato) buildGauges(m telegraf.Metric) ([]*Gauge, error) { return gauges, nil } -func (g *Gauge) verifyValue(v interface{}) bool { +func verifyValue(v interface{}) bool { switch v.(type) { case string: return false @@ -209,12 +258,13 @@ func (g *Gauge) setValue(v interface{}) error { return nil } +//Close is used to close the connection to librato Output func (l *Librato) Close() error { return nil } func init() { outputs.Add("librato", func() telegraf.Output { - return NewLibrato(librato_api) + return NewLibrato(libratoAPI) }) } diff --git a/plugins/outputs/librato/librato_test.go b/plugins/outputs/librato/librato_test.go index e90339928..dd5755a8c 100644 --- a/plugins/outputs/librato/librato_test.go +++ b/plugins/outputs/librato/librato_test.go @@ -1,7 +1,6 @@ package librato import ( - "encoding/json" "fmt" "net/http" "net/http/httptest" @@ -10,141 +9,137 @@ import ( "time" "github.com/influxdata/telegraf" - "github.com/influxdata/telegraf/plugins/serializers/graphite" - "github.com/influxdata/telegraf/testutil" "github.com/stretchr/testify/require" ) var ( - fakeUrl = "http://test.librato.com" + fakeURL = "http://test.librato.com" fakeUser = "telegraf@influxdb.com" fakeToken = "123456" ) func fakeLibrato() *Librato { - l := NewLibrato(fakeUrl) - l.ApiUser = fakeUser - l.ApiToken = fakeToken + l := NewLibrato(fakeURL) + l.APIUser = fakeUser + l.APIToken = fakeToken return l } -func BuildTags(t *testing.T) { - testMetric := testutil.TestMetric(0.0, "test1") - graphiteSerializer := graphite.GraphiteSerializer{} - tags, err := graphiteSerializer.Serialize(testMetric) - fmt.Printf("Tags: %v", tags) - require.NoError(t, err) -} - func TestUriOverride(t *testing.T) { - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(http.StatusOK) - })) + ts := httptest.NewServer( + http.HandlerFunc( + func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + })) defer ts.Close() l := NewLibrato(ts.URL) - l.ApiUser = "telegraf@influxdb.com" - l.ApiToken = "123456" + l.APIUser = "telegraf@influxdb.com" + l.APIToken = "123456" err := l.Connect() require.NoError(t, err) - err = l.Write(testutil.MockMetrics()) + err = l.Write([]telegraf.Metric{newHostMetric(int32(0), "name", "host")}) require.NoError(t, err) } func TestBadStatusCode(t *testing.T) { - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(http.StatusServiceUnavailable) - json.NewEncoder(w).Encode(`{ - "errors": { - "system": [ - "The API is currently down for maintenance. It'll be back shortly." - ] - } - }`) - })) + ts := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusServiceUnavailable) + })) defer ts.Close() l := NewLibrato(ts.URL) - l.ApiUser = "telegraf@influxdb.com" - l.ApiToken = "123456" + l.APIUser = "telegraf@influxdb.com" + l.APIToken = "123456" err := l.Connect() require.NoError(t, err) - err = l.Write(testutil.MockMetrics()) + err = l.Write([]telegraf.Metric{newHostMetric(int32(0), "name", "host")}) if err == nil { t.Errorf("error expected but none returned") } else { - require.EqualError(t, fmt.Errorf("received bad status code, 503\n"), err.Error()) + require.EqualError( + t, + fmt.Errorf("received bad status code, 503\n "), err.Error()) } } func TestBuildGauge(t *testing.T) { + + mtime := time.Date(2009, time.November, 10, 23, 0, 0, 0, time.UTC).Unix() var gaugeTests = []struct { ptIn telegraf.Metric outGauge *Gauge err error }{ { - testutil.TestMetric(0.0, "test1"), + newHostMetric(0.0, "test1", "host1"), &Gauge{ - Name: "value1.test1", - MeasureTime: time.Date(2009, time.November, 10, 23, 0, 0, 0, time.UTC).Unix(), + Name: "test1", + MeasureTime: mtime, Value: 0.0, + Source: "host1", }, nil, }, { - testutil.TestMetric(1.0, "test2"), + newHostMetric(1.0, "test2", "host2"), &Gauge{ - Name: "value1.test2", - MeasureTime: time.Date(2009, time.November, 10, 23, 0, 0, 0, time.UTC).Unix(), + Name: "test2", + MeasureTime: mtime, Value: 1.0, + Source: "host2", }, nil, }, { - testutil.TestMetric(10, "test3"), + newHostMetric(10, "test3", "host3"), &Gauge{ - Name: "value1.test3", - MeasureTime: time.Date(2009, time.November, 10, 23, 0, 0, 0, time.UTC).Unix(), + Name: "test3", + MeasureTime: mtime, Value: 10.0, + Source: "host3", }, nil, }, { - testutil.TestMetric(int32(112345), "test4"), + newHostMetric(int32(112345), "test4", "host4"), &Gauge{ - Name: "value1.test4", - MeasureTime: time.Date(2009, time.November, 10, 23, 0, 0, 0, time.UTC).Unix(), + Name: "test4", + MeasureTime: mtime, Value: 112345.0, + Source: "host4", }, nil, }, { - testutil.TestMetric(int64(112345), "test5"), + newHostMetric(int64(112345), "test5", "host5"), &Gauge{ - Name: "value1.test5", - MeasureTime: time.Date(2009, time.November, 10, 23, 0, 0, 0, time.UTC).Unix(), + Name: "test5", + MeasureTime: mtime, Value: 112345.0, + Source: "host5", }, nil, }, { - testutil.TestMetric(float32(11234.5), "test6"), + newHostMetric(float32(11234.5), "test6", "host6"), &Gauge{ - Name: "value1.test6", - MeasureTime: time.Date(2009, time.November, 10, 23, 0, 0, 0, time.UTC).Unix(), + Name: "test6", + MeasureTime: mtime, Value: 11234.5, + Source: "host6", }, nil, }, { - testutil.TestMetric("11234.5", "test7"), + newHostMetric("11234.5", "test7", "host7"), nil, nil, }, } - l := NewLibrato(fakeUrl) + l := NewLibrato(fakeURL) for _, gt := range gaugeTests { gauges, err := l.buildGauges(gt.ptIn) if err != nil && gt.err == nil { @@ -167,61 +162,121 @@ func TestBuildGauge(t *testing.T) { } } +func newHostMetric(value interface{}, name, host string) (metric telegraf.Metric) { + metric, _ = telegraf.NewMetric( + name, + map[string]string{"host": host}, + map[string]interface{}{"value": value}, + time.Date(2009, time.November, 10, 23, 0, 0, 0, time.UTC), + ) + return +} + func TestBuildGaugeWithSource(t *testing.T) { + mtime := time.Date(2009, time.November, 10, 23, 0, 0, 0, time.UTC) pt1, _ := telegraf.NewMetric( "test1", map[string]string{"hostname": "192.168.0.1", "tag1": "value1"}, map[string]interface{}{"value": 0.0}, - time.Date(2010, time.November, 10, 23, 0, 0, 0, time.UTC), + mtime, ) pt2, _ := telegraf.NewMetric( "test2", map[string]string{"hostnam": "192.168.0.1", "tag1": "value1"}, map[string]interface{}{"value": 1.0}, - time.Date(2010, time.December, 10, 23, 0, 0, 0, time.UTC), + mtime, + ) + pt3, _ := telegraf.NewMetric( + "test3", + map[string]string{ + "hostname": "192.168.0.1", + "tag2": "value2", + "tag1": "value1"}, + map[string]interface{}{"value": 1.0}, + mtime, + ) + pt4, _ := telegraf.NewMetric( + "test4", + map[string]string{ + "hostname": "192.168.0.1", + "tag2": "value2", + "tag1": "value1"}, + map[string]interface{}{"value": 1.0}, + mtime, ) var gaugeTests = []struct { ptIn telegraf.Metric + template string outGauge *Gauge err error }{ { pt1, + "hostname", &Gauge{ - Name: "192_168_0_1.value1.test1", - MeasureTime: time.Date(2010, time.November, 10, 23, 0, 0, 0, time.UTC).Unix(), + Name: "test1", + MeasureTime: mtime.Unix(), Value: 0.0, - Source: "192.168.0.1", + Source: "192_168_0_1", }, nil, }, { pt2, + "hostname", &Gauge{ - Name: "192_168_0_1.value1.test1", - MeasureTime: time.Date(2010, time.December, 10, 23, 0, 0, 0, time.UTC).Unix(), + Name: "test2", + MeasureTime: mtime.Unix(), Value: 1.0, }, fmt.Errorf("undeterminable Source type from Field, hostname"), }, + { + pt3, + "tags", + &Gauge{ + Name: "test3", + MeasureTime: mtime.Unix(), + Value: 1.0, + Source: "192_168_0_1.value1.value2", + }, + nil, + }, + { + pt4, + "hostname.tag2", + &Gauge{ + Name: "test4", + MeasureTime: mtime.Unix(), + Value: 1.0, + Source: "192_168_0_1.value2", + }, + nil, + }, } - l := NewLibrato(fakeUrl) - l.SourceTag = "hostname" + l := NewLibrato(fakeURL) for _, gt := range gaugeTests { + l.Template = gt.template gauges, err := l.buildGauges(gt.ptIn) if err != nil && gt.err == nil { t.Errorf("%s: unexpected error, %+v\n", gt.ptIn.Name(), err) } if gt.err != nil && err == nil { - t.Errorf("%s: expected an error (%s) but none returned", gt.ptIn.Name(), gt.err.Error()) + t.Errorf( + "%s: expected an error (%s) but none returned", + gt.ptIn.Name(), + gt.err.Error()) } if len(gauges) == 0 { continue } if gt.err == nil && !reflect.DeepEqual(gauges[0], gt.outGauge) { - t.Errorf("%s: \nexpected %+v\ngot %+v\n", gt.ptIn.Name(), gt.outGauge, gauges[0]) + t.Errorf( + "%s: \nexpected %+v\ngot %+v\n", + gt.ptIn.Name(), + gt.outGauge, gauges[0]) } } } From abcd19493e1d8c940cbbb5c87f05ce92d44e8e8e Mon Sep 17 00:00:00 2001 From: Cameron Sparr Date: Tue, 9 Aug 2016 07:23:22 +0100 Subject: [PATCH 14/28] If win stat buffer is empty, do not try to index closes #1425 --- CHANGELOG.md | 1 + plugins/inputs/win_perf_counters/win_perf_counters.go | 3 +++ 2 files changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 02c0d4bbd..a252c6752 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -126,6 +126,7 @@ consistent with the behavior of `collection_jitter`. - [#1322](https://github.com/influxdata/telegraf/issues/1322): Fix rare RHEL 5.2 panic in gopsutil diskio gathering function. - [#1586](https://github.com/influxdata/telegraf/pull/1586): Remove IF NOT EXISTS from influxdb output database creation. - [#1600](https://github.com/influxdata/telegraf/issues/1600): Fix quoting with text values in postgresql_extensible plugin. +- [#1425](https://github.com/influxdata/telegraf/issues/1425): Fix win_perf_counter "index out of range" panic. ## v0.13.1 [2016-05-24] diff --git a/plugins/inputs/win_perf_counters/win_perf_counters.go b/plugins/inputs/win_perf_counters/win_perf_counters.go index fb7b093c0..60b9ff55d 100644 --- a/plugins/inputs/win_perf_counters/win_perf_counters.go +++ b/plugins/inputs/win_perf_counters/win_perf_counters.go @@ -272,6 +272,9 @@ func (m *Win_PerfCounters) Gather(acc telegraf.Accumulator) error { &bufCount, &emptyBuf[0]) // uses null ptr here according to MSDN. if ret == win.PDH_MORE_DATA { filledBuf := make([]win.PDH_FMT_COUNTERVALUE_ITEM_DOUBLE, bufCount*size) + if len(filledBuf) == 0 { + continue + } ret = win.PdhGetFormattedCounterArrayDouble(metric.counterHandle, &bufSize, &bufCount, &filledBuf[0]) for i := 0; i < int(bufCount); i++ { From 1989a5855d3ce0603a77ac931ba1f2bdc77bd843 Mon Sep 17 00:00:00 2001 From: Rene Zbinden Date: Fri, 24 Jun 2016 10:18:02 +0200 Subject: [PATCH 15/28] remove cgo dependeny with forking sensors command closes #1414 closes #649 --- CHANGELOG.md | 1 + README.md | 2 +- plugins/inputs/sensors/README.md | 47 +++ plugins/inputs/sensors/sensors.go | 151 +++++---- plugins/inputs/sensors/sensors_nocompile.go | 3 - plugins/inputs/sensors/sensors_notlinux.go | 3 + plugins/inputs/sensors/sensors_test.go | 328 ++++++++++++++++++++ 7 files changed, 469 insertions(+), 66 deletions(-) create mode 100644 plugins/inputs/sensors/README.md delete mode 100644 plugins/inputs/sensors/sensors_nocompile.go create mode 100644 plugins/inputs/sensors/sensors_notlinux.go create mode 100644 plugins/inputs/sensors/sensors_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index a252c6752..762c7ceff 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -80,6 +80,7 @@ consistent with the behavior of `collection_jitter`. - [#1296](https://github.com/influxdata/telegraf/issues/1296): Refactor of flush_jitter argument. - [#1213](https://github.com/influxdata/telegraf/issues/1213): Add inactive & active memory to mem plugin. - [#1543](https://github.com/influxdata/telegraf/pull/1543): Official Windows service. +- [#1414](https://github.com/influxdata/telegraf/pull/1414): Forking sensors command to remove C package dependency. ### Bugfixes diff --git a/README.md b/README.md index 9d2ee3ce1..74bbf2a4f 100644 --- a/README.md +++ b/README.md @@ -188,7 +188,7 @@ Currently implemented sources: * [redis](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/redis) * [rethinkdb](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/rethinkdb) * [riak](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/riak) -* [sensors ](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/sensors) (only available if built from source) +* [sensors](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/sensors) * [snmp](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/snmp) * [sql server](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/sqlserver) (microsoft) * [twemproxy](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/twemproxy) diff --git a/plugins/inputs/sensors/README.md b/plugins/inputs/sensors/README.md new file mode 100644 index 000000000..237a9b789 --- /dev/null +++ b/plugins/inputs/sensors/README.md @@ -0,0 +1,47 @@ +# sensors Input Plugin + +Collect [lm-sensors](https://en.wikipedia.org/wiki/Lm_sensors) metrics - requires the lm-sensors +package installed. + +This plugin collects sensor metrics with the `sensors` executable from the lm-sensor package. + +### Configuration: +``` +# Monitor sensors, requires lm-sensors package +[[inputs.sensors]] + ## Remove numbers from field names. + ## If true, a field name like 'temp1_input' will be changed to 'temp_input'. + # remove_numbers = true +``` + +### Measurements & Fields: +Fields are created dynamicaly depending on the sensors. All fields are float. + +### Tags: + +- All measurements have the following tags: + - chip + - feature + +### Example Output: + +#### Default +``` +$ telegraf -config telegraf.conf -input-filter sensors -test +* Plugin: sensors, Collection 1 +> sensors,chip=power_meter-acpi-0,feature=power1 power_average=0,power_average_interval=300 1466751326000000000 +> sensors,chip=k10temp-pci-00c3,feature=temp1 temp_crit=70,temp_crit_hyst=65,temp_input=29,temp_max=70 1466751326000000000 +> sensors,chip=k10temp-pci-00cb,feature=temp1 temp_input=29,temp_max=70 1466751326000000000 +> sensors,chip=k10temp-pci-00d3,feature=temp1 temp_input=27.5,temp_max=70 1466751326000000000 +> sensors,chip=k10temp-pci-00db,feature=temp1 temp_crit=70,temp_crit_hyst=65,temp_input=29.5,temp_max=70 1466751326000000000 +``` + +#### With remove_numbers=false +``` +* Plugin: sensors, Collection 1 +> sensors,chip=power_meter-acpi-0,feature=power1 power1_average=0,power1_average_interval=300 1466753424000000000 +> sensors,chip=k10temp-pci-00c3,feature=temp1 temp1_crit=70,temp1_crit_hyst=65,temp1_input=29.125,temp1_max=70 1466753424000000000 +> sensors,chip=k10temp-pci-00cb,feature=temp1 temp1_input=29,temp1_max=70 1466753424000000000 +> sensors,chip=k10temp-pci-00d3,feature=temp1 temp1_input=29.5,temp1_max=70 1466753424000000000 +> sensors,chip=k10temp-pci-00db,feature=temp1 temp1_crit=70,temp1_crit_hyst=65,temp1_input=30,temp1_max=70 1466753424000000000 +``` diff --git a/plugins/inputs/sensors/sensors.go b/plugins/inputs/sensors/sensors.go index dbb304b71..6e165e4cb 100644 --- a/plugins/inputs/sensors/sensors.go +++ b/plugins/inputs/sensors/sensors.go @@ -1,91 +1,118 @@ -// +build linux,sensors +// +build linux package sensors import ( + "errors" + "fmt" + "os/exec" + "regexp" + "strconv" "strings" - - "github.com/md14454/gosensors" + "time" "github.com/influxdata/telegraf" + "github.com/influxdata/telegraf/internal" "github.com/influxdata/telegraf/plugins/inputs" ) +var ( + execCommand = exec.Command // execCommand is used to mock commands in tests. + numberRegp = regexp.MustCompile("[0-9]+") +) + type Sensors struct { - Sensors []string + RemoveNumbers bool `toml:"remove_numbers"` + path string } -func (_ *Sensors) Description() string { - return "Monitor sensors using lm-sensors package" +func (*Sensors) Description() string { + return "Monitor sensors, requires lm-sensors package" } -var sensorsSampleConfig = ` - ## By default, telegraf gathers stats from all sensors detected by the - ## lm-sensors module. - ## - ## Only collect stats from the selected sensors. Sensors are listed as - ## :. This information can be found by running the - ## sensors command, e.g. sensors -u - ## - ## A * as the feature name will return all features of the chip - ## - # sensors = ["coretemp-isa-0000:Core 0", "coretemp-isa-0001:*"] +func (*Sensors) SampleConfig() string { + return ` + ## Remove numbers from field names. + ## If true, a field name like 'temp1_input' will be changed to 'temp_input'. + # remove_numbers = true ` -func (_ *Sensors) SampleConfig() string { - return sensorsSampleConfig } func (s *Sensors) Gather(acc telegraf.Accumulator) error { - gosensors.Init() - defer gosensors.Cleanup() - - for _, chip := range gosensors.GetDetectedChips() { - for _, feature := range chip.GetFeatures() { - chipName := chip.String() - featureLabel := feature.GetLabel() - - if len(s.Sensors) != 0 { - var found bool - - for _, sensor := range s.Sensors { - parts := strings.SplitN(sensor, ":", 2) - - if parts[0] == chipName { - if parts[1] == "*" || parts[1] == featureLabel { - found = true - break - } - } - } - - if !found { - continue - } - } - - tags := map[string]string{ - "chip": chipName, - "adapter": chip.AdapterName(), - "feature-name": feature.Name, - "feature-label": featureLabel, - } - - fieldName := chipName + ":" + featureLabel - - fields := map[string]interface{}{ - fieldName: feature.GetValue(), - } - - acc.AddFields("sensors", fields, tags) - } + if len(s.path) == 0 { + return errors.New("sensors not found: verify that lm-sensors package is installed and that sensors is in your PATH") } + return s.parse(acc) +} + +// parse forks the command: +// sensors -u -A +// and parses the output to add it to the telegraf.Accumulator. +func (s *Sensors) parse(acc telegraf.Accumulator) error { + tags := map[string]string{} + fields := map[string]interface{}{} + chip := "" + cmd := execCommand(s.path, "-A", "-u") + out, err := internal.CombinedOutputTimeout(cmd, time.Second*5) + if err != nil { + return fmt.Errorf("failed to run command %s: %s - %s", strings.Join(cmd.Args, " "), err, string(out)) + } + lines := strings.Split(strings.TrimSpace(string(out)), "\n") + for _, line := range lines { + if len(line) == 0 { + acc.AddFields("sensors", fields, tags) + chip = "" + tags = map[string]string{} + fields = map[string]interface{}{} + continue + } + if len(chip) == 0 { + chip = line + tags["chip"] = chip + continue + } + if !strings.HasPrefix(line, " ") { + if len(tags) > 1 { + acc.AddFields("sensors", fields, tags) + } + fields = map[string]interface{}{} + tags = map[string]string{ + "chip": chip, + "feature": strings.TrimRight(snake(line), ":"), + } + } else { + splitted := strings.Split(line, ":") + fieldName := strings.TrimSpace(splitted[0]) + if s.RemoveNumbers { + fieldName = numberRegp.ReplaceAllString(fieldName, "") + } + fieldValue, err := strconv.ParseFloat(strings.TrimSpace(splitted[1]), 64) + if err != nil { + return err + } + fields[fieldName] = fieldValue + } + } + acc.AddFields("sensors", fields, tags) return nil } func init() { + s := Sensors{ + RemoveNumbers: true, + } + path, _ := exec.LookPath("sensors") + if len(path) > 0 { + s.path = path + } inputs.Add("sensors", func() telegraf.Input { - return &Sensors{} + return &s }) } + +// snake converts string to snake case +func snake(input string) string { + return strings.ToLower(strings.Replace(input, " ", "_", -1)) +} diff --git a/plugins/inputs/sensors/sensors_nocompile.go b/plugins/inputs/sensors/sensors_nocompile.go deleted file mode 100644 index 5c38a437b..000000000 --- a/plugins/inputs/sensors/sensors_nocompile.go +++ /dev/null @@ -1,3 +0,0 @@ -// +build !linux !sensors - -package sensors diff --git a/plugins/inputs/sensors/sensors_notlinux.go b/plugins/inputs/sensors/sensors_notlinux.go new file mode 100644 index 000000000..62a621159 --- /dev/null +++ b/plugins/inputs/sensors/sensors_notlinux.go @@ -0,0 +1,3 @@ +// +build !linux + +package sensors diff --git a/plugins/inputs/sensors/sensors_test.go b/plugins/inputs/sensors/sensors_test.go new file mode 100644 index 000000000..01d27abcf --- /dev/null +++ b/plugins/inputs/sensors/sensors_test.go @@ -0,0 +1,328 @@ +// +build linux + +package sensors + +import ( + "fmt" + "os" + "os/exec" + "testing" + + "github.com/influxdata/telegraf/testutil" +) + +func TestGatherDefault(t *testing.T) { + s := Sensors{ + RemoveNumbers: true, + path: "sensors", + } + // overwriting exec commands with mock commands + execCommand = fakeExecCommand + defer func() { execCommand = exec.Command }() + var acc testutil.Accumulator + + err := s.Gather(&acc) + if err != nil { + t.Fatal(err) + } + + var tests = []struct { + tags map[string]string + fields map[string]interface{} + }{ + { + map[string]string{ + "chip": "acpitz-virtual-0", + "feature": "temp1", + }, + map[string]interface{}{ + "temp_input": 8.3, + "temp_crit": 31.3, + }, + }, + { + map[string]string{ + "chip": "power_meter-acpi-0", + "feature": "power1", + }, + map[string]interface{}{ + "power_average": 0.0, + "power_average_interval": 300.0, + }, + }, + { + map[string]string{ + "chip": "coretemp-isa-0000", + "feature": "physical_id_0", + }, + map[string]interface{}{ + "temp_input": 77.0, + "temp_max": 82.0, + "temp_crit": 92.0, + "temp_crit_alarm": 0.0, + }, + }, + { + map[string]string{ + "chip": "coretemp-isa-0000", + "feature": "core_0", + }, + map[string]interface{}{ + "temp_input": 75.0, + "temp_max": 82.0, + "temp_crit": 92.0, + "temp_crit_alarm": 0.0, + }, + }, + { + map[string]string{ + "chip": "coretemp-isa-0000", + "feature": "core_1", + }, + map[string]interface{}{ + "temp_input": 77.0, + "temp_max": 82.0, + "temp_crit": 92.0, + "temp_crit_alarm": 0.0, + }, + }, + { + map[string]string{ + "chip": "coretemp-isa-0001", + "feature": "physical_id_1", + }, + map[string]interface{}{ + "temp_input": 70.0, + "temp_max": 82.0, + "temp_crit": 92.0, + "temp_crit_alarm": 0.0, + }, + }, + { + map[string]string{ + "chip": "coretemp-isa-0001", + "feature": "core_0", + }, + map[string]interface{}{ + "temp_input": 66.0, + "temp_max": 82.0, + "temp_crit": 92.0, + "temp_crit_alarm": 0.0, + }, + }, + { + map[string]string{ + "chip": "coretemp-isa-0001", + "feature": "core_1", + }, + map[string]interface{}{ + "temp_input": 70.0, + "temp_max": 82.0, + "temp_crit": 92.0, + "temp_crit_alarm": 0.0, + }, + }, + } + + for _, test := range tests { + acc.AssertContainsTaggedFields(t, "sensors", test.fields, test.tags) + } +} + +func TestGatherNotRemoveNumbers(t *testing.T) { + s := Sensors{ + RemoveNumbers: false, + path: "sensors", + } + // overwriting exec commands with mock commands + execCommand = fakeExecCommand + defer func() { execCommand = exec.Command }() + var acc testutil.Accumulator + + err := s.Gather(&acc) + if err != nil { + t.Fatal(err) + } + + var tests = []struct { + tags map[string]string + fields map[string]interface{} + }{ + { + map[string]string{ + "chip": "acpitz-virtual-0", + "feature": "temp1", + }, + map[string]interface{}{ + "temp1_input": 8.3, + "temp1_crit": 31.3, + }, + }, + { + map[string]string{ + "chip": "power_meter-acpi-0", + "feature": "power1", + }, + map[string]interface{}{ + "power1_average": 0.0, + "power1_average_interval": 300.0, + }, + }, + { + map[string]string{ + "chip": "coretemp-isa-0000", + "feature": "physical_id_0", + }, + map[string]interface{}{ + "temp1_input": 77.0, + "temp1_max": 82.0, + "temp1_crit": 92.0, + "temp1_crit_alarm": 0.0, + }, + }, + { + map[string]string{ + "chip": "coretemp-isa-0000", + "feature": "core_0", + }, + map[string]interface{}{ + "temp2_input": 75.0, + "temp2_max": 82.0, + "temp2_crit": 92.0, + "temp2_crit_alarm": 0.0, + }, + }, + { + map[string]string{ + "chip": "coretemp-isa-0000", + "feature": "core_1", + }, + map[string]interface{}{ + "temp3_input": 77.0, + "temp3_max": 82.0, + "temp3_crit": 92.0, + "temp3_crit_alarm": 0.0, + }, + }, + { + map[string]string{ + "chip": "coretemp-isa-0001", + "feature": "physical_id_1", + }, + map[string]interface{}{ + "temp1_input": 70.0, + "temp1_max": 82.0, + "temp1_crit": 92.0, + "temp1_crit_alarm": 0.0, + }, + }, + { + map[string]string{ + "chip": "coretemp-isa-0001", + "feature": "core_0", + }, + map[string]interface{}{ + "temp2_input": 66.0, + "temp2_max": 82.0, + "temp2_crit": 92.0, + "temp2_crit_alarm": 0.0, + }, + }, + { + map[string]string{ + "chip": "coretemp-isa-0001", + "feature": "core_1", + }, + map[string]interface{}{ + "temp3_input": 70.0, + "temp3_max": 82.0, + "temp3_crit": 92.0, + "temp3_crit_alarm": 0.0, + }, + }, + } + + for _, test := range tests { + acc.AssertContainsTaggedFields(t, "sensors", test.fields, test.tags) + } +} + +// fackeExecCommand is a helper function that mock +// the exec.Command call (and call the test binary) +func fakeExecCommand(command string, args ...string) *exec.Cmd { + cs := []string{"-test.run=TestHelperProcess", "--", command} + cs = append(cs, args...) + cmd := exec.Command(os.Args[0], cs...) + cmd.Env = []string{"GO_WANT_HELPER_PROCESS=1"} + return cmd +} + +// TestHelperProcess isn't a real test. It's used to mock exec.Command +// For example, if you run: +// GO_WANT_HELPER_PROCESS=1 go test -test.run=TestHelperProcess -- chrony tracking +// it returns below mockData. +func TestHelperProcess(t *testing.T) { + if os.Getenv("GO_WANT_HELPER_PROCESS") != "1" { + return + } + + mockData := `acpitz-virtual-0 +temp1: + temp1_input: 8.300 + temp1_crit: 31.300 + +power_meter-acpi-0 +power1: + power1_average: 0.000 + power1_average_interval: 300.000 + +coretemp-isa-0000 +Physical id 0: + temp1_input: 77.000 + temp1_max: 82.000 + temp1_crit: 92.000 + temp1_crit_alarm: 0.000 +Core 0: + temp2_input: 75.000 + temp2_max: 82.000 + temp2_crit: 92.000 + temp2_crit_alarm: 0.000 +Core 1: + temp3_input: 77.000 + temp3_max: 82.000 + temp3_crit: 92.000 + temp3_crit_alarm: 0.000 + +coretemp-isa-0001 +Physical id 1: + temp1_input: 70.000 + temp1_max: 82.000 + temp1_crit: 92.000 + temp1_crit_alarm: 0.000 +Core 0: + temp2_input: 66.000 + temp2_max: 82.000 + temp2_crit: 92.000 + temp2_crit_alarm: 0.000 +Core 1: + temp3_input: 70.000 + temp3_max: 82.000 + temp3_crit: 92.000 + temp3_crit_alarm: 0.000 +` + + args := os.Args + + // Previous arguments are tests stuff, that looks like : + // /tmp/go-build970079519/…/_test/integration.test -test.run=TestHelperProcess -- + cmd, args := args[3], args[4:] + + if cmd == "sensors" { + fmt.Fprint(os.Stdout, mockData) + } else { + fmt.Fprint(os.Stdout, "command not found") + os.Exit(1) + + } + os.Exit(0) +} From fec9760f72f8c45395185646c48f632ba9b38883 Mon Sep 17 00:00:00 2001 From: jsvisa Date: Mon, 23 May 2016 21:13:00 +0800 Subject: [PATCH 16/28] add pgbouncer plugin add pgbouncer docker for testing add pgbouncer testcase update changlog closes #1400 --- CHANGELOG.md | 1 + Makefile | 17 +- README.md | 1 + plugins/inputs/all/all.go | 1 + plugins/inputs/pgbouncer/README.md | 62 +++++++ plugins/inputs/pgbouncer/pgbouncer.go | 206 +++++++++++++++++++++ plugins/inputs/pgbouncer/pgbouncer_test.go | 180 ++++++++++++++++++ 7 files changed, 466 insertions(+), 2 deletions(-) create mode 100644 plugins/inputs/pgbouncer/README.md create mode 100644 plugins/inputs/pgbouncer/pgbouncer.go create mode 100644 plugins/inputs/pgbouncer/pgbouncer_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 762c7ceff..4ab7df1b1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -59,6 +59,7 @@ consistent with the behavior of `collection_jitter`. - [#1434](https://github.com/influxdata/telegraf/pull/1434): Add measurement name arg to logparser plugin. - [#1479](https://github.com/influxdata/telegraf/pull/1479): logparser: change resp_code from a field to a tag. - [#1411](https://github.com/influxdata/telegraf/pull/1411): Implement support for fetching hddtemp data +- [#1400](https://github.com/influxdata/telegraf/pull/1400): Add supoort for Pgbouncer - [#1340](https://github.com/influxdata/telegraf/issues/1340): statsd: do not log every dropped metric. - [#1368](https://github.com/influxdata/telegraf/pull/1368): Add precision rounding to all metrics on collection. - [#1390](https://github.com/influxdata/telegraf/pull/1390): Add support for Tengine diff --git a/Makefile b/Makefile index 19eccbb70..1859eba6d 100644 --- a/Makefile +++ b/Makefile @@ -57,6 +57,13 @@ docker-run: docker run --name mqtt -p "1883:1883" -d ncarlier/mqtt docker run --name riemann -p "5555:5555" -d blalor/riemann docker run --name snmp -p "31161:31161/udp" -d titilambert/snmpsim + docker run --name pgbouncer \ + -p "6432:6432" \ + -e PGB_USERLISTS="postgres:postgres" \ + -e PGB_ADMIN_USERS="postgres" \ + -e PGB_STATS_USERS="postgres" \ + --link postgres:pg \ + -d jsvisa/pgbouncer # Run docker containers necessary for CircleCI unit tests docker-run-circle: @@ -70,11 +77,17 @@ docker-run-circle: docker run --name mqtt -p "1883:1883" -d ncarlier/mqtt docker run --name riemann -p "5555:5555" -d blalor/riemann docker run --name snmp -p "31161:31161/udp" -d titilambert/snmpsim + docker run --name pgbouncer \ + -p "6432:6432" \ + -e PGB_USERLISTS="postgres:postgres" \ + -e PGB_ADMIN_USERS="postgres" \ + -e PGB_STATS_USERS="postgres" \ + -d jsvisa/pgbouncer # Kill all docker containers, ignore errors docker-kill: - -docker kill nsq aerospike redis rabbitmq postgres memcached mysql kafka mqtt riemann snmp - -docker rm nsq aerospike redis rabbitmq postgres memcached mysql kafka mqtt riemann snmp + -docker kill nsq aerospike redis rabbitmq postgres memcached mysql kafka mqtt riemann snmp pgbouncer + -docker rm nsq aerospike redis rabbitmq postgres memcached mysql kafka mqtt riemann snmp pgbouncer # Run full unit tests using docker containers (includes setup and teardown) test: vet docker-kill docker-run diff --git a/README.md b/README.md index 74bbf2a4f..c9fe12351 100644 --- a/README.md +++ b/README.md @@ -174,6 +174,7 @@ Currently implemented sources: * [nsq](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/nsq) * [nstat](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/nstat) * [ntpq](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/ntpq) +* [pgbouncer](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/pgbouncer) * [phpfpm](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/phpfpm) * [phusion passenger](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/passenger) * [ping](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/ping) diff --git a/plugins/inputs/all/all.go b/plugins/inputs/all/all.go index ddb7d4039..dacbff644 100644 --- a/plugins/inputs/all/all.go +++ b/plugins/inputs/all/all.go @@ -46,6 +46,7 @@ import ( _ "github.com/influxdata/telegraf/plugins/inputs/nstat" _ "github.com/influxdata/telegraf/plugins/inputs/ntpq" _ "github.com/influxdata/telegraf/plugins/inputs/passenger" + _ "github.com/influxdata/telegraf/plugins/inputs/pgbouncer" _ "github.com/influxdata/telegraf/plugins/inputs/phpfpm" _ "github.com/influxdata/telegraf/plugins/inputs/ping" _ "github.com/influxdata/telegraf/plugins/inputs/postgresql" diff --git a/plugins/inputs/pgbouncer/README.md b/plugins/inputs/pgbouncer/README.md new file mode 100644 index 000000000..31e883f11 --- /dev/null +++ b/plugins/inputs/pgbouncer/README.md @@ -0,0 +1,62 @@ +# Pgbouncer plugin + +This pgbouncer plugin provides metrics for your pgbouncer connection information. + +### Configuration: + +```toml +# Description +[[inputs.pgbouncer]] + ## specify address via a url matching: + ## postgres://[pqgotest[:password]]@localhost:port[/dbname]\ + ## ?sslmode=[disable|verify-ca|verify-full] + ## or a simple string: + ## host=localhost user=pqotest port=... password=... sslmode=... dbname=... + ## + ## All connection parameters are optional, except for dbname, + ## you need to set it always as pgbouncer. + address = "host=localhost user=postgres port=6432 sslmode=disable dbname=pgbouncer" + + ## A list of databases to pull metrics about. If not specified, metrics for all + ## databases are gathered. + # databases = ["app_production", "testing"] +` +``` + +### Measurements & Fields: + +Pgbouncer provides two measurement named "pgbouncer_pools" and "pgbouncer_stats", each have the fields as below: + +#### pgbouncer_pools + +- cl_active +- cl_waiting +- maxwait +- pool_mode +- sv_active +- sv_idle +- sv_login +- sv_tested +- sv_used + +### pgbouncer_stats + +- avg_query +- avg_recv +- avg_req +- avg_sent +- total_query_time +- total_received +- total_requests +- total_sent + +More information about the meaning of these metrics can be found in the [PgBouncer usage](https://pgbouncer.github.io/usage.html) + +### Example Output: + +``` +$ ./telegraf -config telegraf.conf -input-filter pgbouncer -test +> pgbouncer_pools,db=pgbouncer,host=localhost,pool_mode=transaction,server=host\=localhost\ user\=elena\ port\=6432\ dbname\=pgbouncer\ sslmode\=disable,user=elena cl_active=1500i,cl_waiting=0i,maxwait=0i,sv_active=0i,sv_idle=5i,sv_login=0i,sv_tested=0i,sv_used=5i 1466594520564518897 +> pgbouncer_stats,db=pgbouncer,host=localhost,server=host\=localhost\ user\=elena\ port\=6432\ dbname\=pgbouncer\ sslmode\=disable avg_query=1157i,avg_recv=36727i,avg_req=131i,avg_sent=23359i,total_query_time=252173878876i,total_received=55956189078i,total_requests=193601888i,total_sent=36703848280i 1466594520564825345 +``` + diff --git a/plugins/inputs/pgbouncer/pgbouncer.go b/plugins/inputs/pgbouncer/pgbouncer.go new file mode 100644 index 000000000..df4179cd6 --- /dev/null +++ b/plugins/inputs/pgbouncer/pgbouncer.go @@ -0,0 +1,206 @@ +package pgbouncer + +import ( + "bytes" + "database/sql" + "regexp" + "strings" + + "github.com/influxdata/telegraf" + "github.com/influxdata/telegraf/plugins/inputs" + + "github.com/lib/pq" +) + +type Pgbouncer struct { + Address string + Databases []string + OrderedColumns []string + AllColumns []string + sanitizedAddress string +} + +var ignoredColumns = map[string]bool{"pool_mode": true, "database": true, "user": true} + +var sampleConfig = ` + ## specify address via a url matching: + ## postgres://[pqgotest[:password]]@localhost:port[/dbname]\ + ## ?sslmode=[disable|verify-ca|verify-full] + ## or a simple string: + ## host=localhost user=pqotest port=6432 password=... sslmode=... dbname=pgbouncer + ## + ## All connection parameters are optional, except for dbname, + ## you need to set it always as pgbouncer. + address = "host=localhost user=postgres port=6432 sslmode=disable dbname=pgbouncer" + + ## A list of databases to pull metrics about. If not specified, metrics for all + ## databases are gathered. + # databases = ["app_production", "testing"] +` + +func (p *Pgbouncer) SampleConfig() string { + return sampleConfig +} + +func (p *Pgbouncer) Description() string { + return "Read metrics from one or many pgbouncer servers" +} + +func (p *Pgbouncer) IgnoredColumns() map[string]bool { + return ignoredColumns +} + +var localhost = "host=localhost port=6432 sslmode=disable dbname=pgbouncer" + +func (p *Pgbouncer) Gather(acc telegraf.Accumulator) error { + if p.Address == "" || p.Address == "localhost" { + p.Address = localhost + } + + db, err := sql.Open("postgres", p.Address) + if err != nil { + return err + } + + defer db.Close() + + queries := map[string]string{"pools": "SHOW POOLS", "stats": "SHOW STATS"} + + for metric, query := range queries { + rows, err := db.Query(query) + if err != nil { + return err + } + + defer rows.Close() + + // grab the column information from the result + p.OrderedColumns, err = rows.Columns() + if err != nil { + return err + } else { + p.AllColumns = make([]string, len(p.OrderedColumns)) + copy(p.AllColumns, p.OrderedColumns) + } + + for rows.Next() { + err = p.accRow(rows, metric, acc) + if err != nil { + return err + } + } + } + return nil +} + +type scanner interface { + Scan(dest ...interface{}) error +} + +var passwordKVMatcher, _ = regexp.Compile("password=\\S+ ?") + +func (p *Pgbouncer) SanitizedAddress() (_ string, err error) { + var canonicalizedAddress string + if strings.HasPrefix(p.Address, "postgres://") || strings.HasPrefix(p.Address, "postgresql://") { + canonicalizedAddress, err = pq.ParseURL(p.Address) + if err != nil { + return p.sanitizedAddress, err + } + } else { + canonicalizedAddress = p.Address + } + p.sanitizedAddress = passwordKVMatcher.ReplaceAllString(canonicalizedAddress, "") + + return p.sanitizedAddress, err +} + +func (p *Pgbouncer) accRow(row scanner, metric string, acc telegraf.Accumulator) error { + var columnVars []interface{} + var tags = make(map[string]string) + var dbname, user, poolMode bytes.Buffer + + // this is where we'll store the column name with its *interface{} + columnMap := make(map[string]*interface{}) + + for _, column := range p.OrderedColumns { + columnMap[column] = new(interface{}) + } + + // populate the array of interface{} with the pointers in the right order + for i := 0; i < len(columnMap); i++ { + columnVars = append(columnVars, columnMap[p.OrderedColumns[i]]) + } + + // deconstruct array of variables and send to Scan + err := row.Scan(columnVars...) + + if err != nil { + return err + } + + // extract the database name from the column map + dbnameChars := (*columnMap["database"]).([]uint8) + for i := 0; i < len(dbnameChars); i++ { + dbname.WriteString(string(dbnameChars[i])) + } + + if p.ignoreDatabase(dbname.String()) { + return nil + } + + tags["db"] = dbname.String() + + if columnMap["user"] != nil { + userChars := (*columnMap["user"]).([]uint8) + for i := 0; i < len(userChars); i++ { + user.WriteString(string(userChars[i])) + } + tags["user"] = user.String() + } + + if columnMap["pool_mode"] != nil { + poolChars := (*columnMap["pool_mode"]).([]uint8) + for i := 0; i < len(poolChars); i++ { + poolMode.WriteString(string(poolChars[i])) + } + tags["pool_mode"] = poolMode.String() + } + + var tagAddress string + tagAddress, err = p.SanitizedAddress() + if err != nil { + return err + } else { + tags["server"] = tagAddress + } + + fields := make(map[string]interface{}) + for col, val := range columnMap { + _, ignore := ignoredColumns[col] + if !ignore { + fields[col] = *val + } + } + acc.AddFields("pgbouncer_"+metric, fields, tags) + + return nil +} + +func (p *Pgbouncer) ignoreDatabase(db string) bool { + if len(p.Databases) == 0 { + return false + } + + for _, dbName := range p.Databases { + if db == dbName { + return false + } + } + return true +} + +func init() { + inputs.Add("pgbouncer", func() telegraf.Input { + return &Pgbouncer{} + }) +} diff --git a/plugins/inputs/pgbouncer/pgbouncer_test.go b/plugins/inputs/pgbouncer/pgbouncer_test.go new file mode 100644 index 000000000..d7d244633 --- /dev/null +++ b/plugins/inputs/pgbouncer/pgbouncer_test.go @@ -0,0 +1,180 @@ +package pgbouncer + +import ( + "fmt" + "testing" + + "github.com/influxdata/telegraf/testutil" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestPgbouncerGeneratesMetrics(t *testing.T) { + if testing.Short() { + t.Skip("Skipping integration test in short mode") + } + + p := &Pgbouncer{ + Address: fmt.Sprintf("host=%s port=6432 user=postgres dbname=pgbouncer sslmode=disable", + testutil.GetLocalHost()), + Databases: []string{"pgbouncer"}, + } + + var acc testutil.Accumulator + err := p.Gather(&acc) + require.NoError(t, err) + + availableColumns := make(map[string]bool) + for _, col := range p.AllColumns { + availableColumns[col] = true + } + poolMetrics := []string{ + "cl_active", + "cl_waiting", + "maxwait", + "pool_mode", + "sv_active", + "sv_idle", + "sv_login", + "sv_tested", + "sv_used", + } + + statMetrics := []string{ + "avg_query", + "avg_recv", + "avg_req", + "avg_sent", + "total_query_time", + "total_received", + "total_requests", + "total_sent", + } + + metricsCounted := 0 + + for _, metric := range poolMetrics { + _, ok := availableColumns[metric] + if ok { + assert.True(t, acc.HasIntField("pgbouncer_pools", metric)) + metricsCounted++ + } + } + + for _, metric := range statMetrics { + _, ok := availableColumns[metric] + if ok { + assert.True(t, acc.HasIntField("pgbouncer_stats", metric)) + metricsCounted++ + } + } + + assert.True(t, metricsCounted > 0) + // assert.Equal(t, len(availableColumns)-len(p.IgnoredColumns()), metricsCounted) +} + +func TestPgbouncerTagsMetricsWithDatabaseName(t *testing.T) { + if testing.Short() { + t.Skip("Skipping integration test in short mode") + } + + p := &Pgbouncer{ + Address: fmt.Sprintf("host=%s port=6432 user=postgres dbname=pgbouncer sslmode=disable", + testutil.GetLocalHost()), + Databases: []string{"pgbouncer"}, + } + + var acc testutil.Accumulator + + err := p.Gather(&acc) + require.NoError(t, err) + + point, ok := acc.Get("pgbouncer_pools") + require.True(t, ok) + + assert.Equal(t, "pgbouncer", point.Tags["db"]) + + point, ok = acc.Get("pgbouncer_stats") + require.True(t, ok) + + assert.Equal(t, "pgbouncer", point.Tags["db"]) +} + +func TestPgbouncerTagsMetricsWithSpecifiedDatabaseName(t *testing.T) { + if testing.Short() { + t.Skip("Skipping integration test in short mode") + } + + p := &Pgbouncer{ + Address: fmt.Sprintf("host=%s port=6432 user=postgres dbname=pgbouncer sslmode=disable", + testutil.GetLocalHost()), + Databases: []string{"foo"}, + } + + var acc testutil.Accumulator + + err := p.Gather(&acc) + require.NoError(t, err) + + _, ok := acc.Get("pgbouncer_pools") + require.False(t, ok) + + _, ok = acc.Get("pgbouncer_stats") + require.False(t, ok) +} + +func TestPgbouncerDefaultsToAllDatabases(t *testing.T) { + if testing.Short() { + t.Skip("Skipping integration test in short mode") + } + + p := &Pgbouncer{ + Address: fmt.Sprintf("host=%s port=6432 user=postgres dbname=pgbouncer sslmode=disable", + testutil.GetLocalHost()), + } + + var acc testutil.Accumulator + + err := p.Gather(&acc) + require.NoError(t, err) + + var found bool + + for _, pnt := range acc.Metrics { + if pnt.Measurement == "pgbouncer_pools" { + if pnt.Tags["db"] == "pgbouncer" { + found = true + break + } + } + + if pnt.Measurement == "pgbouncer_stats" { + if pnt.Tags["db"] == "pgbouncer" { + found = true + break + } + } + } + + assert.True(t, found) +} + +func TestPgbouncerIgnoresUnwantedColumns(t *testing.T) { + if testing.Short() { + t.Skip("Skipping integration test in short mode") + } + + p := &Pgbouncer{ + Address: fmt.Sprintf("host=%s port=6432 user=postgres dbname=pgbouncer sslmode=disable", + testutil.GetLocalHost()), + } + + var acc testutil.Accumulator + + err := p.Gather(&acc) + require.NoError(t, err) + + for col := range p.IgnoredColumns() { + assert.False(t, acc.HasMeasurement(col)) + } +} From c0e895c3a7c14f82fba652f7aa80c14ed43f0954 Mon Sep 17 00:00:00 2001 From: Cameron Sparr Date: Wed, 10 Aug 2016 15:16:01 +0100 Subject: [PATCH 17/28] etc/telegraf.conf update --- etc/telegraf.conf | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/etc/telegraf.conf b/etc/telegraf.conf index 60877af20..b67a17ba6 100644 --- a/etc/telegraf.conf +++ b/etc/telegraf.conf @@ -321,14 +321,13 @@ # api_token = "my-secret-token" # required. # ## Debug # # debug = false -# ## Tag Field to populate source attribute (optional) -# ## This is typically the _hostname_ from which the metric was obtained. -# source_tag = "host" # ## Connection timeout. # # timeout = "5s" -# ## Output Name Template (same as graphite buckets) +# ## Output source Template (same as graphite buckets) # ## see https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_OUTPUT.md#graphite -# template = "host.tags.measurement.field" +# ## This template is used in librato's source (not metric's name) +# template = "host" +# # # Configuration for MQTT server to send metrics to @@ -1151,6 +1150,23 @@ # command = "passenger-status -v --show=xml" +# # Read metrics from one or many pgbouncer servers +# [[inputs.pgbouncer]] +# ## specify address via a url matching: +# ## postgres://[pqgotest[:password]]@localhost:port[/dbname]\ +# ## ?sslmode=[disable|verify-ca|verify-full] +# ## or a simple string: +# ## host=localhost user=pqotest port=6432 password=... sslmode=... dbname=pgbouncer +# ## +# ## All connection parameters are optional, except for dbname, +# ## you need to set it always as pgbouncer. +# address = "host=localhost user=postgres port=6432 sslmode=disable dbname=pgbouncer" +# +# ## A list of databases to pull metrics about. If not specified, metrics for all +# ## databases are gathered. +# # databases = ["app_production", "testing"] + + # # Read metrics of phpfpm, via HTTP status page or socket # [[inputs.phpfpm]] # ## An array of addresses to gather stats about. Specify an ip or hostname From 69e4e862a35b1da35c0846c35ad947b394f51f07 Mon Sep 17 00:00:00 2001 From: Ross McDonald Date: Wed, 10 Aug 2016 11:51:21 -0500 Subject: [PATCH 18/28] Fix typo of 'quorom' to 'quorum' when specifying write consistency. (#1618) --- etc/telegraf.conf | 2 +- plugins/outputs/influxdb/README.md | 4 ++-- plugins/outputs/influxdb/influxdb.go | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/etc/telegraf.conf b/etc/telegraf.conf index b67a17ba6..c934a89ab 100644 --- a/etc/telegraf.conf +++ b/etc/telegraf.conf @@ -83,7 +83,7 @@ ## Retention policy to write to. Empty string writes to the default rp. retention_policy = "" - ## Write consistency (clusters only), can be: "any", "one", "quorom", "all" + ## Write consistency (clusters only), can be: "any", "one", "quorum", "all" write_consistency = "any" ## Write timeout (for the InfluxDB client), formatted as a string. diff --git a/plugins/outputs/influxdb/README.md b/plugins/outputs/influxdb/README.md index 11511c3c4..864177a36 100644 --- a/plugins/outputs/influxdb/README.md +++ b/plugins/outputs/influxdb/README.md @@ -17,7 +17,7 @@ This plugin writes to [InfluxDB](https://www.influxdb.com) via HTTP or UDP. ## Retention policy to write to. Empty string writes to the default rp. retention_policy = "" - ## Write consistency (clusters only), can be: "any", "one", "quorom", "all" + ## Write consistency (clusters only), can be: "any", "one", "quorum", "all" write_consistency = "any" ## Write timeout (for the InfluxDB client), formatted as a string. @@ -48,7 +48,7 @@ to write to. Each URL should start with either `http://` or `udp://` ### Optional parameters: -* `write_consistency`: Write consistency (clusters only), can be: "any", "one", "quorom", "all". +* `write_consistency`: Write consistency (clusters only), can be: "any", "one", "quorum", "all". * `retention_policy`: Retention policy to write to. * `timeout`: Write timeout (for the InfluxDB client), formatted as a string. If not provided, will default to 5s. 0s means no timeout (not recommended). * `username`: Username for influxdb diff --git a/plugins/outputs/influxdb/influxdb.go b/plugins/outputs/influxdb/influxdb.go index 24065d114..1d6110b34 100644 --- a/plugins/outputs/influxdb/influxdb.go +++ b/plugins/outputs/influxdb/influxdb.go @@ -55,7 +55,7 @@ var sampleConfig = ` ## Retention policy to write to. Empty string writes to the default rp. retention_policy = "" - ## Write consistency (clusters only), can be: "any", "one", "quorom", "all" + ## Write consistency (clusters only), can be: "any", "one", "quorum", "all" write_consistency = "any" ## Write timeout (for the InfluxDB client), formatted as a string. From dee98612e2a34767ddf3972c49b13914264bd6ee Mon Sep 17 00:00:00 2001 From: Jack Zampolin Date: Wed, 10 Aug 2016 14:58:47 -0700 Subject: [PATCH 19/28] Modernize zookeeper readme (#1615) * Modernize zookeeper readme * Add configuration --- plugins/inputs/zookeeper/README.md | 65 +++++++++++++++--------------- 1 file changed, 32 insertions(+), 33 deletions(-) diff --git a/plugins/inputs/zookeeper/README.md b/plugins/inputs/zookeeper/README.md index bc7c17a4b..80281a87d 100644 --- a/plugins/inputs/zookeeper/README.md +++ b/plugins/inputs/zookeeper/README.md @@ -27,40 +27,39 @@ echo mntr | nc localhost 2181 zk_max_file_descriptor_count 1024 - only available on Unix platforms ``` -## Measurements: -#### Zookeeper measurements: +## Configuration -Meta: -- units: int64 -- tags: `server= port= state=` +``` +# Reads 'mntr' stats from one or many zookeeper servers +[[inputs.zookeeper]] + ## An array of address to gather stats about. Specify an ip or hostname + ## with port. ie localhost:2181, 10.0.0.1:2181, etc. -Measurement names: -- zookeeper_avg_latency -- zookeeper_max_latency -- zookeeper_min_latency -- zookeeper_packets_received -- zookeeper_packets_sent -- zookeeper_outstanding_requests -- zookeeper_znode_count -- zookeeper_watch_count -- zookeeper_ephemerals_count -- zookeeper_approximate_data_size -- zookeeper_followers #only exposed by the Leader -- zookeeper_synced_followers #only exposed by the Leader -- zookeeper_pending_syncs #only exposed by the Leader -- zookeeper_open_file_descriptor_count -- zookeeper_max_file_descriptor_count + ## If no servers are specified, then localhost is used as the host. + ## If no port is specified, 2181 is used + servers = [":2181"] +``` -#### Zookeeper string measurements: +## InfluxDB Measurement: -Meta: -- units: string -- tags: `server= port= state=` - -Measurement names: -- zookeeper_version - -### Tags: - -- All measurements have the following tags: - - +``` +M zookeeper + T host + T port + T state + + F approximate_data_size integer + F avg_latency integer + F ephemerals_count integer + F max_file_descriptor_count integer + F max_latency integer + F min_latency integer + F num_alive_connections integer + F open_file_descriptor_count integer + F outstanding_requests integer + F packets_received integer + F packets_sent integer + F version string + F watch_count integer + F znode_count integer +``` \ No newline at end of file From a282fb8524bbbbacd388e4c757ab8abefab8fded Mon Sep 17 00:00:00 2001 From: David Bayendor Date: Thu, 11 Aug 2016 02:14:56 -0600 Subject: [PATCH 20/28] Update README.md (#1622) * Update README.md Clean up minor typos and syntax. * Update README.md Fix typo in 'default' --- plugins/inputs/mesos/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/plugins/inputs/mesos/README.md b/plugins/inputs/mesos/README.md index affb66463..1d3a5f7bf 100644 --- a/plugins/inputs/mesos/README.md +++ b/plugins/inputs/mesos/README.md @@ -39,9 +39,9 @@ For more information, please check the [Mesos Observability Metrics](http://meso # slave_tasks = true ``` -By dafault this plugin is not configured to gather metrics from mesos. Since mesos cluster can be deployed in numerous ways it does not provide ane default -values in that matter. User needs to specify master/slave nodes this plugin will gather metrics from. Additionally by enabling `slave_tasks` will allow -agthering metrics from takss runing on specified slaves (this options is disabled by default). +By default this plugin is not configured to gather metrics from mesos. Since a mesos cluster can be deployed in numerous ways it does not provide any default +values. User needs to specify master/slave nodes this plugin will gather metrics from. Additionally, enabling `slave_tasks` will allow +gathering metrics from tasks running on specified slaves (this option is disabled by default). ### Measurements & Fields: From 26315bfbeae34c7feb06a9d8f304a48284785b41 Mon Sep 17 00:00:00 2001 From: politician Date: Thu, 11 Aug 2016 07:35:00 -0700 Subject: [PATCH 21/28] Defines GOOS and GOARCH for windows builds (#1621) * defines GOOS and GOARCH for windows builds * default to amd64 on windows * windows: use latest versions of missing packages --- CHANGELOG.md | 1 + Godeps_windows | 6 ++++++ Makefile | 2 +- 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4ab7df1b1..f8cef8831 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -85,6 +85,7 @@ consistent with the behavior of `collection_jitter`. ### Bugfixes +- [#1619](https://github.com/influxdata/telegraf/issues/1619): Fix `make windows` build target - [#1519](https://github.com/influxdata/telegraf/pull/1519): Fix error race conditions and partial failures. - [#1477](https://github.com/influxdata/telegraf/issues/1477): nstat: fix inaccurate config panic. - [#1481](https://github.com/influxdata/telegraf/issues/1481): jolokia: fix handling multiple multi-dimensional attributes. diff --git a/Godeps_windows b/Godeps_windows index b75da1c32..067c98c1c 100644 --- a/Godeps_windows +++ b/Godeps_windows @@ -4,3 +4,9 @@ github.com/go-ole/go-ole be49f7c07711fcb603cff39e1de7c67926dc0ba7 github.com/lxn/win 950a0e81e7678e63d8e6cd32412bdecb325ccd88 github.com/shirou/w32 3c9377fc6748f222729a8270fe2775d149a249ad golang.org/x/sys a646d33e2ee3172a661fc09bca23bb4889a41bc8 +github.com/go-ini/ini 9144852efba7c4daf409943ee90767da62d55438 +github.com/jmespath/go-jmespath bd40a432e4c76585ef6b72d3fd96fb9b6dc7b68d +github.com/pmezard/go-difflib/difflib 792786c7400a136282c1664665ae0a8db921c6c2 +github.com/stretchr/objx 1a9d0bb9f541897e62256577b352fdbc1fb4fd94 +gopkg.in/fsnotify.v1 a8a77c9133d2d6fd8334f3260d06f60e8d80a5fb +gopkg.in/tomb.v1 dd632973f1e7218eb1089048e0798ec9ae7dceb8 diff --git a/Makefile b/Makefile index 1859eba6d..9e671f210 100644 --- a/Makefile +++ b/Makefile @@ -16,7 +16,7 @@ build: go install -ldflags "-X main.version=$(VERSION)" ./... build-windows: - go build -o telegraf.exe -ldflags \ + GOOS=windows GOARCH=amd64 go build -o telegraf.exe -ldflags \ "-X main.version=$(VERSION)" \ ./cmd/telegraf/telegraf.go From 4ce8dd5f9adb8fefbeec0b1ba12eb7ef56f57866 Mon Sep 17 00:00:00 2001 From: Cameron Sparr Date: Thu, 11 Aug 2016 15:24:38 +0100 Subject: [PATCH 22/28] Rename snmp plugin to snmp_legacy --- CHANGELOG.md | 8 ++++++++ etc/telegraf.conf | 4 ++-- plugins/inputs/all/all.go | 2 +- plugins/inputs/{snmp => snmp_legacy}/README.md | 0 .../inputs/{snmp/snmp.go => snmp_legacy/snmp_legacy.go} | 6 +++--- .../snmp_test.go => snmp_legacy/snmp_legacy_test.go} | 2 +- plugins/inputs/{snmp => snmp_legacy}/testdata/oids.txt | 0 7 files changed, 15 insertions(+), 7 deletions(-) rename plugins/inputs/{snmp => snmp_legacy}/README.md (100%) rename plugins/inputs/{snmp/snmp.go => snmp_legacy/snmp_legacy.go} (99%) rename plugins/inputs/{snmp/snmp_test.go => snmp_legacy/snmp_legacy_test.go} (99%) rename plugins/inputs/{snmp => snmp_legacy}/testdata/oids.txt (100%) diff --git a/CHANGELOG.md b/CHANGELOG.md index f8cef8831..debe91be2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,14 @@ ### Release Notes +**Breaking Change** The SNMP plugin is being deprecated in it's current form. +There is a [new SNMP plugin](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/snmp) +which fixes many of the issues and confusions +of it's predecessor. For users wanting to continue to use the deprecated SNMP +plugin, you will need to change your config file from `[[inputs.snmp]]` to +`[[inputs.snmp_legacy]]`. The configuration of the new SNMP plugin is _not_ +backwards-compatible. + - Telegraf now supports being installed as an official windows service, which can be installed via `> C:\Program Files\Telegraf\telegraf.exe --service install` diff --git a/etc/telegraf.conf b/etc/telegraf.conf index c934a89ab..902c7f7fb 100644 --- a/etc/telegraf.conf +++ b/etc/telegraf.conf @@ -1393,8 +1393,8 @@ # servers = ["http://localhost:8098"] -# # Reads oids value from one or many snmp agents -# [[inputs.snmp]] +# # DEPRECATED! PLEASE USE inputs.snmp INSTEAD. +# [[inputs.snmp_legacy]] # ## Use 'oids.txt' file to translate oids to names # ## To generate 'oids.txt' you need to run: # ## snmptranslate -m all -Tz -On | sed -e 's/"//g' > /tmp/oids.txt diff --git a/plugins/inputs/all/all.go b/plugins/inputs/all/all.go index dacbff644..57f26df08 100644 --- a/plugins/inputs/all/all.go +++ b/plugins/inputs/all/all.go @@ -61,7 +61,7 @@ import ( _ "github.com/influxdata/telegraf/plugins/inputs/rethinkdb" _ "github.com/influxdata/telegraf/plugins/inputs/riak" _ "github.com/influxdata/telegraf/plugins/inputs/sensors" - _ "github.com/influxdata/telegraf/plugins/inputs/snmp" + _ "github.com/influxdata/telegraf/plugins/inputs/snmp_legacy" _ "github.com/influxdata/telegraf/plugins/inputs/sqlserver" _ "github.com/influxdata/telegraf/plugins/inputs/statsd" _ "github.com/influxdata/telegraf/plugins/inputs/sysstat" diff --git a/plugins/inputs/snmp/README.md b/plugins/inputs/snmp_legacy/README.md similarity index 100% rename from plugins/inputs/snmp/README.md rename to plugins/inputs/snmp_legacy/README.md diff --git a/plugins/inputs/snmp/snmp.go b/plugins/inputs/snmp_legacy/snmp_legacy.go similarity index 99% rename from plugins/inputs/snmp/snmp.go rename to plugins/inputs/snmp_legacy/snmp_legacy.go index 3cbfa0db1..b8b9a1232 100644 --- a/plugins/inputs/snmp/snmp.go +++ b/plugins/inputs/snmp_legacy/snmp_legacy.go @@ -1,4 +1,4 @@ -package snmp +package snmp_legacy import ( "io/ioutil" @@ -225,7 +225,7 @@ func (s *Snmp) SampleConfig() string { // Description returns description of Zookeeper plugin func (s *Snmp) Description() string { - return `Reads oids value from one or many snmp agents` + return `DEPRECATED! PLEASE USE inputs.snmp INSTEAD.` } func fillnode(parentNode Node, oid_name string, ids []string) { @@ -812,7 +812,7 @@ func (h *Host) HandleResponse( } func init() { - inputs.Add("snmp", func() telegraf.Input { + inputs.Add("snmp_legacy", func() telegraf.Input { return &Snmp{} }) } diff --git a/plugins/inputs/snmp/snmp_test.go b/plugins/inputs/snmp_legacy/snmp_legacy_test.go similarity index 99% rename from plugins/inputs/snmp/snmp_test.go rename to plugins/inputs/snmp_legacy/snmp_legacy_test.go index 2faaa1408..a6bf2922b 100644 --- a/plugins/inputs/snmp/snmp_test.go +++ b/plugins/inputs/snmp_legacy/snmp_legacy_test.go @@ -1,4 +1,4 @@ -package snmp +package snmp_legacy import ( "testing" diff --git a/plugins/inputs/snmp/testdata/oids.txt b/plugins/inputs/snmp_legacy/testdata/oids.txt similarity index 100% rename from plugins/inputs/snmp/testdata/oids.txt rename to plugins/inputs/snmp_legacy/testdata/oids.txt From 7600757f167af3ee21f32d413337dcbcf07128ce Mon Sep 17 00:00:00 2001 From: Cameron Sparr Date: Tue, 16 Aug 2016 09:06:19 +0100 Subject: [PATCH 23/28] ntpq: don't index ntp fields that dont exist closes #1634 --- CHANGELOG.md | 1 + plugins/inputs/ntpq/ntpq.go | 4 ++-- plugins/inputs/ntpq/ntpq_test.go | 34 ++++++++++++++++++++++++++++++++ 3 files changed, 37 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index debe91be2..1be99a75d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -138,6 +138,7 @@ consistent with the behavior of `collection_jitter`. - [#1586](https://github.com/influxdata/telegraf/pull/1586): Remove IF NOT EXISTS from influxdb output database creation. - [#1600](https://github.com/influxdata/telegraf/issues/1600): Fix quoting with text values in postgresql_extensible plugin. - [#1425](https://github.com/influxdata/telegraf/issues/1425): Fix win_perf_counter "index out of range" panic. +- [#1634](https://github.com/influxdata/telegraf/issues/1634): Fix ntpq panic when field is missing. ## v0.13.1 [2016-05-24] diff --git a/plugins/inputs/ntpq/ntpq.go b/plugins/inputs/ntpq/ntpq.go index e9dc1cc14..0bcaa04e5 100644 --- a/plugins/inputs/ntpq/ntpq.go +++ b/plugins/inputs/ntpq/ntpq.go @@ -119,7 +119,7 @@ func (n *NTPQ) Gather(acc telegraf.Accumulator) error { // Get integer metrics from output for key, index := range intI { - if index == -1 { + if index == -1 || index >= len(fields) { continue } if fields[index] == "-" { @@ -169,7 +169,7 @@ func (n *NTPQ) Gather(acc telegraf.Accumulator) error { // get float metrics from output for key, index := range floatI { - if index == -1 { + if index == -1 || index >= len(fields) { continue } if fields[index] == "-" { diff --git a/plugins/inputs/ntpq/ntpq_test.go b/plugins/inputs/ntpq/ntpq_test.go index 7e83243c0..4b6489949 100644 --- a/plugins/inputs/ntpq/ntpq_test.go +++ b/plugins/inputs/ntpq/ntpq_test.go @@ -41,6 +41,35 @@ func TestSingleNTPQ(t *testing.T) { acc.AssertContainsTaggedFields(t, "ntpq", fields, tags) } +func TestMissingJitterField(t *testing.T) { + tt := tester{ + ret: []byte(missingJitterField), + err: nil, + } + n := &NTPQ{ + runQ: tt.runqTest, + } + + acc := testutil.Accumulator{} + assert.NoError(t, n.Gather(&acc)) + + fields := map[string]interface{}{ + "when": int64(101), + "poll": int64(256), + "reach": int64(37), + "delay": float64(51.016), + "offset": float64(233.010), + } + tags := map[string]string{ + "remote": "uschi5-ntp-002.", + "state_prefix": "*", + "refid": "10.177.80.46", + "stratum": "2", + "type": "u", + } + acc.AssertContainsTaggedFields(t, "ntpq", fields, tags) +} + func TestBadIntNTPQ(t *testing.T) { tt := tester{ ret: []byte(badIntParseNTPQ), @@ -381,6 +410,11 @@ var singleNTPQ = ` remote refid st t when poll reach delay *uschi5-ntp-002. 10.177.80.46 2 u 101 256 37 51.016 233.010 17.462 ` +var missingJitterField = ` remote refid st t when poll reach delay offset jitter +============================================================================== +*uschi5-ntp-002. 10.177.80.46 2 u 101 256 37 51.016 233.010 +` + var badHeaderNTPQ = `remote refid foobar t when poll reach delay offset jitter ============================================================================== *uschi5-ntp-002. 10.177.80.46 2 u 101 256 37 51.016 233.010 17.462 From 94e673fe855bff7416dcb32aadb593bd52d26775 Mon Sep 17 00:00:00 2001 From: Cameron Sparr Date: Wed, 17 Aug 2016 16:50:11 +0100 Subject: [PATCH 24/28] Revert "add pgbouncer plugin" This reverts commit fec9760f72f8c45395185646c48f632ba9b38883. --- CHANGELOG.md | 1 - Makefile | 17 +- README.md | 1 - plugins/inputs/all/all.go | 1 - plugins/inputs/pgbouncer/README.md | 62 ------- plugins/inputs/pgbouncer/pgbouncer.go | 206 --------------------- plugins/inputs/pgbouncer/pgbouncer_test.go | 180 ------------------ 7 files changed, 2 insertions(+), 466 deletions(-) delete mode 100644 plugins/inputs/pgbouncer/README.md delete mode 100644 plugins/inputs/pgbouncer/pgbouncer.go delete mode 100644 plugins/inputs/pgbouncer/pgbouncer_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 1be99a75d..8725d298f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -67,7 +67,6 @@ consistent with the behavior of `collection_jitter`. - [#1434](https://github.com/influxdata/telegraf/pull/1434): Add measurement name arg to logparser plugin. - [#1479](https://github.com/influxdata/telegraf/pull/1479): logparser: change resp_code from a field to a tag. - [#1411](https://github.com/influxdata/telegraf/pull/1411): Implement support for fetching hddtemp data -- [#1400](https://github.com/influxdata/telegraf/pull/1400): Add supoort for Pgbouncer - [#1340](https://github.com/influxdata/telegraf/issues/1340): statsd: do not log every dropped metric. - [#1368](https://github.com/influxdata/telegraf/pull/1368): Add precision rounding to all metrics on collection. - [#1390](https://github.com/influxdata/telegraf/pull/1390): Add support for Tengine diff --git a/Makefile b/Makefile index 9e671f210..2951e175a 100644 --- a/Makefile +++ b/Makefile @@ -57,13 +57,6 @@ docker-run: docker run --name mqtt -p "1883:1883" -d ncarlier/mqtt docker run --name riemann -p "5555:5555" -d blalor/riemann docker run --name snmp -p "31161:31161/udp" -d titilambert/snmpsim - docker run --name pgbouncer \ - -p "6432:6432" \ - -e PGB_USERLISTS="postgres:postgres" \ - -e PGB_ADMIN_USERS="postgres" \ - -e PGB_STATS_USERS="postgres" \ - --link postgres:pg \ - -d jsvisa/pgbouncer # Run docker containers necessary for CircleCI unit tests docker-run-circle: @@ -77,17 +70,11 @@ docker-run-circle: docker run --name mqtt -p "1883:1883" -d ncarlier/mqtt docker run --name riemann -p "5555:5555" -d blalor/riemann docker run --name snmp -p "31161:31161/udp" -d titilambert/snmpsim - docker run --name pgbouncer \ - -p "6432:6432" \ - -e PGB_USERLISTS="postgres:postgres" \ - -e PGB_ADMIN_USERS="postgres" \ - -e PGB_STATS_USERS="postgres" \ - -d jsvisa/pgbouncer # Kill all docker containers, ignore errors docker-kill: - -docker kill nsq aerospike redis rabbitmq postgres memcached mysql kafka mqtt riemann snmp pgbouncer - -docker rm nsq aerospike redis rabbitmq postgres memcached mysql kafka mqtt riemann snmp pgbouncer + -docker kill nsq aerospike redis rabbitmq postgres memcached mysql kafka mqtt riemann snmp + -docker rm nsq aerospike redis rabbitmq postgres memcached mysql kafka mqtt riemann snmp # Run full unit tests using docker containers (includes setup and teardown) test: vet docker-kill docker-run diff --git a/README.md b/README.md index c9fe12351..74bbf2a4f 100644 --- a/README.md +++ b/README.md @@ -174,7 +174,6 @@ Currently implemented sources: * [nsq](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/nsq) * [nstat](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/nstat) * [ntpq](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/ntpq) -* [pgbouncer](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/pgbouncer) * [phpfpm](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/phpfpm) * [phusion passenger](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/passenger) * [ping](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/ping) diff --git a/plugins/inputs/all/all.go b/plugins/inputs/all/all.go index 57f26df08..af759aac8 100644 --- a/plugins/inputs/all/all.go +++ b/plugins/inputs/all/all.go @@ -46,7 +46,6 @@ import ( _ "github.com/influxdata/telegraf/plugins/inputs/nstat" _ "github.com/influxdata/telegraf/plugins/inputs/ntpq" _ "github.com/influxdata/telegraf/plugins/inputs/passenger" - _ "github.com/influxdata/telegraf/plugins/inputs/pgbouncer" _ "github.com/influxdata/telegraf/plugins/inputs/phpfpm" _ "github.com/influxdata/telegraf/plugins/inputs/ping" _ "github.com/influxdata/telegraf/plugins/inputs/postgresql" diff --git a/plugins/inputs/pgbouncer/README.md b/plugins/inputs/pgbouncer/README.md deleted file mode 100644 index 31e883f11..000000000 --- a/plugins/inputs/pgbouncer/README.md +++ /dev/null @@ -1,62 +0,0 @@ -# Pgbouncer plugin - -This pgbouncer plugin provides metrics for your pgbouncer connection information. - -### Configuration: - -```toml -# Description -[[inputs.pgbouncer]] - ## specify address via a url matching: - ## postgres://[pqgotest[:password]]@localhost:port[/dbname]\ - ## ?sslmode=[disable|verify-ca|verify-full] - ## or a simple string: - ## host=localhost user=pqotest port=... password=... sslmode=... dbname=... - ## - ## All connection parameters are optional, except for dbname, - ## you need to set it always as pgbouncer. - address = "host=localhost user=postgres port=6432 sslmode=disable dbname=pgbouncer" - - ## A list of databases to pull metrics about. If not specified, metrics for all - ## databases are gathered. - # databases = ["app_production", "testing"] -` -``` - -### Measurements & Fields: - -Pgbouncer provides two measurement named "pgbouncer_pools" and "pgbouncer_stats", each have the fields as below: - -#### pgbouncer_pools - -- cl_active -- cl_waiting -- maxwait -- pool_mode -- sv_active -- sv_idle -- sv_login -- sv_tested -- sv_used - -### pgbouncer_stats - -- avg_query -- avg_recv -- avg_req -- avg_sent -- total_query_time -- total_received -- total_requests -- total_sent - -More information about the meaning of these metrics can be found in the [PgBouncer usage](https://pgbouncer.github.io/usage.html) - -### Example Output: - -``` -$ ./telegraf -config telegraf.conf -input-filter pgbouncer -test -> pgbouncer_pools,db=pgbouncer,host=localhost,pool_mode=transaction,server=host\=localhost\ user\=elena\ port\=6432\ dbname\=pgbouncer\ sslmode\=disable,user=elena cl_active=1500i,cl_waiting=0i,maxwait=0i,sv_active=0i,sv_idle=5i,sv_login=0i,sv_tested=0i,sv_used=5i 1466594520564518897 -> pgbouncer_stats,db=pgbouncer,host=localhost,server=host\=localhost\ user\=elena\ port\=6432\ dbname\=pgbouncer\ sslmode\=disable avg_query=1157i,avg_recv=36727i,avg_req=131i,avg_sent=23359i,total_query_time=252173878876i,total_received=55956189078i,total_requests=193601888i,total_sent=36703848280i 1466594520564825345 -``` - diff --git a/plugins/inputs/pgbouncer/pgbouncer.go b/plugins/inputs/pgbouncer/pgbouncer.go deleted file mode 100644 index df4179cd6..000000000 --- a/plugins/inputs/pgbouncer/pgbouncer.go +++ /dev/null @@ -1,206 +0,0 @@ -package pgbouncer - -import ( - "bytes" - "database/sql" - "regexp" - "strings" - - "github.com/influxdata/telegraf" - "github.com/influxdata/telegraf/plugins/inputs" - - "github.com/lib/pq" -) - -type Pgbouncer struct { - Address string - Databases []string - OrderedColumns []string - AllColumns []string - sanitizedAddress string -} - -var ignoredColumns = map[string]bool{"pool_mode": true, "database": true, "user": true} - -var sampleConfig = ` - ## specify address via a url matching: - ## postgres://[pqgotest[:password]]@localhost:port[/dbname]\ - ## ?sslmode=[disable|verify-ca|verify-full] - ## or a simple string: - ## host=localhost user=pqotest port=6432 password=... sslmode=... dbname=pgbouncer - ## - ## All connection parameters are optional, except for dbname, - ## you need to set it always as pgbouncer. - address = "host=localhost user=postgres port=6432 sslmode=disable dbname=pgbouncer" - - ## A list of databases to pull metrics about. If not specified, metrics for all - ## databases are gathered. - # databases = ["app_production", "testing"] -` - -func (p *Pgbouncer) SampleConfig() string { - return sampleConfig -} - -func (p *Pgbouncer) Description() string { - return "Read metrics from one or many pgbouncer servers" -} - -func (p *Pgbouncer) IgnoredColumns() map[string]bool { - return ignoredColumns -} - -var localhost = "host=localhost port=6432 sslmode=disable dbname=pgbouncer" - -func (p *Pgbouncer) Gather(acc telegraf.Accumulator) error { - if p.Address == "" || p.Address == "localhost" { - p.Address = localhost - } - - db, err := sql.Open("postgres", p.Address) - if err != nil { - return err - } - - defer db.Close() - - queries := map[string]string{"pools": "SHOW POOLS", "stats": "SHOW STATS"} - - for metric, query := range queries { - rows, err := db.Query(query) - if err != nil { - return err - } - - defer rows.Close() - - // grab the column information from the result - p.OrderedColumns, err = rows.Columns() - if err != nil { - return err - } else { - p.AllColumns = make([]string, len(p.OrderedColumns)) - copy(p.AllColumns, p.OrderedColumns) - } - - for rows.Next() { - err = p.accRow(rows, metric, acc) - if err != nil { - return err - } - } - } - return nil -} - -type scanner interface { - Scan(dest ...interface{}) error -} - -var passwordKVMatcher, _ = regexp.Compile("password=\\S+ ?") - -func (p *Pgbouncer) SanitizedAddress() (_ string, err error) { - var canonicalizedAddress string - if strings.HasPrefix(p.Address, "postgres://") || strings.HasPrefix(p.Address, "postgresql://") { - canonicalizedAddress, err = pq.ParseURL(p.Address) - if err != nil { - return p.sanitizedAddress, err - } - } else { - canonicalizedAddress = p.Address - } - p.sanitizedAddress = passwordKVMatcher.ReplaceAllString(canonicalizedAddress, "") - - return p.sanitizedAddress, err -} - -func (p *Pgbouncer) accRow(row scanner, metric string, acc telegraf.Accumulator) error { - var columnVars []interface{} - var tags = make(map[string]string) - var dbname, user, poolMode bytes.Buffer - - // this is where we'll store the column name with its *interface{} - columnMap := make(map[string]*interface{}) - - for _, column := range p.OrderedColumns { - columnMap[column] = new(interface{}) - } - - // populate the array of interface{} with the pointers in the right order - for i := 0; i < len(columnMap); i++ { - columnVars = append(columnVars, columnMap[p.OrderedColumns[i]]) - } - - // deconstruct array of variables and send to Scan - err := row.Scan(columnVars...) - - if err != nil { - return err - } - - // extract the database name from the column map - dbnameChars := (*columnMap["database"]).([]uint8) - for i := 0; i < len(dbnameChars); i++ { - dbname.WriteString(string(dbnameChars[i])) - } - - if p.ignoreDatabase(dbname.String()) { - return nil - } - - tags["db"] = dbname.String() - - if columnMap["user"] != nil { - userChars := (*columnMap["user"]).([]uint8) - for i := 0; i < len(userChars); i++ { - user.WriteString(string(userChars[i])) - } - tags["user"] = user.String() - } - - if columnMap["pool_mode"] != nil { - poolChars := (*columnMap["pool_mode"]).([]uint8) - for i := 0; i < len(poolChars); i++ { - poolMode.WriteString(string(poolChars[i])) - } - tags["pool_mode"] = poolMode.String() - } - - var tagAddress string - tagAddress, err = p.SanitizedAddress() - if err != nil { - return err - } else { - tags["server"] = tagAddress - } - - fields := make(map[string]interface{}) - for col, val := range columnMap { - _, ignore := ignoredColumns[col] - if !ignore { - fields[col] = *val - } - } - acc.AddFields("pgbouncer_"+metric, fields, tags) - - return nil -} - -func (p *Pgbouncer) ignoreDatabase(db string) bool { - if len(p.Databases) == 0 { - return false - } - - for _, dbName := range p.Databases { - if db == dbName { - return false - } - } - return true -} - -func init() { - inputs.Add("pgbouncer", func() telegraf.Input { - return &Pgbouncer{} - }) -} diff --git a/plugins/inputs/pgbouncer/pgbouncer_test.go b/plugins/inputs/pgbouncer/pgbouncer_test.go deleted file mode 100644 index d7d244633..000000000 --- a/plugins/inputs/pgbouncer/pgbouncer_test.go +++ /dev/null @@ -1,180 +0,0 @@ -package pgbouncer - -import ( - "fmt" - "testing" - - "github.com/influxdata/telegraf/testutil" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestPgbouncerGeneratesMetrics(t *testing.T) { - if testing.Short() { - t.Skip("Skipping integration test in short mode") - } - - p := &Pgbouncer{ - Address: fmt.Sprintf("host=%s port=6432 user=postgres dbname=pgbouncer sslmode=disable", - testutil.GetLocalHost()), - Databases: []string{"pgbouncer"}, - } - - var acc testutil.Accumulator - err := p.Gather(&acc) - require.NoError(t, err) - - availableColumns := make(map[string]bool) - for _, col := range p.AllColumns { - availableColumns[col] = true - } - poolMetrics := []string{ - "cl_active", - "cl_waiting", - "maxwait", - "pool_mode", - "sv_active", - "sv_idle", - "sv_login", - "sv_tested", - "sv_used", - } - - statMetrics := []string{ - "avg_query", - "avg_recv", - "avg_req", - "avg_sent", - "total_query_time", - "total_received", - "total_requests", - "total_sent", - } - - metricsCounted := 0 - - for _, metric := range poolMetrics { - _, ok := availableColumns[metric] - if ok { - assert.True(t, acc.HasIntField("pgbouncer_pools", metric)) - metricsCounted++ - } - } - - for _, metric := range statMetrics { - _, ok := availableColumns[metric] - if ok { - assert.True(t, acc.HasIntField("pgbouncer_stats", metric)) - metricsCounted++ - } - } - - assert.True(t, metricsCounted > 0) - // assert.Equal(t, len(availableColumns)-len(p.IgnoredColumns()), metricsCounted) -} - -func TestPgbouncerTagsMetricsWithDatabaseName(t *testing.T) { - if testing.Short() { - t.Skip("Skipping integration test in short mode") - } - - p := &Pgbouncer{ - Address: fmt.Sprintf("host=%s port=6432 user=postgres dbname=pgbouncer sslmode=disable", - testutil.GetLocalHost()), - Databases: []string{"pgbouncer"}, - } - - var acc testutil.Accumulator - - err := p.Gather(&acc) - require.NoError(t, err) - - point, ok := acc.Get("pgbouncer_pools") - require.True(t, ok) - - assert.Equal(t, "pgbouncer", point.Tags["db"]) - - point, ok = acc.Get("pgbouncer_stats") - require.True(t, ok) - - assert.Equal(t, "pgbouncer", point.Tags["db"]) -} - -func TestPgbouncerTagsMetricsWithSpecifiedDatabaseName(t *testing.T) { - if testing.Short() { - t.Skip("Skipping integration test in short mode") - } - - p := &Pgbouncer{ - Address: fmt.Sprintf("host=%s port=6432 user=postgres dbname=pgbouncer sslmode=disable", - testutil.GetLocalHost()), - Databases: []string{"foo"}, - } - - var acc testutil.Accumulator - - err := p.Gather(&acc) - require.NoError(t, err) - - _, ok := acc.Get("pgbouncer_pools") - require.False(t, ok) - - _, ok = acc.Get("pgbouncer_stats") - require.False(t, ok) -} - -func TestPgbouncerDefaultsToAllDatabases(t *testing.T) { - if testing.Short() { - t.Skip("Skipping integration test in short mode") - } - - p := &Pgbouncer{ - Address: fmt.Sprintf("host=%s port=6432 user=postgres dbname=pgbouncer sslmode=disable", - testutil.GetLocalHost()), - } - - var acc testutil.Accumulator - - err := p.Gather(&acc) - require.NoError(t, err) - - var found bool - - for _, pnt := range acc.Metrics { - if pnt.Measurement == "pgbouncer_pools" { - if pnt.Tags["db"] == "pgbouncer" { - found = true - break - } - } - - if pnt.Measurement == "pgbouncer_stats" { - if pnt.Tags["db"] == "pgbouncer" { - found = true - break - } - } - } - - assert.True(t, found) -} - -func TestPgbouncerIgnoresUnwantedColumns(t *testing.T) { - if testing.Short() { - t.Skip("Skipping integration test in short mode") - } - - p := &Pgbouncer{ - Address: fmt.Sprintf("host=%s port=6432 user=postgres dbname=pgbouncer sslmode=disable", - testutil.GetLocalHost()), - } - - var acc testutil.Accumulator - - err := p.Gather(&acc) - require.NoError(t, err) - - for col := range p.IgnoredColumns() { - assert.False(t, acc.HasMeasurement(col)) - } -} From a0e42f8a6176ac792ebf658e697f4b364bd4042c Mon Sep 17 00:00:00 2001 From: Cameron Sparr Date: Wed, 17 Aug 2016 16:20:32 +0100 Subject: [PATCH 25/28] Sanitize graphite characters in field names also sanitize the names at a higher scope for better clarity closes #1637 --- CHANGELOG.md | 1 + plugins/serializers/graphite/graphite.go | 10 +- plugins/serializers/graphite/graphite_test.go | 92 ++++++++++++++++--- 3 files changed, 84 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8725d298f..5bdd04142 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -138,6 +138,7 @@ consistent with the behavior of `collection_jitter`. - [#1600](https://github.com/influxdata/telegraf/issues/1600): Fix quoting with text values in postgresql_extensible plugin. - [#1425](https://github.com/influxdata/telegraf/issues/1425): Fix win_perf_counter "index out of range" panic. - [#1634](https://github.com/influxdata/telegraf/issues/1634): Fix ntpq panic when field is missing. +- [#1637](https://github.com/influxdata/telegraf/issues/1637): Sanitize graphite output field names. ## v0.13.1 [2016-05-24] diff --git a/plugins/serializers/graphite/graphite.go b/plugins/serializers/graphite/graphite.go index 2cc4add56..6a6fd9cac 100644 --- a/plugins/serializers/graphite/graphite.go +++ b/plugins/serializers/graphite/graphite.go @@ -12,7 +12,7 @@ const DEFAULT_TEMPLATE = "host.tags.measurement.field" var ( fieldDeleter = strings.NewReplacer(".FIELDNAME", "", "FIELDNAME.", "") - sanitizedChars = strings.NewReplacer("/", "-", "@", "-", "*", "-", " ", "_", "..", ".") + sanitizedChars = strings.NewReplacer("/", "-", "@", "-", "*", "-", " ", "_", "..", ".", `\`, "") ) type GraphiteSerializer struct { @@ -36,8 +36,8 @@ func (s *GraphiteSerializer) Serialize(metric telegraf.Metric) ([]string, error) valueS := fmt.Sprintf("%#v", value) point := fmt.Sprintf("%s %s %d", // insert "field" section of template - InsertField(bucket, fieldName), - valueS, + sanitizedChars.Replace(InsertField(bucket, fieldName)), + sanitizedChars.Replace(valueS), timestamp) out = append(out, point) } @@ -100,9 +100,9 @@ func SerializeBucketName( } if prefix == "" { - return sanitizedChars.Replace(strings.Join(out, ".")) + return strings.Join(out, ".") } - return sanitizedChars.Replace(prefix + "." + strings.Join(out, ".")) + return prefix + "." + strings.Join(out, ".") } // InsertField takes the bucket string from SerializeBucketName and replaces the diff --git a/plugins/serializers/graphite/graphite_test.go b/plugins/serializers/graphite/graphite_test.go index 50ba0e2e0..57196b861 100644 --- a/plugins/serializers/graphite/graphite_test.go +++ b/plugins/serializers/graphite/graphite_test.go @@ -160,6 +160,58 @@ func TestSerializeValueField2(t *testing.T) { assert.Equal(t, expS, mS) } +// test that fields with spaces get fixed. +func TestSerializeFieldWithSpaces(t *testing.T) { + now := time.Now() + tags := map[string]string{ + "host": "localhost", + "cpu": "cpu0", + "datacenter": "us-west-2", + } + fields := map[string]interface{}{ + `field\ with\ spaces`: float64(91.5), + } + m, err := telegraf.NewMetric("cpu", tags, fields, now) + assert.NoError(t, err) + + s := GraphiteSerializer{ + Template: "host.tags.measurement.field", + } + mS, err := s.Serialize(m) + assert.NoError(t, err) + + expS := []string{ + fmt.Sprintf("localhost.cpu0.us-west-2.cpu.field_with_spaces 91.5 %d", now.Unix()), + } + assert.Equal(t, expS, mS) +} + +// test that tags with spaces get fixed. +func TestSerializeTagWithSpaces(t *testing.T) { + now := time.Now() + tags := map[string]string{ + "host": "localhost", + "cpu": `cpu\ 0`, + "datacenter": "us-west-2", + } + fields := map[string]interface{}{ + `field_with_spaces`: float64(91.5), + } + m, err := telegraf.NewMetric("cpu", tags, fields, now) + assert.NoError(t, err) + + s := GraphiteSerializer{ + Template: "host.tags.measurement.field", + } + mS, err := s.Serialize(m) + assert.NoError(t, err) + + expS := []string{ + fmt.Sprintf("localhost.cpu_0.us-west-2.cpu.field_with_spaces 91.5 %d", now.Unix()), + } + assert.Equal(t, expS, mS) +} + // test that a field named "value" gets ignored at beginning of template. func TestSerializeValueField3(t *testing.T) { now := time.Now() @@ -186,6 +238,32 @@ func TestSerializeValueField3(t *testing.T) { assert.Equal(t, expS, mS) } +// test that a field named "value" gets ignored at beginning of template. +func TestSerializeValueField5(t *testing.T) { + now := time.Now() + tags := map[string]string{ + "host": "localhost", + "cpu": "cpu0", + "datacenter": "us-west-2", + } + fields := map[string]interface{}{ + "value": float64(91.5), + } + m, err := telegraf.NewMetric("cpu", tags, fields, now) + assert.NoError(t, err) + + s := GraphiteSerializer{ + Template: template5, + } + mS, err := s.Serialize(m) + assert.NoError(t, err) + + expS := []string{ + fmt.Sprintf("localhost.us-west-2.cpu0.cpu 91.5 %d", now.Unix()), + } + assert.Equal(t, expS, mS) +} + func TestSerializeMetricPrefix(t *testing.T) { now := time.Now() tags := map[string]string{ @@ -315,20 +393,6 @@ func TestTemplate4(t *testing.T) { assert.Equal(t, expS, mS) } -func TestTemplate5(t *testing.T) { - now := time.Now() - fields := map[string]interface{}{ - "usage_idle": float64(91.5), - } - m, err := telegraf.NewMetric("cpu", defaultTags, fields, now) - assert.NoError(t, err) - - mS := SerializeBucketName(m.Name(), m.Tags(), template5, "") - - expS := "localhost.us-west-2.cpu0.cpu.FIELDNAME" - assert.Equal(t, expS, mS) -} - func TestTemplate6(t *testing.T) { now := time.Now() fields := map[string]interface{}{ From dbf6380e4b4df61a0ad8d2d32c8a1325fa3307b2 Mon Sep 17 00:00:00 2001 From: Cameron Sparr Date: Wed, 17 Aug 2016 18:24:06 +0100 Subject: [PATCH 26/28] update PR template with changelog note --- .github/PULL_REQUEST_TEMPLATE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 19bb38765..2e838a8e4 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,5 +1,5 @@ ### Required for all PRs: -- [ ] CHANGELOG.md updated +- [ ] CHANGELOG.md updated (we recommend not updating this until the PR has been approved by a maintainer) - [ ] Sign [CLA](https://influxdata.com/community/cla/) (if not already signed) - [ ] README.md updated (if adding a new plugin) From b022b5567d356b8b7f373818729600f048d55a23 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Robert=20K=C3=A1nia?= Date: Mon, 22 Aug 2016 16:35:39 +0200 Subject: [PATCH 27/28] Added missing column, refs #1646 (#1647) --- plugins/inputs/mysql/mysql.go | 1 + 1 file changed, 1 insertion(+) diff --git a/plugins/inputs/mysql/mysql.go b/plugins/inputs/mysql/mysql.go index 10b8c2f75..63744d88c 100644 --- a/plugins/inputs/mysql/mysql.go +++ b/plugins/inputs/mysql/mysql.go @@ -1376,6 +1376,7 @@ func (m *Mysql) gatherPerfEventsStatements(db *sql.DB, serv string, acc telegraf &rowsAffected, &rowsSent, &rowsExamined, &tmpTables, &tmpDiskTables, &sortMergePasses, &sortRows, + &noIndexUsed, ) if err != nil { From 7fac74919cee68d693ca236797b10897733f70c3 Mon Sep 17 00:00:00 2001 From: Patrick Hemmer Date: Mon, 22 Aug 2016 11:37:53 -0400 Subject: [PATCH 28/28] Alternate SNMP plugin (#1389) * Add a new and improved snmp plugin * update gosnmp for duplicate packet fix https://github.com/soniah/gosnmp/issues/68 https://github.com/soniah/gosnmp/pull/69 --- CHANGELOG.md | 1 + Godeps | 2 +- README.md | 1 + plugins/inputs/all/all.go | 1 + plugins/inputs/snmp/README.md | 167 +++++ plugins/inputs/snmp/snmp.go | 791 ++++++++++++++++++++++++ plugins/inputs/snmp/snmp_test.go | 641 +++++++++++++++++++ plugins/inputs/snmp/testdata/snmpd.conf | 17 + plugins/inputs/snmp/testdata/test.mib | 51 ++ 9 files changed, 1671 insertions(+), 1 deletion(-) create mode 100644 plugins/inputs/snmp/README.md create mode 100644 plugins/inputs/snmp/snmp.go create mode 100644 plugins/inputs/snmp/snmp_test.go create mode 100644 plugins/inputs/snmp/testdata/snmpd.conf create mode 100644 plugins/inputs/snmp/testdata/test.mib diff --git a/CHANGELOG.md b/CHANGELOG.md index 5bdd04142..33576d324 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -89,6 +89,7 @@ consistent with the behavior of `collection_jitter`. - [#1213](https://github.com/influxdata/telegraf/issues/1213): Add inactive & active memory to mem plugin. - [#1543](https://github.com/influxdata/telegraf/pull/1543): Official Windows service. - [#1414](https://github.com/influxdata/telegraf/pull/1414): Forking sensors command to remove C package dependency. +- [#1389](https://github.com/influxdata/telegraf/pull/1389): Add a new SNMP plugin. ### Bugfixes diff --git a/Godeps b/Godeps index 3c70bcaf8..bbbd14a24 100644 --- a/Godeps +++ b/Godeps @@ -47,7 +47,7 @@ github.com/prometheus/common e8eabff8812b05acf522b45fdcd725a785188e37 github.com/prometheus/procfs 406e5b7bfd8201a36e2bb5f7bdae0b03380c2ce8 github.com/samuel/go-zookeeper 218e9c81c0dd8b3b18172b2bbfad92cc7d6db55f github.com/shirou/gopsutil 4d0c402af66c78735c5ccf820dc2ca7de5e4ff08 -github.com/soniah/gosnmp b1b4f885b12c5dcbd021c5cee1c904110de6db7d +github.com/soniah/gosnmp eb32571c2410868d85849ad67d1e51d01273eb84 github.com/sparrc/aerospike-client-go d4bb42d2c2d39dae68e054116f4538af189e05d5 github.com/streadway/amqp b4f3ceab0337f013208d31348b578d83c0064744 github.com/stretchr/testify 1f4a1643a57e798696635ea4c126e9127adb7d3c diff --git a/README.md b/README.md index 74bbf2a4f..6e9e186e3 100644 --- a/README.md +++ b/README.md @@ -190,6 +190,7 @@ Currently implemented sources: * [riak](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/riak) * [sensors](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/sensors) * [snmp](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/snmp) +* [snmp_legacy](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/snmp_legacy) * [sql server](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/sqlserver) (microsoft) * [twemproxy](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/twemproxy) * [varnish](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/varnish) diff --git a/plugins/inputs/all/all.go b/plugins/inputs/all/all.go index af759aac8..e9fd9873a 100644 --- a/plugins/inputs/all/all.go +++ b/plugins/inputs/all/all.go @@ -60,6 +60,7 @@ import ( _ "github.com/influxdata/telegraf/plugins/inputs/rethinkdb" _ "github.com/influxdata/telegraf/plugins/inputs/riak" _ "github.com/influxdata/telegraf/plugins/inputs/sensors" + _ "github.com/influxdata/telegraf/plugins/inputs/snmp" _ "github.com/influxdata/telegraf/plugins/inputs/snmp_legacy" _ "github.com/influxdata/telegraf/plugins/inputs/sqlserver" _ "github.com/influxdata/telegraf/plugins/inputs/statsd" diff --git a/plugins/inputs/snmp/README.md b/plugins/inputs/snmp/README.md new file mode 100644 index 000000000..b5a694abd --- /dev/null +++ b/plugins/inputs/snmp/README.md @@ -0,0 +1,167 @@ +# SNMP Plugin + +The SNMP input plugin gathers metrics from SNMP agents. + +## Configuration: + +### Example: + +SNMP data: +``` +.1.0.0.0.1.1.0 octet_str "foo" +.1.0.0.0.1.1.1 octet_str "bar" +.1.0.0.0.1.102 octet_str "bad" +.1.0.0.0.1.2.0 integer 1 +.1.0.0.0.1.2.1 integer 2 +.1.0.0.0.1.3.0 octet_str "0.123" +.1.0.0.0.1.3.1 octet_str "0.456" +.1.0.0.0.1.3.2 octet_str "9.999" +.1.0.0.1.1 octet_str "baz" +.1.0.0.1.2 uinteger 54321 +.1.0.0.1.3 uinteger 234 +``` + +Telegraf config: +```toml +[[inputs.snmp]] + agents = [ "127.0.0.1:161" ] + version = 2 + community = "public" + + name = "system" + [[inputs.snmp.field]] + name = "hostname" + oid = ".1.0.0.1.1" + is_tag = true + [[inputs.snmp.field]] + name = "uptime" + oid = ".1.0.0.1.2" + [[inputs.snmp.field]] + name = "loadavg" + oid = ".1.0.0.1.3" + conversion = "float(2)" + + [[inputs.snmp.table]] + name = "remote_servers" + inherit_tags = [ "hostname" ] + [[inputs.snmp.table.field]] + name = "server" + oid = ".1.0.0.0.1.1" + is_tag = true + [[inputs.snmp.table.field]] + name = "connections" + oid = ".1.0.0.0.1.2" + [[inputs.snmp.table.field]] + name = "latency" + oid = ".1.0.0.0.1.3" + conversion = "float" +``` + +Resulting output: +``` +* Plugin: snmp, Collection 1 +> system,agent_host=127.0.0.1,host=mylocalhost,hostname=baz loadavg=2.34,uptime=54321i 1468953135000000000 +> remote_servers,agent_host=127.0.0.1,host=mylocalhost,hostname=baz,server=foo connections=1i,latency=0.123 1468953135000000000 +> remote_servers,agent_host=127.0.0.1,host=mylocalhost,hostname=baz,server=bar connections=2i,latency=0.456 1468953135000000000 +``` + +#### Configuration via MIB: + +This example uses the SNMP data above, but is configured via the MIB. +The example MIB file can be found in the `testdata` directory. See the [MIB lookups](#mib-lookups) section for more information. + +Telegraf config: +```toml +[[inputs.snmp]] + agents = [ "127.0.0.1:161" ] + version = 2 + community = "public" + + [[inputs.snmp.field]] + oid = "TEST::hostname" + is_tag = true + + [[inputs.snmp.table]] + oid = "TEST::testTable" + inherit_tags = "hostname" +``` + +Resulting output: +``` +* Plugin: snmp, Collection 1 +> testTable,agent_host=127.0.0.1,host=mylocalhost,hostname=baz,server=foo connections=1i,latency="0.123" 1468953135000000000 +> testTable,agent_host=127.0.0.1,host=mylocalhost,hostname=baz,server=bar connections=2i,latency="0.456" 1468953135000000000 +``` + +### Config parameters + +* `agents`: Default: `[]` +List of SNMP agents to connect to in the form of `IP[:PORT]`. If `:PORT` is unspecified, it defaults to `161`. + +* `version`: Default: `2` +SNMP protocol version to use. + +* `community`: Default: `"public"` +SNMP community to use. + +* `max_repetitions`: Default: `50` +Maximum number of iterations for repeating variables. + +* `sec_name`: +Security name for authenticated SNMPv3 requests. + +* `auth_protocol`: Values: `"MD5"`,`"SHA"`,`""`. Default: `""` +Authentication protocol for authenticated SNMPv3 requests. + +* `auth_password`: +Authentication password for authenticated SNMPv3 requests. + +* `sec_level`: Values: `"noAuthNoPriv"`,`"authNoPriv"`,`"authPriv"`. Default: `"noAuthNoPriv"` +Security level used for SNMPv3 messages. + +* `context_name`: +Context name used for SNMPv3 requests. + +* `priv_protocol`: Values: `"DES"`,`"AES"`,`""`. Default: `""` +Privacy protocol used for encrypted SNMPv3 messages. + +* `priv_password`: +Privacy password used for encrypted SNMPv3 messages. + + +* `name`: +Output measurement name. + +#### Field parameters: +* `oid`: +OID to get. May be a numeric or textual OID. + +* `name`: +Output field/tag name. +If not specified, it defaults to the value of `oid`. If `oid` is numeric, an attempt to translate the numeric OID into a texual OID will be made. + +* `is_tag`: +Output this field as a tag. + +* `conversion`: Values: `"float(X)"`,`"float"`,`"int"`,`""`. Default: `""` +Converts the value according to the given specification. + + - `float(X)`: Converts the input value into a float and divides by the Xth power of 10. Efficively just moves the decimal left X places. For example a value of `123` with `float(2)` will result in `1.23`. + - `float`: Converts the value into a float with no adjustment. Same as `float(0)`. + - `int`: Convertes the value into an integer. + +#### Table parameters: +* `oid`: +Automatically populates the table's fields using data from the MIB. + +* `name`: +Output measurement name. +If not specified, it defaults to the value of `oid`. If `oid` is numeric, an attempt to translate the numeric OID into a texual OID will be made. + +* `inherit_tags`: +Which tags to inherit from the top-level config and to use in the output of this table's measurement. + +### MIB lookups +If the plugin is configured such that it needs to perform lookups from the MIB, it will use the net-snmp utilities `snmptranslate` and `snmptable`. + +When performing the lookups, the plugin will load all available MIBs. If your MIB files are in a custom path, you may add the path using the `MIBDIRS` environment variable. See [`man 1 snmpcmd`](http://net-snmp.sourceforge.net/docs/man/snmpcmd.html#lbAK) for more information on the variable. diff --git a/plugins/inputs/snmp/snmp.go b/plugins/inputs/snmp/snmp.go new file mode 100644 index 000000000..3cd8968b4 --- /dev/null +++ b/plugins/inputs/snmp/snmp.go @@ -0,0 +1,791 @@ +package snmp + +import ( + "bytes" + "fmt" + "math" + "net" + "os/exec" + "strconv" + "strings" + "time" + + "github.com/influxdata/telegraf" + "github.com/influxdata/telegraf/internal" + "github.com/influxdata/telegraf/plugins/inputs" + + "github.com/soniah/gosnmp" +) + +const description = `Retrieves SNMP values from remote agents` +const sampleConfig = ` + agents = [ "127.0.0.1:161" ] + timeout = "5s" + version = 2 + + # SNMPv1 & SNMPv2 parameters + community = "public" + + # SNMPv2 & SNMPv3 parameters + max_repetitions = 50 + + # SNMPv3 parameters + #sec_name = "myuser" + #auth_protocol = "md5" # Values: "MD5", "SHA", "" + #auth_password = "password123" + #sec_level = "authNoPriv" # Values: "noAuthNoPriv", "authNoPriv", "authPriv" + #context_name = "" + #priv_protocol = "" # Values: "DES", "AES", "" + #priv_password = "" + + # measurement name + name = "system" + [[inputs.snmp.field]] + name = "hostname" + oid = ".1.0.0.1.1" + [[inputs.snmp.field]] + name = "uptime" + oid = ".1.0.0.1.2" + [[inputs.snmp.field]] + name = "load" + oid = ".1.0.0.1.3" + [[inputs.snmp.field]] + oid = "HOST-RESOURCES-MIB::hrMemorySize" + + [[inputs.snmp.table]] + # measurement name + name = "remote_servers" + inherit_tags = [ "hostname" ] + [[inputs.snmp.table.field]] + name = "server" + oid = ".1.0.0.0.1.0" + is_tag = true + [[inputs.snmp.table.field]] + name = "connections" + oid = ".1.0.0.0.1.1" + [[inputs.snmp.table.field]] + name = "latency" + oid = ".1.0.0.0.1.2" + + [[inputs.snmp.table]] + # auto populate table's fields using the MIB + oid = "HOST-RESOURCES-MIB::hrNetworkTable" +` + +// execCommand is so tests can mock out exec.Command usage. +var execCommand = exec.Command + +// execCmd executes the specified command, returning the STDOUT content. +// If command exits with error status, the output is captured into the returned error. +func execCmd(arg0 string, args ...string) ([]byte, error) { + out, err := execCommand(arg0, args...).Output() + if err != nil { + if err, ok := err.(*exec.ExitError); ok { + return nil, NestedError{ + Err: err, + NestedErr: fmt.Errorf("%s", bytes.TrimRight(err.Stderr, "\n")), + } + } + return nil, err + } + return out, nil +} + +// Snmp holds the configuration for the plugin. +type Snmp struct { + // The SNMP agent to query. Format is ADDR[:PORT] (e.g. 1.2.3.4:161). + Agents []string + // Timeout to wait for a response. + Timeout internal.Duration + Retries int + // Values: 1, 2, 3 + Version uint8 + + // Parameters for Version 1 & 2 + Community string + + // Parameters for Version 2 & 3 + MaxRepetitions uint + + // Parameters for Version 3 + ContextName string + // Values: "noAuthNoPriv", "authNoPriv", "authPriv" + SecLevel string + SecName string + // Values: "MD5", "SHA", "". Default: "" + AuthProtocol string + AuthPassword string + // Values: "DES", "AES", "". Default: "" + PrivProtocol string + PrivPassword string + EngineID string + EngineBoots uint32 + EngineTime uint32 + + Tables []Table `toml:"table"` + + // Name & Fields are the elements of a Table. + // Telegraf chokes if we try to embed a Table. So instead we have to embed the + // fields of a Table, and construct a Table during runtime. + Name string + Fields []Field `toml:"field"` + + connectionCache map[string]snmpConnection + initialized bool +} + +func (s *Snmp) init() error { + if s.initialized { + return nil + } + + for i := range s.Tables { + if err := s.Tables[i].init(); err != nil { + return err + } + } + + for i := range s.Fields { + if err := s.Fields[i].init(); err != nil { + return err + } + } + + s.initialized = true + return nil +} + +// Table holds the configuration for a SNMP table. +type Table struct { + // Name will be the name of the measurement. + Name string + + // Which tags to inherit from the top-level config. + InheritTags []string + + // Fields is the tags and values to look up. + Fields []Field `toml:"field"` + + // OID for automatic field population. + // If provided, init() will populate Fields with all the table columns of the + // given OID. + Oid string + + initialized bool +} + +// init() populates Fields if a table OID is provided. +func (t *Table) init() error { + if t.initialized { + return nil + } + if t.Oid == "" { + t.initialized = true + return nil + } + + mibPrefix := "" + if err := snmpTranslate(&mibPrefix, &t.Oid, &t.Name); err != nil { + return err + } + + // first attempt to get the table's tags + tagOids := map[string]struct{}{} + // We have to guess that the "entry" oid is `t.Oid+".1"`. snmptable and snmptranslate don't seem to have a way to provide the info. + if out, err := execCmd("snmptranslate", "-m", "all", "-Td", t.Oid+".1"); err == nil { + lines := bytes.Split(out, []byte{'\n'}) + // get the MIB name if we didn't get it above + if mibPrefix == "" { + if i := bytes.Index(lines[0], []byte("::")); i != -1 { + mibPrefix = string(lines[0][:i+2]) + } + } + + for _, line := range lines { + if !bytes.HasPrefix(line, []byte(" INDEX")) { + continue + } + + i := bytes.Index(line, []byte("{ ")) + if i == -1 { // parse error + continue + } + line = line[i+2:] + i = bytes.Index(line, []byte(" }")) + if i == -1 { // parse error + continue + } + line = line[:i] + for _, col := range bytes.Split(line, []byte(", ")) { + tagOids[mibPrefix+string(col)] = struct{}{} + } + } + } + + // this won't actually try to run a query. The `-Ch` will just cause it to dump headers. + out, err := execCmd("snmptable", "-m", "all", "-Ch", "-Cl", "-c", "public", "127.0.0.1", t.Oid) + if err != nil { + return Errorf(err, "getting table columns for %s", t.Oid) + } + cols := bytes.SplitN(out, []byte{'\n'}, 2)[0] + if len(cols) == 0 { + return fmt.Errorf("unable to get columns for table %s", t.Oid) + } + for _, col := range bytes.Split(cols, []byte{' '}) { + if len(col) == 0 { + continue + } + col := string(col) + _, isTag := tagOids[mibPrefix+col] + t.Fields = append(t.Fields, Field{Name: col, Oid: mibPrefix + col, IsTag: isTag}) + } + + // initialize all the nested fields + for i := range t.Fields { + if err := t.Fields[i].init(); err != nil { + return err + } + } + + t.initialized = true + return nil +} + +// Field holds the configuration for a Field to look up. +type Field struct { + // Name will be the name of the field. + Name string + // OID is prefix for this field. The plugin will perform a walk through all + // OIDs with this as their parent. For each value found, the plugin will strip + // off the OID prefix, and use the remainder as the index. For multiple fields + // to show up in the same row, they must share the same index. + Oid string + // IsTag controls whether this OID is output as a tag or a value. + IsTag bool + // Conversion controls any type conversion that is done on the value. + // "float"/"float(0)" will convert the value into a float. + // "float(X)" will convert the value into a float, and then move the decimal before Xth right-most digit. + // "int" will conver the value into an integer. + Conversion string + + initialized bool +} + +// init() converts OID names to numbers, and sets the .Name attribute if unset. +func (f *Field) init() error { + if f.initialized { + return nil + } + + if err := snmpTranslate(nil, &f.Oid, &f.Name); err != nil { + return err + } + + //TODO use textual convention conversion from the MIB + + f.initialized = true + return nil +} + +// RTable is the resulting table built from a Table. +type RTable struct { + // Name is the name of the field, copied from Table.Name. + Name string + // Time is the time the table was built. + Time time.Time + // Rows are the rows that were found, one row for each table OID index found. + Rows []RTableRow +} + +// RTableRow is the resulting row containing all the OID values which shared +// the same index. +type RTableRow struct { + // Tags are all the Field values which had IsTag=true. + Tags map[string]string + // Fields are all the Field values which had IsTag=false. + Fields map[string]interface{} +} + +// NestedError wraps an error returned from deeper in the code. +type NestedError struct { + // Err is the error from where the NestedError was constructed. + Err error + // NestedError is the error that was passed back from the called function. + NestedErr error +} + +// Error returns a concatenated string of all the nested errors. +func (ne NestedError) Error() string { + return ne.Err.Error() + ": " + ne.NestedErr.Error() +} + +// Errorf is a convenience function for constructing a NestedError. +func Errorf(err error, msg string, format ...interface{}) error { + return NestedError{ + NestedErr: err, + Err: fmt.Errorf(msg, format...), + } +} + +func init() { + inputs.Add("snmp", func() telegraf.Input { + return &Snmp{ + Retries: 5, + MaxRepetitions: 50, + Timeout: internal.Duration{Duration: 5 * time.Second}, + Version: 2, + Community: "public", + } + }) +} + +// SampleConfig returns the default configuration of the input. +func (s *Snmp) SampleConfig() string { + return sampleConfig +} + +// Description returns a one-sentence description on the input. +func (s *Snmp) Description() string { + return description +} + +// Gather retrieves all the configured fields and tables. +// Any error encountered does not halt the process. The errors are accumulated +// and returned at the end. +func (s *Snmp) Gather(acc telegraf.Accumulator) error { + if err := s.init(); err != nil { + return err + } + + for _, agent := range s.Agents { + gs, err := s.getConnection(agent) + if err != nil { + acc.AddError(Errorf(err, "agent %s", agent)) + continue + } + + // First is the top-level fields. We treat the fields as table prefixes with an empty index. + t := Table{ + Name: s.Name, + Fields: s.Fields, + } + topTags := map[string]string{} + if err := s.gatherTable(acc, gs, t, topTags, false); err != nil { + acc.AddError(Errorf(err, "agent %s", agent)) + } + + // Now is the real tables. + for _, t := range s.Tables { + if err := s.gatherTable(acc, gs, t, topTags, true); err != nil { + acc.AddError(Errorf(err, "agent %s", agent)) + } + } + } + + return nil +} + +func (s *Snmp) gatherTable(acc telegraf.Accumulator, gs snmpConnection, t Table, topTags map[string]string, walk bool) error { + rt, err := t.Build(gs, walk) + if err != nil { + return err + } + + for _, tr := range rt.Rows { + if !walk { + // top-level table. Add tags to topTags. + for k, v := range tr.Tags { + topTags[k] = v + } + } else { + // real table. Inherit any specified tags. + for _, k := range t.InheritTags { + if v, ok := topTags[k]; ok { + tr.Tags[k] = v + } + } + } + if _, ok := tr.Tags["agent_host"]; !ok { + tr.Tags["agent_host"] = gs.Host() + } + acc.AddFields(rt.Name, tr.Fields, tr.Tags, rt.Time) + } + + return nil +} + +// Build retrieves all the fields specified in the table and constructs the RTable. +func (t Table) Build(gs snmpConnection, walk bool) (*RTable, error) { + rows := map[string]RTableRow{} + + tagCount := 0 + for _, f := range t.Fields { + if f.IsTag { + tagCount++ + } + + if len(f.Oid) == 0 { + return nil, fmt.Errorf("cannot have empty OID") + } + var oid string + if f.Oid[0] == '.' { + oid = f.Oid + } else { + // make sure OID has "." because the BulkWalkAll results do, and the prefix needs to match + oid = "." + f.Oid + } + + // ifv contains a mapping of table OID index to field value + ifv := map[string]interface{}{} + + if !walk { + // This is used when fetching non-table fields. Fields configured a the top + // scope of the plugin. + // We fetch the fields directly, and add them to ifv as if the index were an + // empty string. This results in all the non-table fields sharing the same + // index, and being added on the same row. + if pkt, err := gs.Get([]string{oid}); err != nil { + return nil, Errorf(err, "performing get") + } else if pkt != nil && len(pkt.Variables) > 0 && pkt.Variables[0].Type != gosnmp.NoSuchObject { + ent := pkt.Variables[0] + ifv[ent.Name[len(oid):]] = fieldConvert(f.Conversion, ent.Value) + } + } else { + err := gs.Walk(oid, func(ent gosnmp.SnmpPDU) error { + if len(ent.Name) <= len(oid) || ent.Name[:len(oid)+1] != oid+"." { + return NestedError{} // break the walk + } + ifv[ent.Name[len(oid):]] = fieldConvert(f.Conversion, ent.Value) + return nil + }) + if err != nil { + if _, ok := err.(NestedError); !ok { + return nil, Errorf(err, "performing bulk walk") + } + } + } + + for i, v := range ifv { + rtr, ok := rows[i] + if !ok { + rtr = RTableRow{} + rtr.Tags = map[string]string{} + rtr.Fields = map[string]interface{}{} + rows[i] = rtr + } + if f.IsTag { + if vs, ok := v.(string); ok { + rtr.Tags[f.Name] = vs + } else { + rtr.Tags[f.Name] = fmt.Sprintf("%v", v) + } + } else { + rtr.Fields[f.Name] = v + } + } + } + + rt := RTable{ + Name: t.Name, + Time: time.Now(), //TODO record time at start + Rows: make([]RTableRow, 0, len(rows)), + } + for _, r := range rows { + if len(r.Tags) < tagCount { + // don't add rows which are missing tags, as without tags you can't filter + continue + } + rt.Rows = append(rt.Rows, r) + } + return &rt, nil +} + +// snmpConnection is an interface which wraps a *gosnmp.GoSNMP object. +// We interact through an interface so we can mock it out in tests. +type snmpConnection interface { + Host() string + //BulkWalkAll(string) ([]gosnmp.SnmpPDU, error) + Walk(string, gosnmp.WalkFunc) error + Get(oids []string) (*gosnmp.SnmpPacket, error) +} + +// gosnmpWrapper wraps a *gosnmp.GoSNMP object so we can use it as a snmpConnection. +type gosnmpWrapper struct { + *gosnmp.GoSNMP +} + +// Host returns the value of GoSNMP.Target. +func (gsw gosnmpWrapper) Host() string { + return gsw.Target +} + +// Walk wraps GoSNMP.Walk() or GoSNMP.BulkWalk(), depending on whether the +// connection is using SNMPv1 or newer. +// Also, if any error is encountered, it will just once reconnect and try again. +func (gsw gosnmpWrapper) Walk(oid string, fn gosnmp.WalkFunc) error { + var err error + // On error, retry once. + // Unfortunately we can't distinguish between an error returned by gosnmp, and one returned by the walk function. + for i := 0; i < 2; i++ { + if gsw.Version == gosnmp.Version1 { + err = gsw.GoSNMP.Walk(oid, fn) + } else { + err = gsw.GoSNMP.BulkWalk(oid, fn) + } + if err == nil { + return nil + } + if err := gsw.GoSNMP.Connect(); err != nil { + return Errorf(err, "reconnecting") + } + } + return err +} + +// Get wraps GoSNMP.GET(). +// If any error is encountered, it will just once reconnect and try again. +func (gsw gosnmpWrapper) Get(oids []string) (*gosnmp.SnmpPacket, error) { + var err error + var pkt *gosnmp.SnmpPacket + for i := 0; i < 2; i++ { + pkt, err = gsw.GoSNMP.Get(oids) + if err == nil { + return pkt, nil + } + if err := gsw.GoSNMP.Connect(); err != nil { + return nil, Errorf(err, "reconnecting") + } + } + return nil, err +} + +// getConnection creates a snmpConnection (*gosnmp.GoSNMP) object and caches the +// result using `agent` as the cache key. +func (s *Snmp) getConnection(agent string) (snmpConnection, error) { + if s.connectionCache == nil { + s.connectionCache = map[string]snmpConnection{} + } + if gs, ok := s.connectionCache[agent]; ok { + return gs, nil + } + + gs := gosnmpWrapper{&gosnmp.GoSNMP{}} + + host, portStr, err := net.SplitHostPort(agent) + if err != nil { + if err, ok := err.(*net.AddrError); !ok || err.Err != "missing port in address" { + return nil, Errorf(err, "parsing host") + } + host = agent + portStr = "161" + } + gs.Target = host + + port, err := strconv.ParseUint(portStr, 10, 16) + if err != nil { + return nil, Errorf(err, "parsing port") + } + gs.Port = uint16(port) + + gs.Timeout = s.Timeout.Duration + + gs.Retries = s.Retries + + switch s.Version { + case 3: + gs.Version = gosnmp.Version3 + case 2, 0: + gs.Version = gosnmp.Version2c + case 1: + gs.Version = gosnmp.Version1 + default: + return nil, fmt.Errorf("invalid version") + } + + if s.Version < 3 { + if s.Community == "" { + gs.Community = "public" + } else { + gs.Community = s.Community + } + } + + gs.MaxRepetitions = int(s.MaxRepetitions) + + if s.Version == 3 { + gs.ContextName = s.ContextName + + sp := &gosnmp.UsmSecurityParameters{} + gs.SecurityParameters = sp + gs.SecurityModel = gosnmp.UserSecurityModel + + switch strings.ToLower(s.SecLevel) { + case "noauthnopriv", "": + gs.MsgFlags = gosnmp.NoAuthNoPriv + case "authnopriv": + gs.MsgFlags = gosnmp.AuthNoPriv + case "authpriv": + gs.MsgFlags = gosnmp.AuthPriv + default: + return nil, fmt.Errorf("invalid secLevel") + } + + sp.UserName = s.SecName + + switch strings.ToLower(s.AuthProtocol) { + case "md5": + sp.AuthenticationProtocol = gosnmp.MD5 + case "sha": + sp.AuthenticationProtocol = gosnmp.SHA + case "": + sp.AuthenticationProtocol = gosnmp.NoAuth + default: + return nil, fmt.Errorf("invalid authProtocol") + } + + sp.AuthenticationPassphrase = s.AuthPassword + + switch strings.ToLower(s.PrivProtocol) { + case "des": + sp.PrivacyProtocol = gosnmp.DES + case "aes": + sp.PrivacyProtocol = gosnmp.AES + case "": + sp.PrivacyProtocol = gosnmp.NoPriv + default: + return nil, fmt.Errorf("invalid privProtocol") + } + + sp.PrivacyPassphrase = s.PrivPassword + + sp.AuthoritativeEngineID = s.EngineID + + sp.AuthoritativeEngineBoots = s.EngineBoots + + sp.AuthoritativeEngineTime = s.EngineTime + } + + if err := gs.Connect(); err != nil { + return nil, Errorf(err, "setting up connection") + } + + s.connectionCache[agent] = gs + return gs, nil +} + +// fieldConvert converts from any type according to the conv specification +// "float"/"float(0)" will convert the value into a float. +// "float(X)" will convert the value into a float, and then move the decimal before Xth right-most digit. +// "int" will convert the value into an integer. +// "" will convert a byte slice into a string. +// Any other conv will return the input value unchanged. +func fieldConvert(conv string, v interface{}) interface{} { + if conv == "" { + if bs, ok := v.([]byte); ok { + return string(bs) + } + return v + } + + var d int + if _, err := fmt.Sscanf(conv, "float(%d)", &d); err == nil || conv == "float" { + switch vt := v.(type) { + case float32: + v = float64(vt) / math.Pow10(d) + case float64: + v = float64(vt) / math.Pow10(d) + case int: + v = float64(vt) / math.Pow10(d) + case int8: + v = float64(vt) / math.Pow10(d) + case int16: + v = float64(vt) / math.Pow10(d) + case int32: + v = float64(vt) / math.Pow10(d) + case int64: + v = float64(vt) / math.Pow10(d) + case uint: + v = float64(vt) / math.Pow10(d) + case uint8: + v = float64(vt) / math.Pow10(d) + case uint16: + v = float64(vt) / math.Pow10(d) + case uint32: + v = float64(vt) / math.Pow10(d) + case uint64: + v = float64(vt) / math.Pow10(d) + case []byte: + vf, _ := strconv.ParseFloat(string(vt), 64) + v = vf / math.Pow10(d) + case string: + vf, _ := strconv.ParseFloat(vt, 64) + v = vf / math.Pow10(d) + } + } + if conv == "int" { + switch vt := v.(type) { + case float32: + v = int64(vt) + case float64: + v = int64(vt) + case int: + v = int64(vt) + case int8: + v = int64(vt) + case int16: + v = int64(vt) + case int32: + v = int64(vt) + case int64: + v = int64(vt) + case uint: + v = int64(vt) + case uint8: + v = int64(vt) + case uint16: + v = int64(vt) + case uint32: + v = int64(vt) + case uint64: + v = int64(vt) + case []byte: + v, _ = strconv.Atoi(string(vt)) + case string: + v, _ = strconv.Atoi(vt) + } + } + + return v +} + +// snmpTranslate resolves the given OID. +// The contents of the oid parameter will be replaced with the numeric oid value. +// If name is empty, the textual OID value is stored in it. If the textual OID cannot be translated, the numeric OID is stored instead. +// If mibPrefix is non-nil, the MIB in which the OID was found is stored, with a suffix of "::". +func snmpTranslate(mibPrefix *string, oid *string, name *string) error { + if strings.ContainsAny(*oid, ":abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ") { + out, err := execCmd("snmptranslate", "-m", "all", "-On", *oid) + if err != nil { + return Errorf(err, "translating %s", *oid) + } + *oid = string(bytes.TrimSuffix(out, []byte{'\n'})) + } + + if *name == "" { + out, err := execCmd("snmptranslate", "-m", "all", *oid) + if err != nil { + //TODO debug message + *name = *oid + } else { + if i := bytes.Index(out, []byte("::")); i != -1 { + if mibPrefix != nil { + *mibPrefix = string(out[:i+2]) + } + out = out[i+2:] + } + *name = string(bytes.TrimSuffix(out, []byte{'\n'})) + } + } + + return nil +} diff --git a/plugins/inputs/snmp/snmp_test.go b/plugins/inputs/snmp/snmp_test.go new file mode 100644 index 000000000..62f3e6c2f --- /dev/null +++ b/plugins/inputs/snmp/snmp_test.go @@ -0,0 +1,641 @@ +package snmp + +import ( + "fmt" + "net" + "os" + "os/exec" + "strings" + "sync" + "testing" + "time" + + "github.com/influxdata/telegraf/internal" + "github.com/influxdata/telegraf/testutil" + "github.com/influxdata/toml" + "github.com/soniah/gosnmp" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func mockExecCommand(arg0 string, args ...string) *exec.Cmd { + args = append([]string{"-test.run=TestMockExecCommand", "--", arg0}, args...) + cmd := exec.Command(os.Args[0], args...) + cmd.Stderr = os.Stderr // so the test output shows errors + return cmd +} +func TestMockExecCommand(t *testing.T) { + var cmd []string + for _, arg := range os.Args { + if string(arg) == "--" { + cmd = []string{} + continue + } + if cmd == nil { + continue + } + cmd = append(cmd, string(arg)) + } + if cmd == nil { + return + } + + // will not properly handle args with spaces, but it's good enough + cmdStr := strings.Join(cmd, " ") + switch cmdStr { + case "snmptranslate -m all .1.0.0.0": + fmt.Printf("TEST::testTable\n") + case "snmptranslate -m all .1.0.0.0.1.1": + fmt.Printf("server\n") + case "snmptranslate -m all .1.0.0.0.1.1.0": + fmt.Printf("server.0\n") + case "snmptranslate -m all .1.0.0.1.1": + fmt.Printf("hostname\n") + case "snmptranslate -m all .999": + fmt.Printf(".999\n") + case "snmptranslate -m all -On TEST::testTable": + fmt.Printf(".1.0.0.0\n") + case "snmptranslate -m all -On TEST::hostname": + fmt.Printf(".1.0.0.1.1\n") + case "snmptranslate -m all -On TEST::server": + fmt.Printf(".1.0.0.0.1.1\n") + case "snmptranslate -m all -On TEST::connections": + fmt.Printf(".1.0.0.0.1.2\n") + case "snmptranslate -m all -On TEST::latency": + fmt.Printf(".1.0.0.0.1.3\n") + case "snmptranslate -m all -On TEST::server.0": + fmt.Printf(".1.0.0.0.1.1.0\n") + case "snmptranslate -m all -Td .1.0.0.0.1": + fmt.Printf(`TEST::testTableEntry +testTableEntry OBJECT-TYPE + -- FROM TEST + MAX-ACCESS not-accessible + STATUS current + INDEX { server } +::= { iso(1) 2 testOID(3) testTable(0) 1 } +`) + case "snmptable -m all -Ch -Cl -c public 127.0.0.1 .1.0.0.0": + fmt.Printf(`server connections latency +TEST::testTable: No entries +`) + default: + fmt.Fprintf(os.Stderr, "Command not mocked: `%s`\n", cmdStr) + // you get the expected output by running the missing command with `-M testdata` in the plugin directory. + os.Exit(1) + } + os.Exit(0) +} +func init() { + execCommand = mockExecCommand +} + +type testSNMPConnection struct { + host string + values map[string]interface{} +} + +func (tsc *testSNMPConnection) Host() string { + return tsc.host +} + +func (tsc *testSNMPConnection) Get(oids []string) (*gosnmp.SnmpPacket, error) { + sp := &gosnmp.SnmpPacket{} + for _, oid := range oids { + v, ok := tsc.values[oid] + if !ok { + sp.Variables = append(sp.Variables, gosnmp.SnmpPDU{ + Name: oid, + Type: gosnmp.NoSuchObject, + }) + continue + } + sp.Variables = append(sp.Variables, gosnmp.SnmpPDU{ + Name: oid, + Value: v, + }) + } + return sp, nil +} +func (tsc *testSNMPConnection) Walk(oid string, wf gosnmp.WalkFunc) error { + for void, v := range tsc.values { + if void == oid || (len(void) > len(oid) && void[:len(oid)+1] == oid+".") { + if err := wf(gosnmp.SnmpPDU{ + Name: void, + Value: v, + }); err != nil { + return err + } + } + } + return nil +} + +var tsc = &testSNMPConnection{ + host: "tsc", + values: map[string]interface{}{ + ".1.0.0.0.1.1.0": "foo", + ".1.0.0.0.1.1.1": []byte("bar"), + ".1.0.0.0.1.102": "bad", + ".1.0.0.0.1.2.0": 1, + ".1.0.0.0.1.2.1": 2, + ".1.0.0.0.1.3.0": "0.123", + ".1.0.0.0.1.3.1": "0.456", + ".1.0.0.0.1.3.2": "9.999", + ".1.0.0.0.1.4.0": 123456, + ".1.0.0.1.1": "baz", + ".1.0.0.1.2": 234, + ".1.0.0.1.3": []byte("byte slice"), + }, +} + +func TestSampleConfig(t *testing.T) { + conf := struct { + Inputs struct { + Snmp []*Snmp + } + }{} + err := toml.Unmarshal([]byte("[[inputs.snmp]]\n"+(*Snmp)(nil).SampleConfig()), &conf) + assert.NoError(t, err) + + s := Snmp{ + Agents: []string{"127.0.0.1:161"}, + Timeout: internal.Duration{Duration: 5 * time.Second}, + Version: 2, + Community: "public", + MaxRepetitions: 50, + + Name: "system", + Fields: []Field{ + {Name: "hostname", Oid: ".1.0.0.1.1"}, + {Name: "uptime", Oid: ".1.0.0.1.2"}, + {Name: "load", Oid: ".1.0.0.1.3"}, + {Oid: "HOST-RESOURCES-MIB::hrMemorySize"}, + }, + Tables: []Table{ + { + Name: "remote_servers", + InheritTags: []string{"hostname"}, + Fields: []Field{ + {Name: "server", Oid: ".1.0.0.0.1.0", IsTag: true}, + {Name: "connections", Oid: ".1.0.0.0.1.1"}, + {Name: "latency", Oid: ".1.0.0.0.1.2"}, + }, + }, + { + Oid: "HOST-RESOURCES-MIB::hrNetworkTable", + }, + }, + } + assert.Equal(t, s, *conf.Inputs.Snmp[0]) +} + +func TestFieldInit(t *testing.T) { + translations := []struct { + inputOid string + inputName string + expectedOid string + expectedName string + }{ + {".1.0.0.0.1.1", "", ".1.0.0.0.1.1", "server"}, + {".1.0.0.0.1.1.0", "", ".1.0.0.0.1.1.0", "server.0"}, + {".999", "", ".999", ".999"}, + {"TEST::server", "", ".1.0.0.0.1.1", "server"}, + {"TEST::server.0", "", ".1.0.0.0.1.1.0", "server.0"}, + {"TEST::server", "foo", ".1.0.0.0.1.1", "foo"}, + } + + for _, txl := range translations { + f := Field{Oid: txl.inputOid, Name: txl.inputName} + err := f.init() + if !assert.NoError(t, err, "inputOid='%s' inputName='%s'", txl.inputOid, txl.inputName) { + continue + } + assert.Equal(t, txl.expectedOid, f.Oid, "inputOid='%s' inputName='%s'", txl.inputOid, txl.inputName) + assert.Equal(t, txl.expectedName, f.Name, "inputOid='%s' inputName='%s'", txl.inputOid, txl.inputName) + } +} + +func TestTableInit(t *testing.T) { + tbl := Table{ + Oid: ".1.0.0.0", + Fields: []Field{{Oid: ".999", Name: "foo"}}, + } + err := tbl.init() + require.NoError(t, err) + + assert.Equal(t, "testTable", tbl.Name) + + assert.Len(t, tbl.Fields, 4) + assert.Contains(t, tbl.Fields, Field{Oid: ".999", Name: "foo", initialized: true}) + assert.Contains(t, tbl.Fields, Field{Oid: ".1.0.0.0.1.1", Name: "server", IsTag: true, initialized: true}) + assert.Contains(t, tbl.Fields, Field{Oid: ".1.0.0.0.1.2", Name: "connections", initialized: true}) + assert.Contains(t, tbl.Fields, Field{Oid: ".1.0.0.0.1.3", Name: "latency", initialized: true}) +} + +func TestSnmpInit(t *testing.T) { + s := &Snmp{ + Tables: []Table{ + {Oid: "TEST::testTable"}, + }, + Fields: []Field{ + {Oid: "TEST::hostname"}, + }, + } + + err := s.init() + require.NoError(t, err) + + assert.Len(t, s.Tables[0].Fields, 3) + assert.Contains(t, s.Tables[0].Fields, Field{Oid: ".1.0.0.0.1.1", Name: "server", IsTag: true, initialized: true}) + assert.Contains(t, s.Tables[0].Fields, Field{Oid: ".1.0.0.0.1.2", Name: "connections", initialized: true}) + assert.Contains(t, s.Tables[0].Fields, Field{Oid: ".1.0.0.0.1.3", Name: "latency", initialized: true}) + + assert.Equal(t, Field{ + Oid: ".1.0.0.1.1", + Name: "hostname", + initialized: true, + }, s.Fields[0]) +} + +func TestGetSNMPConnection_v2(t *testing.T) { + s := &Snmp{ + Timeout: internal.Duration{Duration: 3 * time.Second}, + Retries: 4, + Version: 2, + Community: "foo", + } + + gsc, err := s.getConnection("1.2.3.4:567") + require.NoError(t, err) + gs := gsc.(gosnmpWrapper) + assert.Equal(t, "1.2.3.4", gs.Target) + assert.EqualValues(t, 567, gs.Port) + assert.Equal(t, gosnmp.Version2c, gs.Version) + assert.Equal(t, "foo", gs.Community) + + gsc, err = s.getConnection("1.2.3.4") + require.NoError(t, err) + gs = gsc.(gosnmpWrapper) + assert.Equal(t, "1.2.3.4", gs.Target) + assert.EqualValues(t, 161, gs.Port) +} + +func TestGetSNMPConnection_v3(t *testing.T) { + s := &Snmp{ + Version: 3, + MaxRepetitions: 20, + ContextName: "mycontext", + SecLevel: "authPriv", + SecName: "myuser", + AuthProtocol: "md5", + AuthPassword: "password123", + PrivProtocol: "des", + PrivPassword: "321drowssap", + EngineID: "myengineid", + EngineBoots: 1, + EngineTime: 2, + } + + gsc, err := s.getConnection("1.2.3.4") + require.NoError(t, err) + gs := gsc.(gosnmpWrapper) + assert.Equal(t, gs.Version, gosnmp.Version3) + sp := gs.SecurityParameters.(*gosnmp.UsmSecurityParameters) + assert.Equal(t, "1.2.3.4", gsc.Host()) + assert.Equal(t, 20, gs.MaxRepetitions) + assert.Equal(t, "mycontext", gs.ContextName) + assert.Equal(t, gosnmp.AuthPriv, gs.MsgFlags&gosnmp.AuthPriv) + assert.Equal(t, "myuser", sp.UserName) + assert.Equal(t, gosnmp.MD5, sp.AuthenticationProtocol) + assert.Equal(t, "password123", sp.AuthenticationPassphrase) + assert.Equal(t, gosnmp.DES, sp.PrivacyProtocol) + assert.Equal(t, "321drowssap", sp.PrivacyPassphrase) + assert.Equal(t, "myengineid", sp.AuthoritativeEngineID) + assert.EqualValues(t, 1, sp.AuthoritativeEngineBoots) + assert.EqualValues(t, 2, sp.AuthoritativeEngineTime) +} + +func TestGetSNMPConnection_caching(t *testing.T) { + s := &Snmp{} + gs1, err := s.getConnection("1.2.3.4") + require.NoError(t, err) + gs2, err := s.getConnection("1.2.3.4") + require.NoError(t, err) + gs3, err := s.getConnection("1.2.3.5") + require.NoError(t, err) + assert.True(t, gs1 == gs2) + assert.False(t, gs2 == gs3) +} + +func TestGosnmpWrapper_walk_retry(t *testing.T) { + srvr, err := net.ListenUDP("udp4", &net.UDPAddr{}) + defer srvr.Close() + require.NoError(t, err) + reqCount := 0 + // Set up a WaitGroup to wait for the server goroutine to exit and protect + // reqCount. + // Even though simultaneous access is impossible because the server will be + // blocked on ReadFrom, without this the race detector gets unhappy. + wg := sync.WaitGroup{} + wg.Add(1) + go func() { + defer wg.Done() + buf := make([]byte, 256) + for { + _, addr, err := srvr.ReadFrom(buf) + if err != nil { + return + } + reqCount++ + + srvr.WriteTo([]byte{'X'}, addr) // will cause decoding error + } + }() + + gs := &gosnmp.GoSNMP{ + Target: srvr.LocalAddr().(*net.UDPAddr).IP.String(), + Port: uint16(srvr.LocalAddr().(*net.UDPAddr).Port), + Version: gosnmp.Version2c, + Community: "public", + Timeout: time.Millisecond * 10, + Retries: 1, + } + err = gs.Connect() + require.NoError(t, err) + conn := gs.Conn + + gsw := gosnmpWrapper{gs} + err = gsw.Walk(".1.0.0", func(_ gosnmp.SnmpPDU) error { return nil }) + srvr.Close() + wg.Wait() + assert.Error(t, err) + assert.False(t, gs.Conn == conn) + assert.Equal(t, (gs.Retries+1)*2, reqCount) +} + +func TestGosnmpWrapper_get_retry(t *testing.T) { + srvr, err := net.ListenUDP("udp4", &net.UDPAddr{}) + defer srvr.Close() + require.NoError(t, err) + reqCount := 0 + // Set up a WaitGroup to wait for the server goroutine to exit and protect + // reqCount. + // Even though simultaneous access is impossible because the server will be + // blocked on ReadFrom, without this the race detector gets unhappy. + wg := sync.WaitGroup{} + wg.Add(1) + go func() { + defer wg.Done() + buf := make([]byte, 256) + for { + _, addr, err := srvr.ReadFrom(buf) + if err != nil { + return + } + reqCount++ + + srvr.WriteTo([]byte{'X'}, addr) // will cause decoding error + } + }() + + gs := &gosnmp.GoSNMP{ + Target: srvr.LocalAddr().(*net.UDPAddr).IP.String(), + Port: uint16(srvr.LocalAddr().(*net.UDPAddr).Port), + Version: gosnmp.Version2c, + Community: "public", + Timeout: time.Millisecond * 10, + Retries: 1, + } + err = gs.Connect() + require.NoError(t, err) + conn := gs.Conn + + gsw := gosnmpWrapper{gs} + _, err = gsw.Get([]string{".1.0.0"}) + srvr.Close() + wg.Wait() + assert.Error(t, err) + assert.False(t, gs.Conn == conn) + assert.Equal(t, (gs.Retries+1)*2, reqCount) +} + +func TestTableBuild_walk(t *testing.T) { + tbl := Table{ + Name: "mytable", + Fields: []Field{ + { + Name: "myfield1", + Oid: ".1.0.0.0.1.1", + IsTag: true, + }, + { + Name: "myfield2", + Oid: ".1.0.0.0.1.2", + }, + { + Name: "myfield3", + Oid: ".1.0.0.0.1.3", + Conversion: "float", + }, + }, + } + + tb, err := tbl.Build(tsc, true) + require.NoError(t, err) + + assert.Equal(t, tb.Name, "mytable") + rtr1 := RTableRow{ + Tags: map[string]string{"myfield1": "foo"}, + Fields: map[string]interface{}{"myfield2": 1, "myfield3": float64(0.123)}, + } + rtr2 := RTableRow{ + Tags: map[string]string{"myfield1": "bar"}, + Fields: map[string]interface{}{"myfield2": 2, "myfield3": float64(0.456)}, + } + assert.Len(t, tb.Rows, 2) + assert.Contains(t, tb.Rows, rtr1) + assert.Contains(t, tb.Rows, rtr2) +} + +func TestTableBuild_noWalk(t *testing.T) { + tbl := Table{ + Name: "mytable", + Fields: []Field{ + { + Name: "myfield1", + Oid: ".1.0.0.1.1", + IsTag: true, + }, + { + Name: "myfield2", + Oid: ".1.0.0.1.2", + }, + { + Name: "myfield3", + Oid: ".1.0.0.1.2", + IsTag: true, + }, + }, + } + + tb, err := tbl.Build(tsc, false) + require.NoError(t, err) + + rtr := RTableRow{ + Tags: map[string]string{"myfield1": "baz", "myfield3": "234"}, + Fields: map[string]interface{}{"myfield2": 234}, + } + assert.Len(t, tb.Rows, 1) + assert.Contains(t, tb.Rows, rtr) +} + +func TestGather(t *testing.T) { + s := &Snmp{ + Agents: []string{"TestGather"}, + Name: "mytable", + Fields: []Field{ + { + Name: "myfield1", + Oid: ".1.0.0.1.1", + IsTag: true, + }, + { + Name: "myfield2", + Oid: ".1.0.0.1.2", + }, + { + Name: "myfield3", + Oid: "1.0.0.1.1", + }, + }, + Tables: []Table{ + { + Name: "myOtherTable", + InheritTags: []string{"myfield1"}, + Fields: []Field{ + { + Name: "myOtherField", + Oid: ".1.0.0.0.1.4", + }, + }, + }, + }, + + connectionCache: map[string]snmpConnection{ + "TestGather": tsc, + }, + } + + acc := &testutil.Accumulator{} + + tstart := time.Now() + s.Gather(acc) + tstop := time.Now() + + require.Len(t, acc.Metrics, 2) + + m := acc.Metrics[0] + assert.Equal(t, "mytable", m.Measurement) + assert.Equal(t, "tsc", m.Tags["agent_host"]) + assert.Equal(t, "baz", m.Tags["myfield1"]) + assert.Len(t, m.Fields, 2) + assert.Equal(t, 234, m.Fields["myfield2"]) + assert.Equal(t, "baz", m.Fields["myfield3"]) + assert.True(t, tstart.Before(m.Time)) + assert.True(t, tstop.After(m.Time)) + + m2 := acc.Metrics[1] + assert.Equal(t, "myOtherTable", m2.Measurement) + assert.Equal(t, "tsc", m2.Tags["agent_host"]) + assert.Equal(t, "baz", m2.Tags["myfield1"]) + assert.Len(t, m2.Fields, 1) + assert.Equal(t, 123456, m2.Fields["myOtherField"]) +} + +func TestGather_host(t *testing.T) { + s := &Snmp{ + Agents: []string{"TestGather"}, + Name: "mytable", + Fields: []Field{ + { + Name: "host", + Oid: ".1.0.0.1.1", + IsTag: true, + }, + { + Name: "myfield2", + Oid: ".1.0.0.1.2", + }, + }, + + connectionCache: map[string]snmpConnection{ + "TestGather": tsc, + }, + } + + acc := &testutil.Accumulator{} + + s.Gather(acc) + + require.Len(t, acc.Metrics, 1) + m := acc.Metrics[0] + assert.Equal(t, "baz", m.Tags["host"]) +} + +func TestFieldConvert(t *testing.T) { + testTable := []struct { + input interface{} + conv string + expected interface{} + }{ + {[]byte("foo"), "", string("foo")}, + {"0.123", "float", float64(0.123)}, + {[]byte("0.123"), "float", float64(0.123)}, + {float32(0.123), "float", float64(float32(0.123))}, + {float64(0.123), "float", float64(0.123)}, + {123, "float", float64(123)}, + {123, "float(0)", float64(123)}, + {123, "float(4)", float64(0.0123)}, + {int8(123), "float(3)", float64(0.123)}, + {int16(123), "float(3)", float64(0.123)}, + {int32(123), "float(3)", float64(0.123)}, + {int64(123), "float(3)", float64(0.123)}, + {uint(123), "float(3)", float64(0.123)}, + {uint8(123), "float(3)", float64(0.123)}, + {uint16(123), "float(3)", float64(0.123)}, + {uint32(123), "float(3)", float64(0.123)}, + {uint64(123), "float(3)", float64(0.123)}, + {"123", "int", int64(123)}, + {[]byte("123"), "int", int64(123)}, + {float32(12.3), "int", int64(12)}, + {float64(12.3), "int", int64(12)}, + {int(123), "int", int64(123)}, + {int8(123), "int", int64(123)}, + {int16(123), "int", int64(123)}, + {int32(123), "int", int64(123)}, + {int64(123), "int", int64(123)}, + {uint(123), "int", int64(123)}, + {uint8(123), "int", int64(123)}, + {uint16(123), "int", int64(123)}, + {uint32(123), "int", int64(123)}, + {uint64(123), "int", int64(123)}, + } + + for _, tc := range testTable { + act := fieldConvert(tc.conv, tc.input) + assert.EqualValues(t, tc.expected, act, "input=%T(%v) conv=%s expected=%T(%v)", tc.input, tc.input, tc.conv, tc.expected, tc.expected) + } +} + +func TestError(t *testing.T) { + e := fmt.Errorf("nested error") + err := Errorf(e, "top error %d", 123) + require.Error(t, err) + + ne, ok := err.(NestedError) + require.True(t, ok) + assert.Equal(t, e, ne.NestedErr) + + assert.Contains(t, err.Error(), "top error 123") + assert.Contains(t, err.Error(), "nested error") +} diff --git a/plugins/inputs/snmp/testdata/snmpd.conf b/plugins/inputs/snmp/testdata/snmpd.conf new file mode 100644 index 000000000..3f3151a65 --- /dev/null +++ b/plugins/inputs/snmp/testdata/snmpd.conf @@ -0,0 +1,17 @@ +# This config provides the data represented in the plugin documentation +# Requires net-snmp >= 5.7 + +#agentaddress UDP:127.0.0.1:1161 +rocommunity public + +override .1.0.0.0.1.1.0 octet_str "foo" +override .1.0.0.0.1.1.1 octet_str "bar" +override .1.0.0.0.1.102 octet_str "bad" +override .1.0.0.0.1.2.0 integer 1 +override .1.0.0.0.1.2.1 integer 2 +override .1.0.0.0.1.3.0 octet_str "0.123" +override .1.0.0.0.1.3.1 octet_str "0.456" +override .1.0.0.0.1.3.2 octet_str "9.999" +override .1.0.0.1.1 octet_str "baz" +override .1.0.0.1.2 uinteger 54321 +override .1.0.0.1.3 uinteger 234 diff --git a/plugins/inputs/snmp/testdata/test.mib b/plugins/inputs/snmp/testdata/test.mib new file mode 100644 index 000000000..d3246673b --- /dev/null +++ b/plugins/inputs/snmp/testdata/test.mib @@ -0,0 +1,51 @@ +TEST DEFINITIONS ::= BEGIN + +testOID ::= { 1 0 0 } + +testTable OBJECT-TYPE + SYNTAX SEQUENCE OF testTableEntry + MAX-ACCESS not-accessible + STATUS current + ::= { testOID 0 } + +testTableEntry OBJECT-TYPE + SYNTAX TestTableEntry + MAX-ACCESS not-accessible + STATUS current + INDEX { + server + } + ::= { testTable 1 } + +TestTableEntry ::= + SEQUENCE { + server OCTET STRING, + connections INTEGER, + latency OCTET STRING, + } + +server OBJECT-TYPE + SYNTAX OCTET STRING + MAX-ACCESS read-only + STATUS current + ::= { testTableEntry 1 } + +connections OBJECT-TYPE + SYNTAX INTEGER + MAX-ACCESS read-only + STATUS current + ::= { testTableEntry 2 } + +latency OBJECT-TYPE + SYNTAX OCTET STRING + MAX-ACCESS read-only + STATUS current + ::= { testTableEntry 3 } + +hostname OBJECT-TYPE + SYNTAX OCTET STRING + MAX-ACCESS read-only + STATUS current + ::= { testOID 1 1 } + +END