Update etc/telegraf.conf for 1.0

Added quotes around spaces in files paths.
Slight change to wording on point 4.

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,5 +1,5 @@
 ### Required for all PRs:
 
-- [ ] CHANGELOG.md updated
+- [ ] CHANGELOG.md updated (we recommend not updating this until the PR has been approved by a maintainer)
 - [ ] Sign [CLA](https://influxdata.com/community/cla/) (if not already signed)
 - [ ] README.md updated (if adding a new plugin)

.gitignore

@@ -1,3 +1,4 @@
+build
 tivan
 .vagrant
 /telegraf

CHANGELOG.md

@@ -1,56 +1,81 @@
-## v1.0
-
-### Features
-
-### Bugfixes
-
-## v1.0 beta 2 [2016-06-21]
-
-### Features
-
-- [#1340](https://github.com/influxdata/telegraf/issues/1340): statsd: do not log every dropped metric.
-- [#1368](https://github.com/influxdata/telegraf/pull/1368): Add precision rounding to all metrics on collection.
-- [#1390](https://github.com/influxdata/telegraf/pull/1390): Add support for Tengine
-- [#1320](https://github.com/influxdata/telegraf/pull/1320): Logparser input plugin for parsing grok-style log patterns.
-
-### Bugfixes
-
-- [#1330](https://github.com/influxdata/telegraf/issues/1330): Fix exec plugin panic when using single binary.
-- [#1336](https://github.com/influxdata/telegraf/issues/1336): Fixed incorrect prometheus metrics source selection.
-- [#1112](https://github.com/influxdata/telegraf/issues/1112): Set default Zookeeper chroot to empty string.
-- [#1335](https://github.com/influxdata/telegraf/issues/1335): Fix overall ping timeout to be calculated based on per-ping timeout.
-- [#1374](https://github.com/influxdata/telegraf/pull/1374): Change "default" retention policy to "".
-- [#1377](https://github.com/influxdata/telegraf/issues/1377): Graphite output mangling '%' character.
-
-## v1.0 beta 1 [2016-06-07]
+## v1.0 Release Candidate 1 [2016-08-23]
 
 ### Release Notes
 
+**Breaking Change** The SNMP plugin is being deprecated in it's current form.
+There is a [new SNMP plugin](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/snmp)
+which fixes many of the issues and confusions
+of it's predecessor. For users wanting to continue to use the deprecated SNMP
+plugin, you will need to change your config file from `[[inputs.snmp]]` to
+`[[inputs.snmp_legacy]]`. The configuration of the new SNMP plugin is _not_
+backwards-compatible.
+
+- Telegraf now supports being installed as an official windows service,
+which can be installed via
+`> C:\Program Files\Telegraf\telegraf.exe --service install`
+
+**Breaking Change**: Aerospike main server node measurements have been renamed
+aerospike_node. Aerospike namespace measurements have been renamed to
+aerospike_namespace. They will also now be tagged with the node_name
+that they correspond to. This has been done to differentiate measurements
+that pertain to node vs. namespace statistics.
+
+**Breaking Change**: users of github_webhooks must change to the new
+`[[inputs.webhooks]]` plugin.
+
+This means that the default github_webhooks config:
+
+```
+# A Github Webhook Event collector
+[[inputs.github_webhooks]]
+  ## Address and port to host Webhook listener on
+  service_address = ":1618"
+```
+
+should now look like:
+
+```
+# A Webhooks Event collector
+[[inputs.webhooks]]
+  ## Address and port to host Webhook listener on
+  service_address = ":1618"
+
+  [inputs.webhooks.github]
+    path = "/"
+```
+
 - `flush_jitter` behavior has been changed. The random jitter will now be
 evaluated at every flush interval, rather than once at startup. This makes it
 consistent with the behavior of `collection_jitter`.
 
-- All AWS plugins now utilize a standard mechanism for evaluating credentials.
-This allows all AWS plugins to support environment variables, shared credential
-files & profiles, and role assumptions. See the specific plugin README for
-details.
-
-- The AWS CloudWatch input plugin can now declare a wildcard value for a metric
-dimension. This causes the plugin to read all metrics that contain the specified
-dimension key regardless of value. This is used to export collections of metrics
-without having to know the dimension values ahead of time.
-
-- The AWS CloudWatch input plugin can now be configured with the `cache_ttl`
-attribute. This configures the TTL of the internal metric cache. This is useful
-in conjunction with wildcard dimension values as it will control the amount of
-time before a new metric is included by the plugin.
-
 ### Features
+
+- [#1413](https://github.com/influxdata/telegraf/issues/1413): Separate container_version from container_image tag.
+- [#1525](https://github.com/influxdata/telegraf/pull/1525): Support setting per-device and total metrics for Docker network and blockio.
+- [#1466](https://github.com/influxdata/telegraf/pull/1466): MongoDB input plugin: adding per DB stats from db.stats()
+- [#1503](https://github.com/influxdata/telegraf/pull/1503): Add tls support for certs to RabbitMQ input plugin
+- [#1289](https://github.com/influxdata/telegraf/pull/1289): webhooks input plugin. Thanks @francois2metz and @cduez!
+- [#1247](https://github.com/influxdata/telegraf/pull/1247): rollbar webhook plugin.
+- [#1408](https://github.com/influxdata/telegraf/pull/1408): mandrill webhook plugin.
+- [#1402](https://github.com/influxdata/telegraf/pull/1402): docker-machine/boot2docker no longer required for unit tests.
+- [#1350](https://github.com/influxdata/telegraf/pull/1350): cgroup input plugin.
+- [#1369](https://github.com/influxdata/telegraf/pull/1369): Add input plugin for consuming metrics from NSQD.
+- [#1369](https://github.com/influxdata/telegraf/pull/1480): add ability to read redis from a socket.
+- [#1387](https://github.com/influxdata/telegraf/pull/1387): **Breaking Change** - Redis `role` tag renamed to `replication_role` to avoid global_tags override
+- [#1437](https://github.com/influxdata/telegraf/pull/1437): Fetching Galera status metrics in MySQL
+- [#1500](https://github.com/influxdata/telegraf/pull/1500): Aerospike plugin refactored to use official client lib.
+- [#1434](https://github.com/influxdata/telegraf/pull/1434): Add measurement name arg to logparser plugin.
+- [#1479](https://github.com/influxdata/telegraf/pull/1479): logparser: change resp_code from a field to a tag.
+- [#1411](https://github.com/influxdata/telegraf/pull/1411): Implement support for fetching hddtemp data
+- [#1340](https://github.com/influxdata/telegraf/issues/1340): statsd: do not log every dropped metric.
+- [#1368](https://github.com/influxdata/telegraf/pull/1368): Add precision rounding to all metrics on collection.
+- [#1390](https://github.com/influxdata/telegraf/pull/1390): Add support for Tengine
+- [#1320](https://github.com/influxdata/telegraf/pull/1320): Logparser input plugin for parsing grok-style log patterns.
+- [#1397](https://github.com/influxdata/telegraf/issues/1397): ElasticSearch: now supports connecting to ElasticSearch via SSL
 - [#1262](https://github.com/influxdata/telegraf/pull/1261): Add graylog input pluging.
 - [#1294](https://github.com/influxdata/telegraf/pull/1294): consul input plugin. Thanks @harnash
 - [#1164](https://github.com/influxdata/telegraf/pull/1164): conntrack input plugin. Thanks @robinpercy!
 - [#1165](https://github.com/influxdata/telegraf/pull/1165): vmstat input plugin. Thanks @jshim-xm!
-- [#1247](https://github.com/influxdata/telegraf/pull/1247): rollbar input plugin. Thanks @francois2metz and @cduez!
 - [#1208](https://github.com/influxdata/telegraf/pull/1208): Standardized AWS credentials evaluation & wildcard CloudWatch dimensions. Thanks @johnrengelman!
 - [#1264](https://github.com/influxdata/telegraf/pull/1264): Add SSL config options to http_response plugin.
 - [#1272](https://github.com/influxdata/telegraf/pull/1272): graphite parser: add ability to specify multiple tag keys, for consistency with influxdb parser.
@@ -62,9 +87,45 @@ time before a new metric is included by the plugin.
 - [#1278](https://github.com/influxdata/telegraf/pull/1278) & [#1288](https://github.com/influxdata/telegraf/pull/1288) & [#1295](https://github.com/influxdata/telegraf/pull/1295): RabbitMQ/Apache/InfluxDB inputs: made url(s) parameter optional by using reasonable input defaults if not specified
 - [#1296](https://github.com/influxdata/telegraf/issues/1296): Refactor of flush_jitter argument.
 - [#1213](https://github.com/influxdata/telegraf/issues/1213): Add inactive & active memory to mem plugin.
+- [#1543](https://github.com/influxdata/telegraf/pull/1543): Official Windows service.
+- [#1414](https://github.com/influxdata/telegraf/pull/1414): Forking sensors command to remove C package dependency.
+- [#1389](https://github.com/influxdata/telegraf/pull/1389): Add a new SNMP plugin.
 
 ### Bugfixes
 
+- [#1619](https://github.com/influxdata/telegraf/issues/1619): Fix `make windows` build target
+- [#1519](https://github.com/influxdata/telegraf/pull/1519): Fix error race conditions and partial failures.
+- [#1477](https://github.com/influxdata/telegraf/issues/1477): nstat: fix inaccurate config panic.
+- [#1481](https://github.com/influxdata/telegraf/issues/1481): jolokia: fix handling multiple multi-dimensional attributes.
+- [#1430](https://github.com/influxdata/telegraf/issues/1430): Fix prometheus character sanitizing. Sanitize more win_perf_counters characters.
+- [#1534](https://github.com/influxdata/telegraf/pull/1534): Add diskio io_time to FreeBSD & report timing metrics as ms (as linux does).
+- [#1379](https://github.com/influxdata/telegraf/issues/1379): Fix covering Amazon Linux for post remove flow.
+- [#1584](https://github.com/influxdata/telegraf/issues/1584): procstat missing fields: read/write bytes & count
+- [#1472](https://github.com/influxdata/telegraf/pull/1472): diskio input plugin: set 'skip_serial_number = true' by default to avoid high cardinality.
+- [#1426](https://github.com/influxdata/telegraf/pull/1426): nil metrics panic fix.
+- [#1384](https://github.com/influxdata/telegraf/pull/1384): Fix datarace in apache input plugin.
+- [#1399](https://github.com/influxdata/telegraf/issues/1399): Add `read_repairs` statistics to riak plugin.
+- [#1405](https://github.com/influxdata/telegraf/issues/1405): Fix memory/connection leak in prometheus input plugin.
+- [#1378](https://github.com/influxdata/telegraf/issues/1378): Trim BOM from config file for Windows support.
+- [#1339](https://github.com/influxdata/telegraf/issues/1339): Prometheus client output panic on service reload.
+- [#1461](https://github.com/influxdata/telegraf/pull/1461): Prometheus parser, protobuf format header fix.
+- [#1334](https://github.com/influxdata/telegraf/issues/1334): Prometheus output, metric refresh and caching fixes.
+- [#1432](https://github.com/influxdata/telegraf/issues/1432): Panic fix for multiple graphite outputs under very high load.
+- [#1412](https://github.com/influxdata/telegraf/pull/1412): Instrumental output has better reconnect behavior
+- [#1460](https://github.com/influxdata/telegraf/issues/1460): Remove PID from procstat plugin to fix cardinality issues.
+- [#1427](https://github.com/influxdata/telegraf/issues/1427): Cassandra input: version 2.x "column family" fix.
+- [#1463](https://github.com/influxdata/telegraf/issues/1463): Shared WaitGroup in Exec plugin
+- [#1436](https://github.com/influxdata/telegraf/issues/1436): logparser: honor modifiers in "pattern" config.
+- [#1418](https://github.com/influxdata/telegraf/issues/1418): logparser: error and exit on file permissions/missing errors.
+- [#1499](https://github.com/influxdata/telegraf/pull/1499): Make the user able to specify full path for HAproxy stats
+- [#1521](https://github.com/influxdata/telegraf/pull/1521): Fix Redis url, an extra "tcp://" was added.
+- [#1330](https://github.com/influxdata/telegraf/issues/1330): Fix exec plugin panic when using single binary.
+- [#1336](https://github.com/influxdata/telegraf/issues/1336): Fixed incorrect prometheus metrics source selection.
+- [#1112](https://github.com/influxdata/telegraf/issues/1112): Set default Zookeeper chroot to empty string.
+- [#1335](https://github.com/influxdata/telegraf/issues/1335): Fix overall ping timeout to be calculated based on per-ping timeout.
+- [#1374](https://github.com/influxdata/telegraf/pull/1374): Change "default" retention policy to "".
+- [#1377](https://github.com/influxdata/telegraf/issues/1377): Graphite output mangling '%' character.
+- [#1396](https://github.com/influxdata/telegraf/pull/1396): Prometheus input plugin now supports x509 certs authentication
 - [#1252](https://github.com/influxdata/telegraf/pull/1252) & [#1279](https://github.com/influxdata/telegraf/pull/1279): Fix systemd service. Thanks @zbindenren & @PierreF!
 - [#1221](https://github.com/influxdata/telegraf/pull/1221): Fix influxdb n_shards counter.
 - [#1258](https://github.com/influxdata/telegraf/pull/1258): Fix potential kernel plugin integer parse error.
@@ -74,6 +135,11 @@ time before a new metric is included by the plugin.
 - [#1316](https://github.com/influxdata/telegraf/pull/1316): Removed leaked "database" tag on redis metrics. Thanks @PierreF!
 - [#1323](https://github.com/influxdata/telegraf/issues/1323): Processes plugin: fix potential error with /proc/net/stat directory.
 - [#1322](https://github.com/influxdata/telegraf/issues/1322): Fix rare RHEL 5.2 panic in gopsutil diskio gathering function.
+- [#1586](https://github.com/influxdata/telegraf/pull/1586): Remove IF NOT EXISTS from influxdb output database creation.
+- [#1600](https://github.com/influxdata/telegraf/issues/1600): Fix quoting with text values in postgresql_extensible plugin.
+- [#1425](https://github.com/influxdata/telegraf/issues/1425): Fix win_perf_counter "index out of range" panic.
+- [#1634](https://github.com/influxdata/telegraf/issues/1634): Fix ntpq panic when field is missing.
+- [#1637](https://github.com/influxdata/telegraf/issues/1637): Sanitize graphite output field names.
 
 ## v0.13.1 [2016-05-24]
 

CONTRIBUTING.md

@@ -11,6 +11,8 @@ Output plugins READMEs are less structured,
 but any information you can provide on how the data will look is appreciated.
 See the [OpenTSDB output](https://github.com/influxdata/telegraf/tree/master/plugins/outputs/opentsdb)
 for a good example.
+1. **Optional:** Help users of your plugin by including example queries for populating dashboards. Include these sample queries in the `README.md` for the plugin.
+1. **Optional:** Write a [tickscript](https://docs.influxdata.com/kapacitor/v1.0/tick/syntax/) for your plugin and add it to [Kapacitor](https://github.com/influxdata/kapacitor/tree/master/examples/telegraf). Or mention @jackzampolin in a PR comment with some common queries that you would want to alert on and he will write one for you.
 
 ## GoDoc
 
@@ -114,7 +116,7 @@ creating the `Parser` object.
 You should also add the following to your SampleConfig() return:
 
 ```toml
-  ## Data format to consume. 
+  ## Data format to consume.
   ## Each data format has it's own unique set of configuration options, read
   ## more about them here:
   ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md
@@ -244,7 +246,7 @@ instantiating and creating the `Serializer` object.
 You should also add the following to your SampleConfig() return:
 
 ```toml
-  ## Data format to output. 
+  ## Data format to output.
   ## Each data format has it's own unique set of configuration options, read
   ## more about them here:
   ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_OUTPUT.md
@@ -290,10 +292,6 @@ To execute Telegraf tests follow these simple steps:
 instructions
 - execute `make test`
 
-**OSX users**: you will need to install `boot2docker` or `docker-machine`.
-The Makefile will assume that you have a `docker-machine` box called `default` to
-get the IP address.
-
 ### Unit test troubleshooting
 
 Try cleaning up your test environment by executing `make docker-kill` and

Godeps

@@ -1,5 +1,6 @@
 github.com/Shopify/sarama 8aadb476e66ca998f2f6bb3c993e9a2daa3666b9
 github.com/Sirupsen/logrus 219c8cb75c258c552e999735be6df753ffc7afdc
+github.com/aerospike/aerospike-client-go 45863b7fd8640dc12f7fdd397104d97e1986f25a
 github.com/amir/raidman 53c1b967405155bfc8758557863bf2e14f814687
 github.com/aws/aws-sdk-go 13a12060f716145019378a10e2806c174356b857
 github.com/beorn7/perks 3ac7bf7a47d159a033b107610db8a1b6575507a4
@@ -28,6 +29,8 @@ github.com/hpcloud/tail b2940955ab8b26e19d43a43c4da0475dd81bdb56
 github.com/influxdata/config b79f6829346b8d6e78ba73544b1e1038f1f1c9da
 github.com/influxdata/influxdb e094138084855d444195b252314dfee9eae34cab
 github.com/influxdata/toml af4df43894b16e3fd2b788d01bd27ad0776ef2d0
+github.com/kardianos/osext 29ae4ffbc9a6fe9fb2bc5029050ce6996ea1d3bc
+github.com/kardianos/service 5e335590050d6d00f3aa270217d288dda1c94d0a
 github.com/klauspost/crc32 19b0b332c9e4516a6370a0456e6182c3b5036720
 github.com/lib/pq e182dc4027e2ded4b19396d638610f2653295f36
 github.com/matttproud/golang_protobuf_extensions d0c3fe89de86839aecf2e0579c40ba3bb336a453
@@ -43,13 +46,15 @@ github.com/prometheus/client_model fa8ad6fec33561be4280a8f0514318c79d7f6cb6
 github.com/prometheus/common e8eabff8812b05acf522b45fdcd725a785188e37
 github.com/prometheus/procfs 406e5b7bfd8201a36e2bb5f7bdae0b03380c2ce8
 github.com/samuel/go-zookeeper 218e9c81c0dd8b3b18172b2bbfad92cc7d6db55f
-github.com/shirou/gopsutil 586bb697f3ec9f8ec08ffefe18f521a64534037c
-github.com/soniah/gosnmp b1b4f885b12c5dcbd021c5cee1c904110de6db7d
+github.com/shirou/gopsutil 4d0c402af66c78735c5ccf820dc2ca7de5e4ff08
+github.com/soniah/gosnmp eb32571c2410868d85849ad67d1e51d01273eb84
+github.com/sparrc/aerospike-client-go d4bb42d2c2d39dae68e054116f4538af189e05d5
 github.com/streadway/amqp b4f3ceab0337f013208d31348b578d83c0064744
 github.com/stretchr/testify 1f4a1643a57e798696635ea4c126e9127adb7d3c
 github.com/vjeantet/grok 83bfdfdfd1a8146795b28e547a8e3c8b28a466c2
 github.com/wvanbergen/kafka 46f9a1cf3f670edec492029fadded9c2d9e18866
 github.com/wvanbergen/kazoo-go 0f768712ae6f76454f987c3356177e138df258f8
+github.com/yuin/gopher-lua bf3808abd44b1e55143a2d7f08571aaa80db1808
 github.com/zensqlmonitor/go-mssqldb ffe5510c6fa5e15e6d983210ab501c815b56b363
 golang.org/x/crypto 5dc8cb4b8a8eb076cbb5a06bc3b8682c15bdbbd3
 golang.org/x/net 6acef71eb69611914f7a30939ea9f6e194c78172

Godeps_windows

@@ -1,59 +1,12 @@
-github.com/Microsoft/go-winio 9f57cbbcbcb41dea496528872a4f0e37a4f7ae98
-github.com/Shopify/sarama 8aadb476e66ca998f2f6bb3c993e9a2daa3666b9
-github.com/Sirupsen/logrus 219c8cb75c258c552e999735be6df753ffc7afdc
+github.com/Microsoft/go-winio ce2922f643c8fd76b46cadc7f404a06282678b34
 github.com/StackExchange/wmi f3e2bae1e0cb5aef83e319133eabfee30013a4a5
-github.com/amir/raidman 53c1b967405155bfc8758557863bf2e14f814687
-github.com/aws/aws-sdk-go 13a12060f716145019378a10e2806c174356b857
-github.com/beorn7/perks 3ac7bf7a47d159a033b107610db8a1b6575507a4
-github.com/cenkalti/backoff 4dc77674aceaabba2c7e3da25d4c823edfb73f99
-github.com/couchbase/go-couchbase cb664315a324d87d19c879d9cc67fda6be8c2ac1
-github.com/couchbase/gomemcached a5ea6356f648fec6ab89add00edd09151455b4b2
-github.com/couchbase/goutils 5823a0cbaaa9008406021dc5daf80125ea30bba6
-github.com/dancannon/gorethink e7cac92ea2bc52638791a021f212145acfedb1fc
-github.com/davecgh/go-spew 5215b55f46b2b919f50a1df0eaa5886afe4e3b3d
-github.com/docker/engine-api 8924d6900370b4c7e7984be5adc61f50a80d7537
-github.com/docker/go-connections f549a9393d05688dff0992ef3efd8bbe6c628aeb
-github.com/docker/go-units 5d2041e26a699eaca682e2ea41c8f891e1060444
-github.com/eapache/go-resiliency b86b1ec0dd4209a588dc1285cdd471e73525c0b3
-github.com/eapache/queue ded5959c0d4e360646dc9e9908cff48666781367
-github.com/eclipse/paho.mqtt.golang 0f7a459f04f13a41b7ed752d47944528d4bf9a86
-github.com/go-ole/go-ole 50055884d646dd9434f16bbb5c9801749b9bafe4
-github.com/go-sql-driver/mysql 1fca743146605a172a266e1654e01e5cd5669bee
-github.com/golang/protobuf 552c7b9542c194800fd493123b3798ef0a832032
-github.com/golang/snappy 427fb6fc07997f43afa32f35e850833760e489a7
-github.com/gonuts/go-shellquote e842a11b24c6abfb3dd27af69a17f482e4b483c2
-github.com/gorilla/context 1ea25387ff6f684839d82767c1733ff4d4d15d0a
-github.com/gorilla/mux c9e326e2bdec29039a3761c07bece13133863e1e
-github.com/hailocab/go-hostpool e80d13ce29ede4452c43dea11e79b9bc8a15b478
-github.com/influxdata/config b79f6829346b8d6e78ba73544b1e1038f1f1c9da
-github.com/influxdata/influxdb e3fef5593c21644f2b43af55d6e17e70910b0e48
-github.com/influxdata/toml af4df43894b16e3fd2b788d01bd27ad0776ef2d0
-github.com/klauspost/crc32 19b0b332c9e4516a6370a0456e6182c3b5036720
-github.com/lib/pq e182dc4027e2ded4b19396d638610f2653295f36
-github.com/lxn/win 9a7734ea4db26bc593d52f6a8a957afdad39c5c1
-github.com/matttproud/golang_protobuf_extensions d0c3fe89de86839aecf2e0579c40ba3bb336a453
-github.com/miekg/dns cce6c130cdb92c752850880fd285bea1d64439dd
-github.com/mreiferson/go-snappystream 028eae7ab5c4c9e2d1cb4c4ca1e53259bbe7e504
-github.com/naoina/go-stringutil 6b638e95a32d0c1131db0e7fe83775cbea4a0d0b
-github.com/nats-io/nats b13fc9d12b0b123ebc374e6b808c6228ae4234a3
-github.com/nats-io/nuid 4f84f5f3b2786224e336af2e13dba0a0a80b76fa
-github.com/nsqio/go-nsq 0b80d6f05e15ca1930e0c5e1d540ed627e299980
-github.com/prometheus/client_golang 18acf9993a863f4c4b40612e19cdd243e7c86831
-github.com/prometheus/client_model fa8ad6fec33561be4280a8f0514318c79d7f6cb6
-github.com/prometheus/common e8eabff8812b05acf522b45fdcd725a785188e37
-github.com/prometheus/procfs 406e5b7bfd8201a36e2bb5f7bdae0b03380c2ce8
-github.com/samuel/go-zookeeper 218e9c81c0dd8b3b18172b2bbfad92cc7d6db55f
-github.com/shirou/gopsutil 1f32ce1bb380845be7f5d174ac641a2c592c0c42
-github.com/shirou/w32 ada3ba68f000aa1b58580e45c9d308fe0b7fc5c5
-github.com/soniah/gosnmp b1b4f885b12c5dcbd021c5cee1c904110de6db7d
-github.com/streadway/amqp b4f3ceab0337f013208d31348b578d83c0064744
-github.com/stretchr/testify 1f4a1643a57e798696635ea4c126e9127adb7d3c
-github.com/wvanbergen/kafka 46f9a1cf3f670edec492029fadded9c2d9e18866
-github.com/wvanbergen/kazoo-go 0f768712ae6f76454f987c3356177e138df258f8
-github.com/zensqlmonitor/go-mssqldb ffe5510c6fa5e15e6d983210ab501c815b56b363
-golang.org/x/net 6acef71eb69611914f7a30939ea9f6e194c78172
-golang.org/x/text a71fd10341b064c10f4a81ceac72bcf70f26ea34
-gopkg.in/dancannon/gorethink.v1 7d1af5be49cb5ecc7b177bf387d232050299d6ef
-gopkg.in/fatih/pool.v2 cba550ebf9bce999a02e963296d4bc7a486cb715
-gopkg.in/mgo.v2 d90005c5262a3463800497ea5a89aed5fe22c886
-gopkg.in/yaml.v2 a83829b6f1293c91addabc89d0571c246397bbf4
+github.com/go-ole/go-ole be49f7c07711fcb603cff39e1de7c67926dc0ba7
+github.com/lxn/win 950a0e81e7678e63d8e6cd32412bdecb325ccd88
+github.com/shirou/w32 3c9377fc6748f222729a8270fe2775d149a249ad
+golang.org/x/sys a646d33e2ee3172a661fc09bca23bb4889a41bc8
+github.com/go-ini/ini 9144852efba7c4daf409943ee90767da62d55438
+github.com/jmespath/go-jmespath bd40a432e4c76585ef6b72d3fd96fb9b6dc7b68d
+github.com/pmezard/go-difflib/difflib 792786c7400a136282c1664665ae0a8db921c6c2
+github.com/stretchr/objx 1a9d0bb9f541897e62256577b352fdbc1fb4fd94
+gopkg.in/fsnotify.v1 a8a77c9133d2d6fd8334f3260d06f60e8d80a5fb
+gopkg.in/tomb.v1 dd632973f1e7218eb1089048e0798ec9ae7dceb8

Makefile

@@ -1,4 +1,3 @@
-UNAME := $(shell sh -c 'uname')
 VERSION := $(shell sh -c 'git describe --always --tags')
 ifdef GOBIN
 PATH := $(GOBIN):$(PATH)
@@ -17,7 +16,7 @@ build:
 	go install -ldflags "-X main.version=$(VERSION)" ./...
 
 build-windows:
-	go build -o telegraf.exe -ldflags \
+	GOOS=windows GOARCH=amd64 go build -o telegraf.exe -ldflags \
 		"-X main.version=$(VERSION)" \
 		./cmd/telegraf/telegraf.go
 
@@ -26,10 +25,6 @@ build-for-docker:
 					"-s -X main.version=$(VERSION)" \
 					./cmd/telegraf/telegraf.go
 
-# Build with race detector
-dev: prepare
-	go build -race -ldflags "-X main.version=$(VERSION)" ./...
-
 # run package script
 package:
 	./scripts/build.py --package --version="$(VERSION)" --platform=linux --arch=all --upload
@@ -42,30 +37,22 @@ prepare:
 # Use the windows godeps file to prepare dependencies
 prepare-windows:
 	go get github.com/sparrc/gdm
+	gdm restore
 	gdm restore -f Godeps_windows
 
 # Run all docker containers necessary for unit tests
 docker-run:
-ifeq ($(UNAME), Darwin)
-	docker run --name kafka \
-		-e ADVERTISED_HOST=$(shell sh -c 'boot2docker ip || docker-machine ip default') \
-		-e ADVERTISED_PORT=9092 \
-		-p "2181:2181" -p "9092:9092" \
-		-d spotify/kafka
-endif
-ifeq ($(UNAME), Linux)
 	docker run --name kafka \
 		-e ADVERTISED_HOST=localhost \
 		-e ADVERTISED_PORT=9092 \
 		-p "2181:2181" -p "9092:9092" \
 		-d spotify/kafka
-endif
 	docker run --name mysql -p "3306:3306" -e MYSQL_ALLOW_EMPTY_PASSWORD=yes -d mysql
 	docker run --name memcached -p "11211:11211" -d memcached
 	docker run --name postgres -p "5432:5432" -d postgres
 	docker run --name rabbitmq -p "15672:15672" -p "5672:5672" -d rabbitmq:3-management
 	docker run --name redis -p "6379:6379" -d redis
-	docker run --name aerospike -p "3000:3000" -d aerospike
+	docker run --name aerospike -p "3000:3000" -d aerospike/aerospike-server
 	docker run --name nsq -p "4150:4150" -d nsqio/nsq /nsqd
 	docker run --name mqtt -p "1883:1883" -d ncarlier/mqtt
 	docker run --name riemann -p "5555:5555" -d blalor/riemann
@@ -78,7 +65,7 @@ docker-run-circle:
 		-e ADVERTISED_PORT=9092 \
 		-p "2181:2181" -p "9092:9092" \
 		-d spotify/kafka
-	docker run --name aerospike -p "3000:3000" -d aerospike
+	docker run --name aerospike -p "3000:3000" -d aerospike/aerospike-server
 	docker run --name nsq -p "4150:4150" -d nsqio/nsq /nsqd
 	docker run --name mqtt -p "1883:1883" -d ncarlier/mqtt
 	docker run --name riemann -p "5555:5555" -d blalor/riemann

README.md

@@ -20,12 +20,12 @@ new plugins.
 ### Linux deb and rpm Packages:
 
 Latest:
-* https://dl.influxdata.com/telegraf/releases/telegraf_1.0.0-beta2_amd64.deb
-* https://dl.influxdata.com/telegraf/releases/telegraf-1.0.0_beta2.x86_64.rpm
+* https://dl.influxdata.com/telegraf/releases/telegraf_1.0.0-rc1_amd64.deb
+* https://dl.influxdata.com/telegraf/releases/telegraf-1.0.0_rc1.x86_64.rpm
 
 Latest (arm):
-* https://dl.influxdata.com/telegraf/releases/telegraf_1.0.0-beta2_armhf.deb
-* https://dl.influxdata.com/telegraf/releases/telegraf-1.0.0_beta2.armhf.rpm
+* https://dl.influxdata.com/telegraf/releases/telegraf_1.0.0-rc1_armhf.deb
+* https://dl.influxdata.com/telegraf/releases/telegraf-1.0.0_rc1.armhf.rpm
 
 ##### Package Instructions:
 
@@ -46,14 +46,14 @@ to use this repo to install & update telegraf.
 ### Linux tarballs:
 
 Latest:
-* https://dl.influxdata.com/telegraf/releases/telegraf-1.0.0-beta2_linux_amd64.tar.gz
-* https://dl.influxdata.com/telegraf/releases/telegraf-1.0.0-beta2_linux_i386.tar.gz
-* https://dl.influxdata.com/telegraf/releases/telegraf-1.0.0-beta2_linux_armhf.tar.gz
+* https://dl.influxdata.com/telegraf/releases/telegraf-1.0.0-rc1_linux_amd64.tar.gz
+* https://dl.influxdata.com/telegraf/releases/telegraf-1.0.0-rc1_linux_i386.tar.gz
+* https://dl.influxdata.com/telegraf/releases/telegraf-1.0.0-rc1_linux_armhf.tar.gz
 
 ### FreeBSD tarball:
 
 Latest:
-* https://dl.influxdata.com/telegraf/releases/telegraf-1.0.0-beta2_freebsd_amd64.tar.gz
+* https://dl.influxdata.com/telegraf/releases/telegraf-1.0.0-rc1_freebsd_amd64.tar.gz
 
 ### Ansible Role:
 
@@ -69,7 +69,7 @@ brew install telegraf
 ### Windows Binaries (EXPERIMENTAL)
 
 Latest:
-* https://dl.influxdata.com/telegraf/releases/telegraf-1.0.0-beta2_windows_amd64.zip
+* https://dl.influxdata.com/telegraf/releases/telegraf-1.0.0-rc1_windows_amd64.zip
 
 ### From Source:
 
@@ -156,6 +156,7 @@ Currently implemented sources:
 * [exec](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/exec) (generic executable plugin, support JSON, influx, graphite and nagios)
 * [filestat](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/filestat)
 * [haproxy](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/haproxy)
+* [hddtemp](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/hddtemp)
 * [http_response](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/http_response)
 * [httpjson](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/httpjson) (generic JSON-emitting http service plugin)
 * [influxdb](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/influxdb)
@@ -187,8 +188,9 @@ Currently implemented sources:
 * [redis](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/redis)
 * [rethinkdb](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/rethinkdb)
 * [riak](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/riak)
-* [sensors ](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/sensors) (only available if built from source)
+* [sensors](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/sensors)
 * [snmp](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/snmp)
+* [snmp_legacy](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/snmp_legacy)
 * [sql server](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/sqlserver) (microsoft)
 * [twemproxy](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/twemproxy)
 * [varnish](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/varnish)
@@ -217,8 +219,11 @@ Telegraf can also collect metrics via the following service plugins:
 * [mqtt_consumer](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/mqtt_consumer)
 * [kafka_consumer](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/kafka_consumer)
 * [nats_consumer](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/nats_consumer)
-* [github_webhooks](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/github_webhooks)
-* [rollbar_webhooks](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/rollbar_webhooks)
+* [webhooks](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/webhooks)
+  * [github](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/webhooks/github)
+  * [mandrill](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/webhooks/mandrill)
+  * [rollbar](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/webhooks/rollbar)
+* [nsq_consumer](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/nsq_consumer)
 
 We'll be adding support for many more over the coming months. Read on if you
 want to add support for another service or third-party API.

accumulator.go

@@ -16,6 +16,8 @@ type Accumulator interface {
 		tags map[string]string,
 		t ...time.Time)
 
+	AddError(err error)
+
 	Debug() bool
 	SetDebug(enabled bool)
 

agent/accumulator.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"log"
 	"math"
+	"sync/atomic"
 	"time"
 
 	"github.com/influxdata/telegraf"
@@ -11,7 +12,7 @@ import (
 )
 
 func NewAccumulator(
-	inputConfig *internal_models.InputConfig,
+	inputConfig *models.InputConfig,
 	metrics chan telegraf.Metric,
 ) *accumulator {
 	acc := accumulator{}
@@ -30,11 +31,11 @@ type accumulator struct {
 	// print every point added to the accumulator
 	trace bool
 
-	inputConfig *internal_models.InputConfig
-
-	prefix string
+	inputConfig *models.InputConfig
 
 	precision time.Duration
+
+	errCount uint64
 }
 
 func (ac *accumulator) Add(
@@ -146,10 +147,6 @@ func (ac *accumulator) AddFields(
 	}
 	timestamp = timestamp.Round(ac.precision)
 
-	if ac.prefix != "" {
-		measurement = ac.prefix + measurement
-	}
-
 	m, err := telegraf.NewMetric(measurement, tags, result, timestamp)
 	if err != nil {
 		log.Printf("Error adding point [%s]: %s\n", measurement, err.Error())
@@ -161,6 +158,17 @@ func (ac *accumulator) AddFields(
 	ac.metrics <- m
 }
 
+// AddError passes a runtime error to the accumulator.
+// The error will be tagged with the plugin name and written to the log.
+func (ac *accumulator) AddError(err error) {
+	if err == nil {
+		return
+	}
+	atomic.AddUint64(&ac.errCount, 1)
+	//TODO suppress/throttle consecutive duplicate errors?
+	log.Printf("ERROR in input [%s]: %s", ac.inputConfig.Name, err)
+}
+
 func (ac *accumulator) Debug() bool {
 	return ac.debug
 }

agent/accumulator_test.go

@@ -1,8 +1,11 @@
 package agent
 
 import (
+	"bytes"
 	"fmt"
+	"log"
 	"math"
+	"os"
 	"testing"
 	"time"
 
@@ -10,6 +13,7 @@ import (
 	"github.com/influxdata/telegraf/internal/models"
 
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
 func TestAdd(t *testing.T) {
@@ -17,7 +21,7 @@ func TestAdd(t *testing.T) {
 	now := time.Now()
 	a.metrics = make(chan telegraf.Metric, 10)
 	defer close(a.metrics)
-	a.inputConfig = &internal_models.InputConfig{}
+	a.inputConfig = &models.InputConfig{}
 
 	a.Add("acctest", float64(101), map[string]string{})
 	a.Add("acctest", float64(101), map[string]string{"acc": "test"})
@@ -43,7 +47,7 @@ func TestAddNoPrecisionWithInterval(t *testing.T) {
 	now := time.Date(2006, time.February, 10, 12, 0, 0, 82912748, time.UTC)
 	a.metrics = make(chan telegraf.Metric, 10)
 	defer close(a.metrics)
-	a.inputConfig = &internal_models.InputConfig{}
+	a.inputConfig = &models.InputConfig{}
 
 	a.SetPrecision(0, time.Second)
 	a.Add("acctest", float64(101), map[string]string{})
@@ -70,7 +74,7 @@ func TestAddNoIntervalWithPrecision(t *testing.T) {
 	now := time.Date(2006, time.February, 10, 12, 0, 0, 82912748, time.UTC)
 	a.metrics = make(chan telegraf.Metric, 10)
 	defer close(a.metrics)
-	a.inputConfig = &internal_models.InputConfig{}
+	a.inputConfig = &models.InputConfig{}
 
 	a.SetPrecision(time.Second, time.Millisecond)
 	a.Add("acctest", float64(101), map[string]string{})
@@ -97,7 +101,7 @@ func TestAddDisablePrecision(t *testing.T) {
 	now := time.Date(2006, time.February, 10, 12, 0, 0, 82912748, time.UTC)
 	a.metrics = make(chan telegraf.Metric, 10)
 	defer close(a.metrics)
-	a.inputConfig = &internal_models.InputConfig{}
+	a.inputConfig = &models.InputConfig{}
 
 	a.SetPrecision(time.Second, time.Millisecond)
 	a.DisablePrecision()
@@ -125,7 +129,7 @@ func TestDifferentPrecisions(t *testing.T) {
 	now := time.Date(2006, time.February, 10, 12, 0, 0, 82912748, time.UTC)
 	a.metrics = make(chan telegraf.Metric, 10)
 	defer close(a.metrics)
-	a.inputConfig = &internal_models.InputConfig{}
+	a.inputConfig = &models.InputConfig{}
 
 	a.SetPrecision(0, time.Second)
 	a.Add("acctest", float64(101), map[string]string{"acc": "test"}, now)
@@ -166,7 +170,7 @@ func TestAddDefaultTags(t *testing.T) {
 	now := time.Now()
 	a.metrics = make(chan telegraf.Metric, 10)
 	defer close(a.metrics)
-	a.inputConfig = &internal_models.InputConfig{}
+	a.inputConfig = &models.InputConfig{}
 
 	a.Add("acctest", float64(101), map[string]string{})
 	a.Add("acctest", float64(101), map[string]string{"acc": "test"})
@@ -192,7 +196,7 @@ func TestAddFields(t *testing.T) {
 	now := time.Now()
 	a.metrics = make(chan telegraf.Metric, 10)
 	defer close(a.metrics)
-	a.inputConfig = &internal_models.InputConfig{}
+	a.inputConfig = &models.InputConfig{}
 
 	fields := map[string]interface{}{
 		"usage": float64(99),
@@ -225,7 +229,7 @@ func TestAddInfFields(t *testing.T) {
 	now := time.Now()
 	a.metrics = make(chan telegraf.Metric, 10)
 	defer close(a.metrics)
-	a.inputConfig = &internal_models.InputConfig{}
+	a.inputConfig = &models.InputConfig{}
 
 	fields := map[string]interface{}{
 		"usage":  inf,
@@ -253,7 +257,7 @@ func TestAddNaNFields(t *testing.T) {
 	now := time.Now()
 	a.metrics = make(chan telegraf.Metric, 10)
 	defer close(a.metrics)
-	a.inputConfig = &internal_models.InputConfig{}
+	a.inputConfig = &models.InputConfig{}
 
 	fields := map[string]interface{}{
 		"usage": nan,
@@ -277,7 +281,7 @@ func TestAddUint64Fields(t *testing.T) {
 	now := time.Now()
 	a.metrics = make(chan telegraf.Metric, 10)
 	defer close(a.metrics)
-	a.inputConfig = &internal_models.InputConfig{}
+	a.inputConfig = &models.InputConfig{}
 
 	fields := map[string]interface{}{
 		"usage": uint64(99),
@@ -306,7 +310,7 @@ func TestAddUint64Overflow(t *testing.T) {
 	now := time.Now()
 	a.metrics = make(chan telegraf.Metric, 10)
 	defer close(a.metrics)
-	a.inputConfig = &internal_models.InputConfig{}
+	a.inputConfig = &models.InputConfig{}
 
 	fields := map[string]interface{}{
 		"usage": uint64(9223372036854775808),
@@ -336,7 +340,7 @@ func TestAddInts(t *testing.T) {
 	now := time.Now()
 	a.metrics = make(chan telegraf.Metric, 10)
 	defer close(a.metrics)
-	a.inputConfig = &internal_models.InputConfig{}
+	a.inputConfig = &models.InputConfig{}
 
 	a.Add("acctest", int(101), map[string]string{})
 	a.Add("acctest", int32(101), map[string]string{"acc": "test"})
@@ -363,7 +367,7 @@ func TestAddFloats(t *testing.T) {
 	now := time.Now()
 	a.metrics = make(chan telegraf.Metric, 10)
 	defer close(a.metrics)
-	a.inputConfig = &internal_models.InputConfig{}
+	a.inputConfig = &models.InputConfig{}
 
 	a.Add("acctest", float32(101), map[string]string{"acc": "test"})
 	a.Add("acctest", float64(101), map[string]string{"acc": "test"}, now)
@@ -385,7 +389,7 @@ func TestAddStrings(t *testing.T) {
 	now := time.Now()
 	a.metrics = make(chan telegraf.Metric, 10)
 	defer close(a.metrics)
-	a.inputConfig = &internal_models.InputConfig{}
+	a.inputConfig = &models.InputConfig{}
 
 	a.Add("acctest", "test", map[string]string{"acc": "test"})
 	a.Add("acctest", "foo", map[string]string{"acc": "test"}, now)
@@ -407,7 +411,7 @@ func TestAddBools(t *testing.T) {
 	now := time.Now()
 	a.metrics = make(chan telegraf.Metric, 10)
 	defer close(a.metrics)
-	a.inputConfig = &internal_models.InputConfig{}
+	a.inputConfig = &models.InputConfig{}
 
 	a.Add("acctest", true, map[string]string{"acc": "test"})
 	a.Add("acctest", false, map[string]string{"acc": "test"}, now)
@@ -429,11 +433,11 @@ func TestAccFilterTags(t *testing.T) {
 	now := time.Now()
 	a.metrics = make(chan telegraf.Metric, 10)
 	defer close(a.metrics)
-	filter := internal_models.Filter{
+	filter := models.Filter{
 		TagExclude: []string{"acc"},
 	}
 	assert.NoError(t, filter.CompileFilter())
-	a.inputConfig = &internal_models.InputConfig{}
+	a.inputConfig = &models.InputConfig{}
 	a.inputConfig.Filter = filter
 
 	a.Add("acctest", float64(101), map[string]string{})
@@ -454,3 +458,27 @@ func TestAccFilterTags(t *testing.T) {
 		fmt.Sprintf("acctest value=101 %d", now.UnixNano()),
 		actual)
 }
+
+func TestAccAddError(t *testing.T) {
+	errBuf := bytes.NewBuffer(nil)
+	log.SetOutput(errBuf)
+	defer log.SetOutput(os.Stderr)
+
+	a := accumulator{}
+	a.inputConfig = &models.InputConfig{}
+	a.inputConfig.Name = "mock_plugin"
+
+	a.AddError(fmt.Errorf("foo"))
+	a.AddError(fmt.Errorf("bar"))
+	a.AddError(fmt.Errorf("baz"))
+
+	errs := bytes.Split(errBuf.Bytes(), []byte{'\n'})
+	assert.EqualValues(t, 3, a.errCount)
+	require.Len(t, errs, 4) // 4 because of trailing newline
+	assert.Contains(t, string(errs[0]), "mock_plugin")
+	assert.Contains(t, string(errs[0]), "foo")
+	assert.Contains(t, string(errs[1]), "mock_plugin")
+	assert.Contains(t, string(errs[1]), "bar")
+	assert.Contains(t, string(errs[2]), "mock_plugin")
+	assert.Contains(t, string(errs[2]), "baz")
+}

agent/agent.go

@@ -88,7 +88,7 @@ func (a *Agent) Close() error {
 	return err
 }
 
-func panicRecover(input *internal_models.RunningInput) {
+func panicRecover(input *models.RunningInput) {
 	if err := recover(); err != nil {
 		trace := make([]byte, 2048)
 		runtime.Stack(trace, true)
@@ -104,7 +104,7 @@ func panicRecover(input *internal_models.RunningInput) {
 // reporting interval.
 func (a *Agent) gatherer(
 	shutdown chan struct{},
-	input *internal_models.RunningInput,
+	input *models.RunningInput,
 	interval time.Duration,
 	metricC chan telegraf.Metric,
 ) error {
@@ -152,7 +152,7 @@ func (a *Agent) gatherer(
 //   over.
 func gatherWithTimeout(
 	shutdown chan struct{},
-	input *internal_models.RunningInput,
+	input *models.RunningInput,
 	acc *accumulator,
 	timeout time.Duration,
 ) {
@@ -215,6 +215,9 @@ func (a *Agent) Test() error {
 		if err := input.Input.Gather(acc); err != nil {
 			return err
 		}
+		if acc.errCount > 0 {
+			return fmt.Errorf("Errors encountered during processing")
+		}
 
 		// Special instructions for some inputs. cpu, for example, needs to be
 		// run twice in order to return cpu usage percentages.
@@ -237,7 +240,7 @@ func (a *Agent) flush() {
 
 	wg.Add(len(a.Config.Outputs))
 	for _, o := range a.Config.Outputs {
-		go func(output *internal_models.RunningOutput) {
+		go func(output *models.RunningOutput) {
 			defer wg.Done()
 			err := output.Write()
 			if err != nil {
@@ -268,13 +271,33 @@ func (a *Agent) flusher(shutdown chan struct{}, metricC chan telegraf.Metric) er
 			internal.RandomSleep(a.Config.Agent.FlushJitter.Duration, shutdown)
 			a.flush()
 		case m := <-metricC:
-			for _, o := range a.Config.Outputs {
-				o.AddMetric(m)
+			for i, o := range a.Config.Outputs {
+				if i == len(a.Config.Outputs)-1 {
+					o.AddMetric(m)
+				} else {
+					o.AddMetric(copyMetric(m))
+				}
 			}
 		}
 	}
 }
 
+func copyMetric(m telegraf.Metric) telegraf.Metric {
+	t := time.Time(m.Time())
+
+	tags := make(map[string]string)
+	fields := make(map[string]interface{})
+	for k, v := range m.Tags() {
+		tags[k] = v
+	}
+	for k, v := range m.Fields() {
+		fields[k] = v
+	}
+
+	out, _ := telegraf.NewMetric(m.Name(), tags, fields, t)
+	return out
+}
+
 // Run runs the agent daemon, gathering every Interval
 func (a *Agent) Run(shutdown chan struct{}) error {
 	var wg sync.WaitGroup
@@ -328,7 +351,7 @@ func (a *Agent) Run(shutdown chan struct{}) error {
 		if input.Config.Interval != 0 {
 			interval = input.Config.Interval
 		}
-		go func(in *internal_models.RunningInput, interv time.Duration) {
+		go func(in *models.RunningInput, interv time.Duration) {
 			defer wg.Done()
 			if err := a.gatherer(shutdown, in, interv, metricC); err != nil {
 				log.Printf(err.Error())

cmd/telegraf/telegraf.go

@@ -6,6 +6,7 @@ import (
 	"log"
 	"os"
 	"os/signal"
+	"runtime"
 	"strings"
 	"syscall"
 
@@ -15,6 +16,7 @@ import (
 	_ "github.com/influxdata/telegraf/plugins/inputs/all"
 	"github.com/influxdata/telegraf/plugins/outputs"
 	_ "github.com/influxdata/telegraf/plugins/outputs/all"
+	"github.com/kardianos/service"
 )
 
 var fDebug = flag.Bool("debug", false,
@@ -39,12 +41,8 @@ var fOutputList = flag.Bool("output-list", false,
 	"print available output plugins.")
 var fUsage = flag.String("usage", "",
 	"print usage for a plugin, ie, 'telegraf -usage mysql'")
-var fInputFiltersLegacy = flag.String("filter", "",
-	"filter the inputs to enable, separator is :")
-var fOutputFiltersLegacy = flag.String("outputfilter", "",
-	"filter the outputs to enable, separator is :")
-var fConfigDirectoryLegacy = flag.String("configdirectory", "",
-	"directory containing additional *.conf files")
+var fService = flag.String("service", "",
+	"operate on the service")
 
 // Telegraf version, populated linker.
 //   ie, -ldflags "-X main.version=`git describe --always --tags`"
@@ -74,6 +72,7 @@ The flags are:
   -debug             print metrics as they're generated to stdout
   -quiet             run in quiet mode
   -version           print the version to stdout
+  -service           Control the service, ie, 'telegraf -service install (windows only)'
 
 In addition to the -config flag, telegraf will also load the config file from
 an environment variable or default location. Precedence is:
@@ -100,7 +99,22 @@ Examples:
   telegraf -config telegraf.conf -input-filter cpu:mem -output-filter influxdb
 `
 
-func main() {
+var logger service.Logger
+
+var stop chan struct{}
+
+var srvc service.Service
+var svcConfig *service.Config
+
+type program struct{}
+
+func reloadLoop(stop chan struct{}, s service.Service) {
+	defer func() {
+		if service.Interactive() {
+			os.Exit(0)
+		}
+		return
+	}()
 	reload := make(chan bool, 1)
 	reload <- true
 	for <-reload {
@@ -110,24 +124,11 @@ func main() {
 		args := flag.Args()
 
 		var inputFilters []string
-		if *fInputFiltersLegacy != "" {
-			fmt.Printf("WARNING '--filter' flag is deprecated, please use" +
-				" '--input-filter'")
-			inputFilter := strings.TrimSpace(*fInputFiltersLegacy)
-			inputFilters = strings.Split(":"+inputFilter+":", ":")
-		}
 		if *fInputFilters != "" {
 			inputFilter := strings.TrimSpace(*fInputFilters)
 			inputFilters = strings.Split(":"+inputFilter+":", ":")
 		}
-
 		var outputFilters []string
-		if *fOutputFiltersLegacy != "" {
-			fmt.Printf("WARNING '--outputfilter' flag is deprecated, please use" +
-				" '--output-filter'")
-			outputFilter := strings.TrimSpace(*fOutputFiltersLegacy)
-			outputFilters = strings.Split(":"+outputFilter+":", ":")
-		}
 		if *fOutputFilters != "" {
 			outputFilter := strings.TrimSpace(*fOutputFilters)
 			outputFilters = strings.Split(":"+outputFilter+":", ":")
@@ -145,40 +146,43 @@ func main() {
 			}
 		}
 
-		if *fOutputList {
+		// switch for flags which just do something and exit immediately
+		switch {
+		case *fOutputList:
 			fmt.Println("Available Output Plugins:")
 			for k, _ := range outputs.Outputs {
 				fmt.Printf("  %s\n", k)
 			}
 			return
-		}
-
-		if *fInputList {
+		case *fInputList:
 			fmt.Println("Available Input Plugins:")
 			for k, _ := range inputs.Inputs {
 				fmt.Printf("  %s\n", k)
 			}
 			return
-		}
-
-		if *fVersion {
+		case *fVersion:
 			v := fmt.Sprintf("Telegraf - version %s", version)
 			fmt.Println(v)
 			return
-		}
-
-		if *fSampleConfig {
+		case *fSampleConfig:
 			config.PrintSampleConfig(inputFilters, outputFilters)
 			return
-		}
-
-		if *fUsage != "" {
+		case *fUsage != "":
 			if err := config.PrintInputConfig(*fUsage); err != nil {
 				if err2 := config.PrintOutputConfig(*fUsage); err2 != nil {
 					log.Fatalf("%s and %s", err, err2)
 				}
 			}
 			return
+		case *fService != "" && runtime.GOOS == "windows":
+			if *fConfig != "" {
+				(*svcConfig).Arguments = []string{"-config", *fConfig}
+			}
+			err := service.Control(s, *fService)
+			if err != nil {
+				log.Fatal(err)
+			}
+			return
 		}
 
 		// If no other options are specified, load the config file and run.
@@ -191,15 +195,6 @@ func main() {
 			os.Exit(1)
 		}
 
-		if *fConfigDirectoryLegacy != "" {
-			fmt.Printf("WARNING '--configdirectory' flag is deprecated, please use" +
-				" '--config-directory'")
-			err = c.LoadDirectory(*fConfigDirectoryLegacy)
-			if err != nil {
-				log.Fatal(err)
-			}
-		}
-
 		if *fConfigDirectory != "" {
 			err = c.LoadDirectory(*fConfigDirectory)
 			if err != nil {
@@ -243,14 +238,18 @@ func main() {
 		signals := make(chan os.Signal)
 		signal.Notify(signals, os.Interrupt, syscall.SIGHUP)
 		go func() {
-			sig := <-signals
-			if sig == os.Interrupt {
-				close(shutdown)
-			}
-			if sig == syscall.SIGHUP {
-				log.Printf("Reloading Telegraf config\n")
-				<-reload
-				reload <- true
+			select {
+			case sig := <-signals:
+				if sig == os.Interrupt {
+					close(shutdown)
+				}
+				if sig == syscall.SIGHUP {
+					log.Printf("Reloading Telegraf config\n")
+					<-reload
+					reload <- true
+					close(shutdown)
+				}
+			case <-stop:
 				close(shutdown)
 			}
 		}()
@@ -279,3 +278,46 @@ func usageExit(rc int) {
 	fmt.Println(usage)
 	os.Exit(rc)
 }
+
+func (p *program) Start(s service.Service) error {
+	srvc = s
+	go p.run()
+	return nil
+}
+func (p *program) run() {
+	stop = make(chan struct{})
+	reloadLoop(stop, srvc)
+}
+func (p *program) Stop(s service.Service) error {
+	close(stop)
+	return nil
+}
+
+func main() {
+	if runtime.GOOS == "windows" {
+		svcConfig = &service.Config{
+			Name:        "telegraf",
+			DisplayName: "Telegraf Data Collector Service",
+			Description: "Collects data using a series of plugins and publishes it to" +
+				"another series of plugins.",
+			Arguments: []string{"-config", "C:\\Program Files\\Telegraf\\telegraf.conf"},
+		}
+
+		prg := &program{}
+		s, err := service.New(prg, svcConfig)
+		if err != nil {
+			log.Fatal(err)
+		}
+		logger, err = s.Logger(nil)
+		if err != nil {
+			log.Fatal(err)
+		}
+		err = s.Run()
+		if err != nil {
+			logger.Error(err)
+		}
+	} else {
+		stop = make(chan struct{})
+		reloadLoop(stop, nil)
+	}
+}

docs/LICENSE_OF_DEPENDENCIES.md

@@ -16,6 +16,7 @@
 - github.com/hashicorp/go-msgpack [BSD LICENSE](https://github.com/hashicorp/go-msgpack/blob/master/LICENSE)
 - github.com/hashicorp/raft [MPL LICENSE](https://github.com/hashicorp/raft/blob/master/LICENSE)
 - github.com/hashicorp/raft-boltdb [MPL LICENSE](https://github.com/hashicorp/raft-boltdb/blob/master/LICENSE)
+- github.com/kardianos/service [ZLIB LICENSE](https://github.com/kardianos/service/blob/master/LICENSE) (License not named but matches word for word with ZLib)
 - github.com/lib/pq [MIT LICENSE](https://github.com/lib/pq/blob/master/LICENSE.md)
 - github.com/matttproud/golang_protobuf_extensions [APACHE LICENSE](https://github.com/matttproud/golang_protobuf_extensions/blob/master/LICENSE)
 - github.com/naoina/go-stringutil [MIT LICENSE](https://github.com/naoina/go-stringutil/blob/master/LICENSE)

docs/WINDOWS_SERVICE.md

@@ -1,36 +1,40 @@
 # Running Telegraf as a Windows Service
 
-If you have tried to install Go binaries as Windows Services with the **sc.exe**
-tool you may have seen that the service errors and stops running after a while.
+Telegraf natively supports running as a Windows Service. Outlined below is are
+the general steps to set it up.
 
-**NSSM** (the Non-Sucking Service Manager) is a tool that helps you in a 
-[number of scenarios](http://nssm.cc/scenarios) including running Go binaries
-that were not specifically designed to run only in Windows platforms.
+1. Obtain the telegraf windows distribution
+2. Create the directory `C:\Program Files\Telegraf` (if you install in a different
+   location simply specify the `-config` parameter with the desired location)
+3. Place the telegraf.exe and the config file into `C:\Program Files\Telegraf`
+4. To install the service into the Windows Service Manager, run (as an
+   administrator):
 
-## NSSM Installation via Chocolatey
+   ```
+   > C:\Program Files\Telegraf\telegraf.exe --service install
+   ```
 
-You can install [Chocolatey](https://chocolatey.org/) and [NSSM](http://nssm.cc/) 
-with these commands
+5. Edit the configuration file to meet your needs
+6. To check that it works, run:
 
-```powershell
-iex ((new-object net.webclient).DownloadString('https://chocolatey.org/install.ps1'))
-choco install -y nssm
-```
+   ```
+   > C:\Program Files\Telegraf\telegraf.exe --config C:\Program Files\Telegraf\telegraf.conf --test
+   ```
 
-## Installing Telegraf as a Windows Service with NSSM
+7. To start collecting data, run:
 
-You can download the latest Telegraf Windows binaries (still Experimental at 
-the moment) from [the Telegraf Github repo](https://github.com/influxdata/telegraf).
+   ```
+   > net start telegraf
+   ```
 
-Then you can create a C:\telegraf folder, unzip the binary there and modify the 
-**telegraf.conf** sample to allocate the metrics you want to send to **InfluxDB**.
+## Other supported operations
 
-Once you have NSSM installed in your system, the process is quite straightforward.
-You only need to type this command in your Windows shell
+Telegraf can manage its own service through the --service flag:
 
-```powershell
-nssm install Telegraf c:\telegraf\telegraf.exe -config c:\telegraf\telegraf.config
-```
+| Command                            | Effect                        |
+|------------------------------------|-------------------------------|
+| `telegraf.exe --service install`   | Install telegraf as a service |
+| `telegraf.exe --service uninstall` | Remove the telegraf service   |
+| `telegraf.exe --service start`     | Start the telegraf service    |
+| `telegraf.exe --service stop`      | Stop the telegraf service     |
 
-And now your service will be installed in Windows and you will be able to start and
-stop it gracefully

etc/telegraf.conf

@@ -55,7 +55,7 @@
   ## By default, precision will be set to the same timestamp order as the
   ## collection interval, with the maximum being 1s.
   ## Precision will NOT be used for service inputs, such as logparser and statsd.
-  ## Valid values are "Nns", "Nus" (or "Nµs"), "Nms", "Ns".
+  ## Valid values are "ns", "us" (or "µs"), "ms", "s".
   precision = ""
   ## Run telegraf in debug mode
   debug = false
@@ -83,7 +83,7 @@
 
   ## Retention policy to write to. Empty string writes to the default rp.
   retention_policy = ""
-  ## Write consistency (clusters only), can be: "any", "one", "quorom", "all"
+  ## Write consistency (clusters only), can be: "any", "one", "quorum", "all"
   write_consistency = "any"
 
   ## Write timeout (for the InfluxDB client), formatted as a string.
@@ -197,6 +197,8 @@
 # # Configuration for Graphite server to send metrics to
 # [[outputs.graphite]]
 #   ## TCP endpoint for your graphite instance.
+#   ## If multiple endpoints are configured, output will be load balanced.
+#   ## Only one of the endpoints will be written to with each iteration.
 #   servers = ["localhost:2003"]
 #   ## Prefix metrics name
 #   prefix = ""
@@ -319,14 +321,13 @@
 #   api_token = "my-secret-token" # required.
 #   ## Debug
 #   # debug = false
-#   ## Tag Field to populate source attribute (optional)
-#   ## This is typically the _hostname_ from which the metric was obtained.
-#   source_tag = "host"
 #   ## Connection timeout.
 #   # timeout = "5s"
-#   ## Output Name Template (same as graphite buckets)
+#   ## Output source Template (same as graphite buckets)
 #   ## see https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_OUTPUT.md#graphite
-#   template = "host.tags.measurement.field"
+#   ## This template is used in librato's source (not metric's name)
+#   template = "host"
+#
 
 
 # # Configuration for MQTT server to send metrics to
@@ -434,8 +435,8 @@
   ## disk partitions.
   ## Setting devices will restrict the stats to the specified devices.
   # devices = ["sda", "sdb"]
-  ## Uncomment the following line if you do not need disk serial numbers.
-  # skip_serial_number = true
+  ## Uncomment the following line if you need disk serial numbers.
+  # skip_serial_number = false
 
 
 # Get kernel statistics from /proc/stat
@@ -463,7 +464,7 @@
   # no configuration
 
 
-# # Read stats from an aerospike server
+# # Read stats from aerospike server(s)
 # [[inputs.aerospike]]
 #   ## Aerospike servers to connect to (with port)
 #   ## This plugin will query all namespaces the aerospike
@@ -526,6 +527,19 @@
 #   socket_suffix = "asok"
 
 
+# # Read specific statistics per cgroup
+# [[inputs.cgroup]]
+#     ## Directories in which to look for files, globs are supported.
+# 	# paths = [
+# 	#   "/cgroup/memory",
+# 	#   "/cgroup/memory/child1",
+# 	#   "/cgroup/memory/child2/*",
+# 	# ]
+# 	## cgroup stat fields, as file names, globs are supported.
+# 	## these file names are appended to each path from above.
+# 	# files = ["memory.*usage*", "memory.limit_in_bytes"]
+
+
 # # Pull Metric Statistics from Amazon CloudWatch
 # [[inputs.cloudwatch]]
 #   ## Amazon Region
@@ -651,6 +665,13 @@
 #   container_names = []
 #   ## Timeout for docker list, info, and stats commands
 #   timeout = "5s"
+#
+#   ## Whether to report for each container per-device blkio (8:0, 8:1...) and
+#   ## network (eth0, eth1, ...) stats or not
+#   perdevice = true
+#   ## Whether to report for each container total blkio and network stats or not
+#   total = false
+#
 
 
 # # Read statistics from one or many dovecot servers
@@ -679,6 +700,13 @@
 #
 #   ## set cluster_health to true when you want to also obtain cluster level stats
 #   cluster_health = false
+#
+#   ## Optional SSL Config
+#   # ssl_ca = "/etc/telegraf/ca.pem"
+#   # ssl_cert = "/etc/telegraf/cert.pem"
+#   # ssl_key = "/etc/telegraf/key.pem"
+#   ## Use SSL but skip chain & host verification
+#   # insecure_skip_verify = false
 
 
 # # Read metrics from one or more commands that can output to stdout
@@ -760,9 +788,11 @@
 # [[inputs.haproxy]]
 #   ## An array of address to gather stats about. Specify an ip on hostname
 #   ## with optional port. ie localhost, 10.10.3.33:1936, etc.
-#
-#   ## If no servers are specified, then default to 127.0.0.1:1936
-#   servers = ["http://myhaproxy.com:1936", "http://anotherhaproxy.com:1936"]
+#   ## Make sure you specify the complete path to the stats endpoint
+#   ## ie 10.10.3.33:1936/haproxy?stats
+#   #
+#   ## If no servers are specified, then default to 127.0.0.1:1936/haproxy?stats
+#   servers = ["http://myhaproxy.com:1936/haproxy?stats"]
 #   ## Or you can also use local socket
 #   ## servers = ["socket:/run/haproxy/admin.sock"]
 
@@ -948,21 +978,35 @@
 
 # # Telegraf plugin for gathering metrics from N Mesos masters
 # [[inputs.mesos]]
-#   # Timeout, in ms.
+#   ## Timeout, in ms.
 #   timeout = 100
-#   # A list of Mesos masters, default value is localhost:5050.
+#   ## A list of Mesos masters.
 #   masters = ["localhost:5050"]
-#   # Metrics groups to be collected, by default, all enabled.
+#   ## Master metrics groups to be collected, by default, all enabled.
 #   master_collections = [
 #     "resources",
 #     "master",
 #     "system",
-#     "slaves",
+#     "agents",
 #     "frameworks",
+#     "tasks",
 #     "messages",
 #     "evqueue",
 #     "registrar",
 #   ]
+#   ## A list of Mesos slaves, default is []
+#   # slaves = []
+#   ## Slave metrics groups to be collected, by default, all enabled.
+#   # slave_collections = [
+#   #   "resources",
+#   #   "agent",
+#   #   "system",
+#   #   "executors",
+#   #   "tasks",
+#   #   "messages",
+#   # ]
+#   ## Include mesos tasks statistics, default is false
+#   # slave_tasks = true
 
 
 # # Read metrics from one or many MongoDB servers
@@ -973,6 +1017,7 @@
 #   ##   mongodb://10.10.3.33:18832,
 #   ##   10.0.0.1:10000, etc.
 #   servers = ["127.0.0.1:27017"]
+#   gather_perdb_stats = false
 
 
 # # Read metrics from one or many mysql servers
@@ -1079,9 +1124,9 @@
 #   ## file paths for proc files. If empty default paths will be used:
 #   ##    /proc/net/netstat, /proc/net/snmp, /proc/net/snmp6
 #   ## These can also be overridden with env variables, see README.
-#   proc_net_netstat = ""
-#   proc_net_snmp    = ""
-#   proc_net_snmp6   = ""
+#   proc_net_netstat = "/proc/net/netstat"
+#   proc_net_snmp = "/proc/net/snmp"
+#   proc_net_snmp6 = "/proc/net/snmp6"
 #   ## dump metrics with 0 values too
 #   dump_zeros       = true
 
@@ -1259,10 +1304,15 @@
 #   ## An array of urls to scrape metrics from.
 #   urls = ["http://localhost:9100/metrics"]
 #
-#   ## Use SSL but skip chain & host verification
-#   # insecure_skip_verify = false
 #   ## Use bearer token for authorization
 #   # bearer_token = /path/to/bearer/token
+#
+#   ## Optional SSL Config
+#   # ssl_ca = /path/to/cafile
+#   # ssl_cert = /path/to/certfile
+#   # ssl_key = /path/to/keyfile
+#   ## Use SSL but skip chain & host verification
+#   # insecure_skip_verify = false
 
 
 # # Reads last_run_summary.yaml file and converts to measurments
@@ -1278,6 +1328,13 @@
 #   # username = "guest"
 #   # password = "guest"
 #
+#   ## Optional SSL Config
+#   # ssl_ca = "/etc/telegraf/ca.pem"
+#   # ssl_cert = "/etc/telegraf/cert.pem"
+#   # ssl_key = "/etc/telegraf/key.pem"
+#   ## Use SSL but skip chain & host verification
+#   # insecure_skip_verify = false
+#
 #   ## A list of nodes to pull metrics about. If not specified, metrics for
 #   ## all nodes are gathered.
 #   # nodes = ["rabbit@node1", "rabbit@node2"]
@@ -1296,6 +1353,7 @@
 #   ##  e.g.
 #   ##    tcp://localhost:6379
 #   ##    tcp://:password@192.168.99.100
+#   ##    unix:///var/run/redis.sock
 #   ##
 #   ## If no servers are specified, then localhost is used as the host.
 #   ## If no port is specified, 6379 is used
@@ -1318,8 +1376,63 @@
 #   servers = ["http://localhost:8098"]
 
 
-# # Reads oids value from one or many snmp agents
+# # Retrieves SNMP values from remote agents
 # [[inputs.snmp]]
+#   agents = [ "127.0.0.1:161" ]
+#   timeout = "5s"
+#   version = 2
+#
+#   # SNMPv1 & SNMPv2 parameters
+#   community = "public"
+#
+#   # SNMPv2 & SNMPv3 parameters
+#   max_repetitions = 50
+#
+#   # SNMPv3 parameters
+#   #sec_name = "myuser"
+#   #auth_protocol = "md5"         # Values: "MD5", "SHA", ""
+#   #auth_password = "password123"
+#   #sec_level = "authNoPriv"      # Values: "noAuthNoPriv", "authNoPriv", "authPriv"
+#   #context_name = ""
+#   #priv_protocol = ""            # Values: "DES", "AES", ""
+#   #priv_password = ""
+#
+#   # measurement name
+#   name = "system"
+#   [[inputs.snmp.field]]
+#     name = "hostname"
+#     oid = ".1.0.0.1.1"
+#   [[inputs.snmp.field]]
+#     name = "uptime"
+#     oid = ".1.0.0.1.2"
+#   [[inputs.snmp.field]]
+#     name = "load"
+#     oid = ".1.0.0.1.3"
+#   [[inputs.snmp.field]]
+#     oid = "HOST-RESOURCES-MIB::hrMemorySize"
+#
+#   [[inputs.snmp.table]]
+#     # measurement name
+#     name = "remote_servers"
+#     inherit_tags = [ "hostname" ]
+#     [[inputs.snmp.table.field]]
+#       name = "server"
+#       oid = ".1.0.0.0.1.0"
+#       is_tag = true
+#     [[inputs.snmp.table.field]]
+#       name = "connections"
+#       oid = ".1.0.0.0.1.1"
+#     [[inputs.snmp.table.field]]
+#       name = "latency"
+#       oid = ".1.0.0.0.1.2"
+#
+#   [[inputs.snmp.table]]
+#     # auto populate table's fields using the MIB
+#     oid = "HOST-RESOURCES-MIB::hrNetworkTable"
+
+
+# # DEPRECATED! PLEASE USE inputs.snmp INSTEAD.
+# [[inputs.snmp_legacy]]
 #   ## Use 'oids.txt' file to translate oids to names
 #   ## To generate 'oids.txt' you need to run:
 #   ##   snmptranslate -m all -Tz -On | sed -e 's/"//g' > /tmp/oids.txt
@@ -1490,12 +1603,6 @@
 #                            SERVICE INPUT PLUGINS                            #
 ###############################################################################
 
-# # A Github Webhook Event collector
-# [[inputs.github_webhooks]]
-#   ## Address and port to host Webhook listener on
-#   service_address = ":1618"
-
-
 # # Read metrics from Kafka topic(s)
 # [[inputs.kafka_consumer]]
 #   ## topic(s) to consume
@@ -1524,7 +1631,7 @@
 #   ##   /var/log/**.log     -> recursively find all .log files in /var/log
 #   ##   /var/log/*/*.log    -> find all .log files with a parent dir in /var/log
 #   ##   /var/log/apache.log -> only tail the apache log file
-#   files = ["/var/log/influxdb/influxdb.log"]
+#   files = ["/var/log/apache/access.log"]
 #   ## Read file from beginning.
 #   from_beginning = false
 #
@@ -1537,7 +1644,9 @@
 #     ## Other common built-in patterns are:
 #     ##   %{COMMON_LOG_FORMAT}   (plain apache & nginx access logs)
 #     ##   %{COMBINED_LOG_FORMAT} (access logs + referrer & agent)
-#     patterns = ["%{INFLUXDB_HTTPD_LOG}"]
+#     patterns = ["%{COMBINED_LOG_FORMAT}"]
+#     ## Name of the outputted measurement name.
+#     measurement = "apache_access_log"
 #     ## Full path(s) to custom pattern files.
 #     custom_pattern_files = []
 #     ## Custom patterns can also be defined here. Put one pattern per line.
@@ -1601,10 +1710,19 @@
 #   data_format = "influx"
 
 
-# # A Rollbar Webhook Event collector
-# [[inputs.rollbar_webhooks]]
-#   ## Address and port to host Webhook listener on
-#   service_address = ":1619"
+# # Read NSQ topic for metrics.
+# [[inputs.nsq_consumer]]
+#   ## An string representing the NSQD TCP Endpoint
+#   server = "localhost:4150"
+#   topic = "telegraf"
+#   channel = "consumer"
+#   max_in_flight = 100
+#
+#   ## Data format to consume.
+#   ## Each data format has it's own unique set of configuration options, read
+#   ## more about them here:
+#   ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md
+#   data_format = "influx"
 
 
 # # Statsd Server
@@ -1701,3 +1819,18 @@
 #   ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md
 #   data_format = "influx"
 
+
+# # A Webhooks Event collector
+# [[inputs.webhooks]]
+#   ## Address and port to host Webhook listener on
+#   service_address = ":1619"
+#
+#   [inputs.webhooks.github]
+#     path = "/github"
+#
+#   [inputs.webhooks.mandrill]
+#     path = "/mandrill"
+#
+#   [inputs.webhooks.rollbar]
+#     path = "/rollbar"
+

internal/config/config.go

@@ -9,6 +9,7 @@ import (
 	"os"
 	"path/filepath"
 	"regexp"
+	"runtime"
 	"sort"
 	"strings"
 	"time"
@@ -47,8 +48,8 @@ type Config struct {
 	OutputFilters []string
 
 	Agent   *AgentConfig
-	Inputs  []*internal_models.RunningInput
-	Outputs []*internal_models.RunningOutput
+	Inputs  []*models.RunningInput
+	Outputs []*models.RunningOutput
 }
 
 func NewConfig() *Config {
@@ -61,8 +62,8 @@ func NewConfig() *Config {
 		},
 
 		Tags:          make(map[string]string),
-		Inputs:        make([]*internal_models.RunningInput, 0),
-		Outputs:       make([]*internal_models.RunningOutput, 0),
+		Inputs:        make([]*models.RunningInput, 0),
+		Outputs:       make([]*models.RunningOutput, 0),
 		InputFilters:  make([]string, 0),
 		OutputFilters: make([]string, 0),
 	}
@@ -139,7 +140,7 @@ func (c *Config) InputNames() []string {
 	return name
 }
 
-// Outputs returns a list of strings of the configured inputs.
+// Outputs returns a list of strings of the configured outputs.
 func (c *Config) OutputNames() []string {
 	var name []string
 	for _, output := range c.Outputs {
@@ -219,7 +220,7 @@ var header = `# Telegraf Configuration
   ## By default, precision will be set to the same timestamp order as the
   ## collection interval, with the maximum being 1s.
   ## Precision will NOT be used for service inputs, such as logparser and statsd.
-  ## Valid values are "Nns", "Nus" (or "Nµs"), "Nms", "Ns".
+  ## Valid values are "ns", "us" (or "µs"), "ms", "s".
   precision = ""
   ## Run telegraf in debug mode
   debug = false
@@ -432,6 +433,9 @@ func getDefaultConfigPath() (string, error) {
 	envfile := os.Getenv("TELEGRAF_CONFIG_PATH")
 	homefile := os.ExpandEnv("${HOME}/.telegraf/telegraf.conf")
 	etcfile := "/etc/telegraf/telegraf.conf"
+	if runtime.GOOS == "windows" {
+		etcfile = `C:\Program Files\Telegraf\telegraf.conf`
+	}
 	for _, path := range []string{envfile, homefile, etcfile} {
 		if _, err := os.Stat(path); err == nil {
 			log.Printf("Using config file: %s", path)
@@ -539,6 +543,13 @@ func (c *Config) LoadConfig(path string) error {
 	return nil
 }
 
+// trimBOM trims the Byte-Order-Marks from the beginning of the file.
+// this is for Windows compatability only.
+// see https://github.com/influxdata/telegraf/issues/1378
+func trimBOM(f []byte) []byte {
+	return bytes.TrimPrefix(f, []byte("\xef\xbb\xbf"))
+}
+
 // parseFile loads a TOML configuration from a provided path and
 // returns the AST produced from the TOML parser. When loading the file, it
 // will find environment variables and replace them.
@@ -547,6 +558,8 @@ func parseFile(fpath string) (*ast.Table, error) {
 	if err != nil {
 		return nil, err
 	}
+	// ugh windows why
+	contents = trimBOM(contents)
 
 	env_vars := envVarRe.FindAll(contents, -1)
 	for _, env_var := range env_vars {
@@ -589,7 +602,7 @@ func (c *Config) addOutput(name string, table *ast.Table) error {
 		return err
 	}
 
-	ro := internal_models.NewRunningOutput(name, output, outputConfig,
+	ro := models.NewRunningOutput(name, output, outputConfig,
 		c.Agent.MetricBatchSize, c.Agent.MetricBufferLimit)
 	c.Outputs = append(c.Outputs, ro)
 	return nil
@@ -630,7 +643,7 @@ func (c *Config) addInput(name string, table *ast.Table) error {
 		return err
 	}
 
-	rp := &internal_models.RunningInput{
+	rp := &models.RunningInput{
 		Name:   name,
 		Input:  input,
 		Config: pluginConfig,
@@ -641,10 +654,10 @@ func (c *Config) addInput(name string, table *ast.Table) error {
 
 // buildFilter builds a Filter
 // (tagpass/tagdrop/namepass/namedrop/fieldpass/fielddrop) to
-// be inserted into the internal_models.OutputConfig/internal_models.InputConfig
+// be inserted into the models.OutputConfig/models.InputConfig
 // to be used for glob filtering on tags and measurements
-func buildFilter(tbl *ast.Table) (internal_models.Filter, error) {
-	f := internal_models.Filter{}
+func buildFilter(tbl *ast.Table) (models.Filter, error) {
+	f := models.Filter{}
 
 	if node, ok := tbl.Fields["namepass"]; ok {
 		if kv, ok := node.(*ast.KeyValue); ok {
@@ -708,7 +721,7 @@ func buildFilter(tbl *ast.Table) (internal_models.Filter, error) {
 		if subtbl, ok := node.(*ast.Table); ok {
 			for name, val := range subtbl.Fields {
 				if kv, ok := val.(*ast.KeyValue); ok {
-					tagfilter := &internal_models.TagFilter{Name: name}
+					tagfilter := &models.TagFilter{Name: name}
 					if ary, ok := kv.Value.(*ast.Array); ok {
 						for _, elem := range ary.Value {
 							if str, ok := elem.(*ast.String); ok {
@@ -727,7 +740,7 @@ func buildFilter(tbl *ast.Table) (internal_models.Filter, error) {
 		if subtbl, ok := node.(*ast.Table); ok {
 			for name, val := range subtbl.Fields {
 				if kv, ok := val.(*ast.KeyValue); ok {
-					tagfilter := &internal_models.TagFilter{Name: name}
+					tagfilter := &models.TagFilter{Name: name}
 					if ary, ok := kv.Value.(*ast.Array); ok {
 						for _, elem := range ary.Value {
 							if str, ok := elem.(*ast.String); ok {
@@ -784,9 +797,9 @@ func buildFilter(tbl *ast.Table) (internal_models.Filter, error) {
 
 // buildInput parses input specific items from the ast.Table,
 // builds the filter and returns a
-// internal_models.InputConfig to be inserted into internal_models.RunningInput
-func buildInput(name string, tbl *ast.Table) (*internal_models.InputConfig, error) {
-	cp := &internal_models.InputConfig{Name: name}
+// models.InputConfig to be inserted into models.RunningInput
+func buildInput(name string, tbl *ast.Table) (*models.InputConfig, error) {
+	cp := &models.InputConfig{Name: name}
 	if node, ok := tbl.Fields["interval"]; ok {
 		if kv, ok := node.(*ast.KeyValue); ok {
 			if str, ok := kv.Value.(*ast.String); ok {
@@ -960,14 +973,14 @@ func buildSerializer(name string, tbl *ast.Table) (serializers.Serializer, error
 
 // buildOutput parses output specific items from the ast.Table,
 // builds the filter and returns an
-// internal_models.OutputConfig to be inserted into internal_models.RunningInput
+// models.OutputConfig to be inserted into models.RunningInput
 // Note: error exists in the return for future calls that might require error
-func buildOutput(name string, tbl *ast.Table) (*internal_models.OutputConfig, error) {
+func buildOutput(name string, tbl *ast.Table) (*models.OutputConfig, error) {
 	filter, err := buildFilter(tbl)
 	if err != nil {
 		return nil, err
 	}
-	oc := &internal_models.OutputConfig{
+	oc := &models.OutputConfig{
 		Name:   name,
 		Filter: filter,
 	}

internal/config/config_test.go

@@ -26,19 +26,19 @@ func TestConfig_LoadSingleInputWithEnvVars(t *testing.T) {
 	memcached := inputs.Inputs["memcached"]().(*memcached.Memcached)
 	memcached.Servers = []string{"192.168.1.1"}
 
-	filter := internal_models.Filter{
+	filter := models.Filter{
 		NameDrop:  []string{"metricname2"},
 		NamePass:  []string{"metricname1"},
 		FieldDrop: []string{"other", "stuff"},
 		FieldPass: []string{"some", "strings"},
-		TagDrop: []internal_models.TagFilter{
-			internal_models.TagFilter{
+		TagDrop: []models.TagFilter{
+			models.TagFilter{
 				Name:   "badtag",
 				Filter: []string{"othertag"},
 			},
 		},
-		TagPass: []internal_models.TagFilter{
-			internal_models.TagFilter{
+		TagPass: []models.TagFilter{
+			models.TagFilter{
 				Name:   "goodtag",
 				Filter: []string{"mytag"},
 			},
@@ -46,7 +46,7 @@ func TestConfig_LoadSingleInputWithEnvVars(t *testing.T) {
 		IsActive: true,
 	}
 	assert.NoError(t, filter.CompileFilter())
-	mConfig := &internal_models.InputConfig{
+	mConfig := &models.InputConfig{
 		Name:     "memcached",
 		Filter:   filter,
 		Interval: 10 * time.Second,
@@ -66,19 +66,19 @@ func TestConfig_LoadSingleInput(t *testing.T) {
 	memcached := inputs.Inputs["memcached"]().(*memcached.Memcached)
 	memcached.Servers = []string{"localhost"}
 
-	filter := internal_models.Filter{
+	filter := models.Filter{
 		NameDrop:  []string{"metricname2"},
 		NamePass:  []string{"metricname1"},
 		FieldDrop: []string{"other", "stuff"},
 		FieldPass: []string{"some", "strings"},
-		TagDrop: []internal_models.TagFilter{
-			internal_models.TagFilter{
+		TagDrop: []models.TagFilter{
+			models.TagFilter{
 				Name:   "badtag",
 				Filter: []string{"othertag"},
 			},
 		},
-		TagPass: []internal_models.TagFilter{
-			internal_models.TagFilter{
+		TagPass: []models.TagFilter{
+			models.TagFilter{
 				Name:   "goodtag",
 				Filter: []string{"mytag"},
 			},
@@ -86,7 +86,7 @@ func TestConfig_LoadSingleInput(t *testing.T) {
 		IsActive: true,
 	}
 	assert.NoError(t, filter.CompileFilter())
-	mConfig := &internal_models.InputConfig{
+	mConfig := &models.InputConfig{
 		Name:     "memcached",
 		Filter:   filter,
 		Interval: 5 * time.Second,
@@ -113,19 +113,19 @@ func TestConfig_LoadDirectory(t *testing.T) {
 	memcached := inputs.Inputs["memcached"]().(*memcached.Memcached)
 	memcached.Servers = []string{"localhost"}
 
-	filter := internal_models.Filter{
+	filter := models.Filter{
 		NameDrop:  []string{"metricname2"},
 		NamePass:  []string{"metricname1"},
 		FieldDrop: []string{"other", "stuff"},
 		FieldPass: []string{"some", "strings"},
-		TagDrop: []internal_models.TagFilter{
-			internal_models.TagFilter{
+		TagDrop: []models.TagFilter{
+			models.TagFilter{
 				Name:   "badtag",
 				Filter: []string{"othertag"},
 			},
 		},
-		TagPass: []internal_models.TagFilter{
-			internal_models.TagFilter{
+		TagPass: []models.TagFilter{
+			models.TagFilter{
 				Name:   "goodtag",
 				Filter: []string{"mytag"},
 			},
@@ -133,7 +133,7 @@ func TestConfig_LoadDirectory(t *testing.T) {
 		IsActive: true,
 	}
 	assert.NoError(t, filter.CompileFilter())
-	mConfig := &internal_models.InputConfig{
+	mConfig := &models.InputConfig{
 		Name:     "memcached",
 		Filter:   filter,
 		Interval: 5 * time.Second,
@@ -150,7 +150,7 @@ func TestConfig_LoadDirectory(t *testing.T) {
 	assert.NoError(t, err)
 	ex.SetParser(p)
 	ex.Command = "/usr/bin/myothercollector --foo=bar"
-	eConfig := &internal_models.InputConfig{
+	eConfig := &models.InputConfig{
 		Name:              "exec",
 		MeasurementSuffix: "_myothercollector",
 	}
@@ -169,7 +169,7 @@ func TestConfig_LoadDirectory(t *testing.T) {
 	pstat := inputs.Inputs["procstat"]().(*procstat.Procstat)
 	pstat.PidFile = "/var/run/grafana-server.pid"
 
-	pConfig := &internal_models.InputConfig{Name: "procstat"}
+	pConfig := &models.InputConfig{Name: "procstat"}
 	pConfig.Tags = make(map[string]string)
 
 	assert.Equal(t, pstat, c.Inputs[3].Input,

internal/internal.go

@@ -133,8 +133,8 @@ func GetTLSConfig(
 		cert, err := tls.LoadX509KeyPair(SSLCert, SSLKey)
 		if err != nil {
 			return nil, errors.New(fmt.Sprintf(
-				"Could not load TLS client key/certificate: %s",
-				err))
+				"Could not load TLS client key/certificate from %s:%s: %s",
+				SSLKey, SSLCert, err))
 		}
 
 		t.Certificates = []tls.Certificate{cert}

internal/models/filter.go

@@ -1,4 +1,4 @@
-package internal_models
+package models
 
 import (
 	"fmt"

internal/models/filter_test.go

@@ -1,4 +1,4 @@
-package internal_models
+package models
 
 import (
 	"testing"

internal/models/running_input.go

@@ -1,4 +1,4 @@
-package internal_models
+package models
 
 import (
 	"time"

internal/models/running_output.go

@@ -1,4 +1,4 @@
-package internal_models
+package models
 
 import (
 	"log"
@@ -138,7 +138,7 @@ func (ro *RunningOutput) Write() error {
 }
 
 func (ro *RunningOutput) write(metrics []telegraf.Metric) error {
-	if len(metrics) == 0 {
+	if metrics == nil || len(metrics) == 0 {
 		return nil
 	}
 	start := time.Now()

internal/models/running_output_test.go

@@ -1,4 +1,4 @@
-package internal_models
+package models
 
 import (
 	"fmt"

plugins/inputs/EXAMPLE_README.md

@@ -27,6 +27,14 @@ The example plugin gathers metrics about example things
     - tag2
 - measurement2 has the following tags:
     - tag3
+    
+### Sample Queries:
+
+These are some useful queries (to generate dashboards or other) to run against data from this plugin:
+
+```
+SELECT max(field1), mean(field1), min(field1) FROM measurement1 WHERE tag1=bar AND time > now() - 1h GROUP BY tag
+```
 
 ### Example Output:
 

plugins/inputs/aerospike/aerospike.go

@@ -1,104 +1,19 @@
 package aerospike
 
 import (
-	"bytes"
-	"encoding/binary"
-	"fmt"
-	"github.com/influxdata/telegraf"
-	"github.com/influxdata/telegraf/plugins/inputs"
 	"net"
 	"strconv"
 	"strings"
 	"sync"
+	"time"
+
+	"github.com/influxdata/telegraf"
+	"github.com/influxdata/telegraf/internal/errchan"
+	"github.com/influxdata/telegraf/plugins/inputs"
+
+	as "github.com/sparrc/aerospike-client-go"
 )
 
-const (
-	MSG_HEADER_SIZE = 8
-	MSG_TYPE        = 1 // Info is 1
-	MSG_VERSION     = 2
-)
-
-var (
-	STATISTICS_COMMAND = []byte("statistics\n")
-	NAMESPACES_COMMAND = []byte("namespaces\n")
-)
-
-type aerospikeMessageHeader struct {
-	Version uint8
-	Type    uint8
-	DataLen [6]byte
-}
-
-type aerospikeMessage struct {
-	aerospikeMessageHeader
-	Data []byte
-}
-
-// Taken from aerospike-client-go/types/message.go
-func (msg *aerospikeMessage) Serialize() []byte {
-	msg.DataLen = msgLenToBytes(int64(len(msg.Data)))
-	buf := bytes.NewBuffer([]byte{})
-	binary.Write(buf, binary.BigEndian, msg.aerospikeMessageHeader)
-	binary.Write(buf, binary.BigEndian, msg.Data[:])
-	return buf.Bytes()
-}
-
-type aerospikeInfoCommand struct {
-	msg *aerospikeMessage
-}
-
-// Taken from aerospike-client-go/info.go
-func (nfo *aerospikeInfoCommand) parseMultiResponse() (map[string]string, error) {
-	responses := make(map[string]string)
-	offset := int64(0)
-	begin := int64(0)
-
-	dataLen := int64(len(nfo.msg.Data))
-
-	// Create reusable StringBuilder for performance.
-	for offset < dataLen {
-		b := nfo.msg.Data[offset]
-
-		if b == '\t' {
-			name := nfo.msg.Data[begin:offset]
-			offset++
-			begin = offset
-
-			// Parse field value.
-			for offset < dataLen {
-				if nfo.msg.Data[offset] == '\n' {
-					break
-				}
-				offset++
-			}
-
-			if offset > begin {
-				value := nfo.msg.Data[begin:offset]
-				responses[string(name)] = string(value)
-			} else {
-				responses[string(name)] = ""
-			}
-			offset++
-			begin = offset
-		} else if b == '\n' {
-			if offset > begin {
-				name := nfo.msg.Data[begin:offset]
-				responses[string(name)] = ""
-			}
-			offset++
-			begin = offset
-		} else {
-			offset++
-		}
-	}
-
-	if offset > begin {
-		name := nfo.msg.Data[begin:offset]
-		responses[string(name)] = ""
-	}
-	return responses, nil
-}
-
 type Aerospike struct {
 	Servers []string
 }
@@ -115,7 +30,7 @@ func (a *Aerospike) SampleConfig() string {
 }
 
 func (a *Aerospike) Description() string {
-	return "Read stats from an aerospike server"
+	return "Read stats from aerospike server(s)"
 }
 
 func (a *Aerospike) Gather(acc telegraf.Accumulator) error {
@@ -124,214 +39,101 @@ func (a *Aerospike) Gather(acc telegraf.Accumulator) error {
 	}
 
 	var wg sync.WaitGroup
-
-	var outerr error
-
+	errChan := errchan.New(len(a.Servers))
+	wg.Add(len(a.Servers))
 	for _, server := range a.Servers {
-		wg.Add(1)
-		go func(server string) {
+		go func(serv string) {
 			defer wg.Done()
-			outerr = a.gatherServer(server, acc)
+			errChan.C <- a.gatherServer(serv, acc)
 		}(server)
 	}
 
 	wg.Wait()
-	return outerr
+	return errChan.Error()
 }
 
-func (a *Aerospike) gatherServer(host string, acc telegraf.Accumulator) error {
-	aerospikeInfo, err := getMap(STATISTICS_COMMAND, host)
+func (a *Aerospike) gatherServer(hostport string, acc telegraf.Accumulator) error {
+	host, port, err := net.SplitHostPort(hostport)
 	if err != nil {
-		return fmt.Errorf("Aerospike info failed: %s", err)
+		return err
 	}
-	readAerospikeStats(aerospikeInfo, acc, host, "")
-	namespaces, err := getList(NAMESPACES_COMMAND, host)
+
+	iport, err := strconv.Atoi(port)
 	if err != nil {
-		return fmt.Errorf("Aerospike namespace list failed: %s", err)
+		iport = 3000
 	}
-	for ix := range namespaces {
-		nsInfo, err := getMap([]byte("namespace/"+namespaces[ix]+"\n"), host)
-		if err != nil {
-			return fmt.Errorf("Aerospike namespace '%s' query failed: %s", namespaces[ix], err)
+
+	c, err := as.NewClient(host, iport)
+	if err != nil {
+		return err
+	}
+	defer c.Close()
+
+	nodes := c.GetNodes()
+	for _, n := range nodes {
+		tags := map[string]string{
+			"aerospike_host": hostport,
+		}
+		fields := map[string]interface{}{
+			"node_name": n.GetName(),
+		}
+		stats, err := as.RequestNodeStats(n)
+		if err != nil {
+			return err
+		}
+		for k, v := range stats {
+			fields[strings.Replace(k, "-", "_", -1)] = parseValue(v)
+		}
+		acc.AddFields("aerospike_node", fields, tags, time.Now())
+
+		info, err := as.RequestNodeInfo(n, "namespaces")
+		if err != nil {
+			return err
+		}
+		namespaces := strings.Split(info["namespaces"], ";")
+
+		for _, namespace := range namespaces {
+			nTags := map[string]string{
+				"aerospike_host": hostport,
+			}
+			nTags["namespace"] = namespace
+			nFields := map[string]interface{}{
+				"node_name": n.GetName(),
+			}
+			info, err := as.RequestNodeInfo(n, "namespace/"+namespace)
+			if err != nil {
+				continue
+			}
+			stats := strings.Split(info["namespace/"+namespace], ";")
+			for _, stat := range stats {
+				parts := strings.Split(stat, "=")
+				if len(parts) < 2 {
+					continue
+				}
+				nFields[strings.Replace(parts[0], "-", "_", -1)] = parseValue(parts[1])
+			}
+			acc.AddFields("aerospike_namespace", nFields, nTags, time.Now())
 		}
-		readAerospikeStats(nsInfo, acc, host, namespaces[ix])
 	}
 	return nil
 }
 
-func getMap(key []byte, host string) (map[string]string, error) {
-	data, err := get(key, host)
-	if err != nil {
-		return nil, fmt.Errorf("Failed to get data: %s", err)
+func parseValue(v string) interface{} {
+	if parsed, err := strconv.ParseInt(v, 10, 64); err == nil {
+		return parsed
+	} else if parsed, err := strconv.ParseBool(v); err == nil {
+		return parsed
+	} else {
+		return v
 	}
-	parsed, err := unmarshalMapInfo(data, string(key))
-	if err != nil {
-		return nil, fmt.Errorf("Failed to unmarshal data: %s", err)
-	}
-
-	return parsed, nil
 }
 
-func getList(key []byte, host string) ([]string, error) {
-	data, err := get(key, host)
-	if err != nil {
-		return nil, fmt.Errorf("Failed to get data: %s", err)
+func copyTags(m map[string]string) map[string]string {
+	out := make(map[string]string)
+	for k, v := range m {
+		out[k] = v
 	}
-	parsed, err := unmarshalListInfo(data, string(key))
-	if err != nil {
-		return nil, fmt.Errorf("Failed to unmarshal data: %s", err)
-	}
-
-	return parsed, nil
-}
-
-func get(key []byte, host string) (map[string]string, error) {
-	var err error
-	var data map[string]string
-
-	asInfo := &aerospikeInfoCommand{
-		msg: &aerospikeMessage{
-			aerospikeMessageHeader: aerospikeMessageHeader{
-				Version: uint8(MSG_VERSION),
-				Type:    uint8(MSG_TYPE),
-				DataLen: msgLenToBytes(int64(len(key))),
-			},
-			Data: key,
-		},
-	}
-
-	cmd := asInfo.msg.Serialize()
-	addr, err := net.ResolveTCPAddr("tcp", host)
-	if err != nil {
-		return data, fmt.Errorf("Lookup failed for '%s': %s", host, err)
-	}
-
-	conn, err := net.DialTCP("tcp", nil, addr)
-	if err != nil {
-		return data, fmt.Errorf("Connection failed for '%s': %s", host, err)
-	}
-	defer conn.Close()
-
-	_, err = conn.Write(cmd)
-	if err != nil {
-		return data, fmt.Errorf("Failed to send to '%s': %s", host, err)
-	}
-
-	msgHeader := bytes.NewBuffer(make([]byte, MSG_HEADER_SIZE))
-	_, err = readLenFromConn(conn, msgHeader.Bytes(), MSG_HEADER_SIZE)
-	if err != nil {
-		return data, fmt.Errorf("Failed to read header: %s", err)
-	}
-	err = binary.Read(msgHeader, binary.BigEndian, &asInfo.msg.aerospikeMessageHeader)
-	if err != nil {
-		return data, fmt.Errorf("Failed to unmarshal header: %s", err)
-	}
-
-	msgLen := msgLenFromBytes(asInfo.msg.aerospikeMessageHeader.DataLen)
-
-	if int64(len(asInfo.msg.Data)) != msgLen {
-		asInfo.msg.Data = make([]byte, msgLen)
-	}
-
-	_, err = readLenFromConn(conn, asInfo.msg.Data, len(asInfo.msg.Data))
-	if err != nil {
-		return data, fmt.Errorf("Failed to read from connection to '%s': %s", host, err)
-	}
-
-	data, err = asInfo.parseMultiResponse()
-	if err != nil {
-		return data, fmt.Errorf("Failed to parse response from '%s': %s", host, err)
-	}
-
-	return data, err
-}
-
-func readAerospikeStats(
-	stats map[string]string,
-	acc telegraf.Accumulator,
-	host string,
-	namespace string,
-) {
-	fields := make(map[string]interface{})
-	tags := map[string]string{
-		"aerospike_host": host,
-		"namespace":      "_service",
-	}
-
-	if namespace != "" {
-		tags["namespace"] = namespace
-	}
-	for key, value := range stats {
-		// We are going to ignore all string based keys
-		val, err := strconv.ParseInt(value, 10, 64)
-		if err == nil {
-			if strings.Contains(key, "-") {
-				key = strings.Replace(key, "-", "_", -1)
-			}
-			fields[key] = val
-		}
-	}
-	acc.AddFields("aerospike", fields, tags)
-}
-
-func unmarshalMapInfo(infoMap map[string]string, key string) (map[string]string, error) {
-	key = strings.TrimSuffix(key, "\n")
-	res := map[string]string{}
-
-	v, exists := infoMap[key]
-	if !exists {
-		return res, fmt.Errorf("Key '%s' missing from info", key)
-	}
-
-	values := strings.Split(v, ";")
-	for i := range values {
-		kv := strings.Split(values[i], "=")
-		if len(kv) > 1 {
-			res[kv[0]] = kv[1]
-		}
-	}
-
-	return res, nil
-}
-
-func unmarshalListInfo(infoMap map[string]string, key string) ([]string, error) {
-	key = strings.TrimSuffix(key, "\n")
-
-	v, exists := infoMap[key]
-	if !exists {
-		return []string{}, fmt.Errorf("Key '%s' missing from info", key)
-	}
-
-	values := strings.Split(v, ";")
-	return values, nil
-}
-
-func readLenFromConn(c net.Conn, buffer []byte, length int) (total int, err error) {
-	var r int
-	for total < length {
-		r, err = c.Read(buffer[total:length])
-		total += r
-		if err != nil {
-			break
-		}
-	}
-	return
-}
-
-// Taken from aerospike-client-go/types/message.go
-func msgLenToBytes(DataLen int64) [6]byte {
-	b := make([]byte, 8)
-	binary.BigEndian.PutUint64(b, uint64(DataLen))
-	res := [6]byte{}
-	copy(res[:], b[2:])
-	return res
-}
-
-// Taken from aerospike-client-go/types/message.go
-func msgLenFromBytes(buf [6]byte) int64 {
-	nbytes := append([]byte{0, 0}, buf[:]...)
-	DataLen := binary.BigEndian.Uint64(nbytes)
-	return int64(DataLen)
+	return out
 }
 
 func init() {

plugins/inputs/aerospike/aerospike_test.go

@@ -1,7 +1,6 @@
 package aerospike
 
 import (
-	"reflect"
 	"testing"
 
 	"github.com/influxdata/telegraf/testutil"
@@ -23,96 +22,29 @@ func TestAerospikeStatistics(t *testing.T) {
 	err := a.Gather(&acc)
 	require.NoError(t, err)
 
-	// Only use a few of the metrics
-	asMetrics := []string{
-		"transactions",
-		"stat_write_errs",
-		"stat_read_reqs",
-		"stat_write_reqs",
-	}
-
-	for _, metric := range asMetrics {
-		assert.True(t, acc.HasIntField("aerospike", metric), metric)
-	}
-
+	assert.True(t, acc.HasMeasurement("aerospike_node"))
+	assert.True(t, acc.HasMeasurement("aerospike_namespace"))
+	assert.True(t, acc.HasIntField("aerospike_node", "batch_error"))
 }
 
-func TestAerospikeMsgLenFromToBytes(t *testing.T) {
-	var i int64 = 8
-	assert.True(t, i == msgLenFromBytes(msgLenToBytes(i)))
-}
+func TestAerospikeStatisticsPartialErr(t *testing.T) {
+	if testing.Short() {
+		t.Skip("Skipping integration test in short mode")
+	}
+
+	a := &Aerospike{
+		Servers: []string{
+			testutil.GetLocalHost() + ":3000",
+			testutil.GetLocalHost() + ":9999",
+		},
+	}
 
-func TestReadAerospikeStatsNoNamespace(t *testing.T) {
-	// Also test for re-writing
 	var acc testutil.Accumulator
-	stats := map[string]string{
-		"stat-write-errs": "12345",
-		"stat_read_reqs":  "12345",
-	}
-	readAerospikeStats(stats, &acc, "host1", "")
 
-	fields := map[string]interface{}{
-		"stat_write_errs": int64(12345),
-		"stat_read_reqs":  int64(12345),
-	}
-	tags := map[string]string{
-		"aerospike_host": "host1",
-		"namespace":      "_service",
-	}
-	acc.AssertContainsTaggedFields(t, "aerospike", fields, tags)
-}
-
-func TestReadAerospikeStatsNamespace(t *testing.T) {
-	var acc testutil.Accumulator
-	stats := map[string]string{
-		"stat_write_errs": "12345",
-		"stat_read_reqs":  "12345",
-	}
-	readAerospikeStats(stats, &acc, "host1", "test")
-
-	fields := map[string]interface{}{
-		"stat_write_errs": int64(12345),
-		"stat_read_reqs":  int64(12345),
-	}
-	tags := map[string]string{
-		"aerospike_host": "host1",
-		"namespace":      "test",
-	}
-	acc.AssertContainsTaggedFields(t, "aerospike", fields, tags)
-}
-
-func TestAerospikeUnmarshalList(t *testing.T) {
-	i := map[string]string{
-		"test": "one;two;three",
-	}
-
-	expected := []string{"one", "two", "three"}
-
-	list, err := unmarshalListInfo(i, "test2")
-	assert.True(t, err != nil)
-
-	list, err = unmarshalListInfo(i, "test")
-	assert.True(t, err == nil)
-	equal := true
-	for ix := range expected {
-		if list[ix] != expected[ix] {
-			equal = false
-			break
-		}
-	}
-	assert.True(t, equal)
-}
-
-func TestAerospikeUnmarshalMap(t *testing.T) {
-	i := map[string]string{
-		"test": "key1=value1;key2=value2",
-	}
-
-	expected := map[string]string{
-		"key1": "value1",
-		"key2": "value2",
-	}
-	m, err := unmarshalMapInfo(i, "test")
-	assert.True(t, err == nil)
-	assert.True(t, reflect.DeepEqual(m, expected))
+	err := a.Gather(&acc)
+	require.Error(t, err)
+
+	assert.True(t, acc.HasMeasurement("aerospike_node"))
+	assert.True(t, acc.HasMeasurement("aerospike_namespace"))
+	assert.True(t, acc.HasIntField("aerospike_node", "batch_error"))
 }

plugins/inputs/all/all.go

@@ -6,6 +6,7 @@ import (
 	_ "github.com/influxdata/telegraf/plugins/inputs/bcache"
 	_ "github.com/influxdata/telegraf/plugins/inputs/cassandra"
 	_ "github.com/influxdata/telegraf/plugins/inputs/ceph"
+	_ "github.com/influxdata/telegraf/plugins/inputs/cgroup"
 	_ "github.com/influxdata/telegraf/plugins/inputs/chrony"
 	_ "github.com/influxdata/telegraf/plugins/inputs/cloudwatch"
 	_ "github.com/influxdata/telegraf/plugins/inputs/conntrack"
@@ -19,9 +20,9 @@ import (
 	_ "github.com/influxdata/telegraf/plugins/inputs/elasticsearch"
 	_ "github.com/influxdata/telegraf/plugins/inputs/exec"
 	_ "github.com/influxdata/telegraf/plugins/inputs/filestat"
-	_ "github.com/influxdata/telegraf/plugins/inputs/github_webhooks"
 	_ "github.com/influxdata/telegraf/plugins/inputs/graylog"
 	_ "github.com/influxdata/telegraf/plugins/inputs/haproxy"
+	_ "github.com/influxdata/telegraf/plugins/inputs/hddtemp"
 	_ "github.com/influxdata/telegraf/plugins/inputs/http_response"
 	_ "github.com/influxdata/telegraf/plugins/inputs/httpjson"
 	_ "github.com/influxdata/telegraf/plugins/inputs/influxdb"
@@ -41,6 +42,7 @@ import (
 	_ "github.com/influxdata/telegraf/plugins/inputs/net_response"
 	_ "github.com/influxdata/telegraf/plugins/inputs/nginx"
 	_ "github.com/influxdata/telegraf/plugins/inputs/nsq"
+	_ "github.com/influxdata/telegraf/plugins/inputs/nsq_consumer"
 	_ "github.com/influxdata/telegraf/plugins/inputs/nstat"
 	_ "github.com/influxdata/telegraf/plugins/inputs/ntpq"
 	_ "github.com/influxdata/telegraf/plugins/inputs/passenger"
@@ -57,9 +59,9 @@ import (
 	_ "github.com/influxdata/telegraf/plugins/inputs/redis"
 	_ "github.com/influxdata/telegraf/plugins/inputs/rethinkdb"
 	_ "github.com/influxdata/telegraf/plugins/inputs/riak"
-	_ "github.com/influxdata/telegraf/plugins/inputs/rollbar_webhooks"
 	_ "github.com/influxdata/telegraf/plugins/inputs/sensors"
 	_ "github.com/influxdata/telegraf/plugins/inputs/snmp"
+	_ "github.com/influxdata/telegraf/plugins/inputs/snmp_legacy"
 	_ "github.com/influxdata/telegraf/plugins/inputs/sqlserver"
 	_ "github.com/influxdata/telegraf/plugins/inputs/statsd"
 	_ "github.com/influxdata/telegraf/plugins/inputs/sysstat"
@@ -70,6 +72,7 @@ import (
 	_ "github.com/influxdata/telegraf/plugins/inputs/twemproxy"
 	_ "github.com/influxdata/telegraf/plugins/inputs/udp_listener"
 	_ "github.com/influxdata/telegraf/plugins/inputs/varnish"
+	_ "github.com/influxdata/telegraf/plugins/inputs/webhooks"
 	_ "github.com/influxdata/telegraf/plugins/inputs/win_perf_counters"
 	_ "github.com/influxdata/telegraf/plugins/inputs/zfs"
 	_ "github.com/influxdata/telegraf/plugins/inputs/zookeeper"

plugins/inputs/apache/apache.go

@@ -8,7 +8,6 @@ import (
 	"net/url"
 	"strconv"
 	"strings"
-	"sync"
 	"time"
 
 	"github.com/influxdata/telegraf"
@@ -38,8 +37,8 @@ func (n *Apache) Gather(acc telegraf.Accumulator) error {
 		n.Urls = []string{"http://localhost/server-status?auto"}
 	}
 
-	var wg sync.WaitGroup
 	var outerr error
+	var errch = make(chan error)
 
 	for _, u := range n.Urls {
 		addr, err := url.Parse(u)
@@ -47,14 +46,17 @@ func (n *Apache) Gather(acc telegraf.Accumulator) error {
 			return fmt.Errorf("Unable to parse address '%s': %s", u, err)
 		}
 
-		wg.Add(1)
 		go func(addr *url.URL) {
-			defer wg.Done()
-			outerr = n.gatherUrl(addr, acc)
+			errch <- n.gatherUrl(addr, acc)
 		}(addr)
 	}
 
-	wg.Wait()
+	// Drain channel, waiting for all requests to finish and save last error.
+	for range n.Urls {
+		if err := <-errch; err != nil {
+			outerr = err
+		}
+	}
 
 	return outerr
 }

plugins/inputs/apache/apache_test.go

@@ -36,7 +36,8 @@ func TestHTTPApache(t *testing.T) {
 	defer ts.Close()
 
 	a := Apache{
-		Urls: []string{ts.URL},
+		// Fetch it 2 times to catch possible data races.
+		Urls: []string{ts.URL, ts.URL},
 	}
 
 	var acc testutil.Accumulator

plugins/inputs/cassandra/cassandra.go

@@ -148,7 +148,7 @@ func (c cassandraMetric) addTagsFields(out map[string]interface{}) {
 	tokens := parseJmxMetricRequest(r.(map[string]interface{})["mbean"].(string))
 	// Requests with wildcards for keyspace or table names will return nested
 	// maps in the json response
-	if tokens["type"] == "Table" && (tokens["keyspace"] == "*" ||
+	if (tokens["type"] == "Table" || tokens["type"] == "ColumnFamily") && (tokens["keyspace"] == "*" ||
 		tokens["scope"] == "*") {
 		if valuesMap, ok := out["value"]; ok {
 			for k, v := range valuesMap.(map[string]interface{}) {

plugins/inputs/ceph/README.md

@@ -1,18 +1,18 @@
 # Ceph Storage Input Plugin
 
-Collects performance metrics from the MON and OSD nodes in a Ceph storage cluster.  
+Collects performance metrics from the MON and OSD nodes in a Ceph storage cluster.
 
 The plugin works by scanning the configured SocketDir for OSD and MON socket files.  When it finds
-a MON socket, it runs **ceph --admin-daemon $file perfcounters_dump**. For OSDs it runs **ceph --admin-daemon $file perf dump** 
+a MON socket, it runs **ceph --admin-daemon $file perfcounters_dump**. For OSDs it runs **ceph --admin-daemon $file perf dump**
 
 The resulting JSON is parsed and grouped into collections, based on top-level key.  Top-level keys are
 used as collection tags, and all sub-keys are flattened. For example:
 
 ```
- { 
-   "paxos": { 
+ {
+   "paxos": {
      "refresh": 9363435,
-     "refresh_latency": { 
+     "refresh_latency": {
        "avgcount": 9363435,
        "sum": 5378.794002000
      }
@@ -50,7 +50,7 @@ Would be parsed into the following metrics, all of which would be tagged with co
 
 ### Measurements & Fields:
 
-All fields are collected under the **ceph** measurement and stored as float64s. For a full list of fields, see the sample perf dumps in ceph_test.go. 
+All fields are collected under the **ceph** measurement and stored as float64s. For a full list of fields, see the sample perf dumps in ceph_test.go.
 
 
 ### Tags:
@@ -95,7 +95,7 @@ All measurements will have the following tags:
     - throttle-objecter_ops
     - throttle-osd_client_bytes
     - throttle-osd_client_messages
- 
+
 
 ### Example Output:
 

plugins/inputs/cgroup/README.md

@@ -0,0 +1,59 @@
+# CGroup Input Plugin For Telegraf Agent
+
+This input plugin will capture specific statistics per cgroup.
+
+Following file formats are supported:
+
+* Single value
+
+```
+VAL\n
+```
+
+* New line separated values
+
+```
+VAL0\n
+VAL1\n
+```
+
+* Space separated values
+
+```
+VAL0 VAL1 ...\n
+```
+
+* New line separated key-space-value's
+
+```
+KEY0 VAL0\n
+KEY1 VAL1\n
+```
+
+
+### Tags:
+
+Measurements don't have any specific tags unless you define them at the telegraf level (defaults). We
+used to have the path listed as a tag, but to keep cardinality in check it's easier to move this 
+value to a field. Thanks @sebito91!
+
+
+### Configuration:
+
+```
+# [[inputs.cgroup]]
+  # paths = [
+  #   "/cgroup/memory",           # root cgroup
+  #   "/cgroup/memory/child1",    # container cgroup
+  #   "/cgroup/memory/child2/*",  # all children cgroups under child2, but not child2 itself
+  # ]
+  # files = ["memory.*usage*", "memory.limit_in_bytes"]
+
+# [[inputs.cgroup]]
+  # paths = [
+  #   "/cgroup/cpu",              # root cgroup
+  #   "/cgroup/cpu/*",            # all container cgroups
+  #   "/cgroup/cpu/*/*",          # all children cgroups under each container cgroup
+  # ]
+  # files = ["cpuacct.usage", "cpu.cfs_period_us", "cpu.cfs_quota_us"]
+```

plugins/inputs/cgroup/cgroup.go

@@ -0,0 +1,35 @@
+package cgroup
+
+import (
+	"github.com/influxdata/telegraf"
+	"github.com/influxdata/telegraf/plugins/inputs"
+)
+
+type CGroup struct {
+	Paths []string `toml:"paths"`
+	Files []string `toml:"files"`
+}
+
+var sampleConfig = `
+    ## Directories in which to look for files, globs are supported.
+	# paths = [
+	#   "/cgroup/memory",
+	#   "/cgroup/memory/child1",
+	#   "/cgroup/memory/child2/*",
+	# ]
+	## cgroup stat fields, as file names, globs are supported.
+	## these file names are appended to each path from above.
+	# files = ["memory.*usage*", "memory.limit_in_bytes"]
+`
+
+func (g *CGroup) SampleConfig() string {
+	return sampleConfig
+}
+
+func (g *CGroup) Description() string {
+	return "Read specific statistics per cgroup"
+}
+
+func init() {
+	inputs.Add("cgroup", func() telegraf.Input { return &CGroup{} })
+}

plugins/inputs/cgroup/cgroup_linux.go

@@ -0,0 +1,243 @@
+// +build linux
+
+package cgroup
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path"
+	"path/filepath"
+	"regexp"
+	"strconv"
+
+	"github.com/influxdata/telegraf"
+)
+
+const metricName = "cgroup"
+
+func (g *CGroup) Gather(acc telegraf.Accumulator) error {
+	list := make(chan pathInfo)
+	go g.generateDirs(list)
+
+	for dir := range list {
+		if dir.err != nil {
+			return dir.err
+		}
+		if err := g.gatherDir(dir.path, acc); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (g *CGroup) gatherDir(dir string, acc telegraf.Accumulator) error {
+	fields := make(map[string]interface{})
+
+	list := make(chan pathInfo)
+	go g.generateFiles(dir, list)
+
+	for file := range list {
+		if file.err != nil {
+			return file.err
+		}
+
+		raw, err := ioutil.ReadFile(file.path)
+		if err != nil {
+			return err
+		}
+		if len(raw) == 0 {
+			continue
+		}
+
+		fd := fileData{data: raw, path: file.path}
+		if err := fd.parse(fields); err != nil {
+			return err
+		}
+	}
+	fields["path"] = dir
+
+	acc.AddFields(metricName, fields, nil)
+
+	return nil
+}
+
+// ======================================================================
+
+type pathInfo struct {
+	path string
+	err  error
+}
+
+func isDir(path string) (bool, error) {
+	result, err := os.Stat(path)
+	if err != nil {
+		return false, err
+	}
+	return result.IsDir(), nil
+}
+
+func (g *CGroup) generateDirs(list chan<- pathInfo) {
+	for _, dir := range g.Paths {
+		// getting all dirs that match the pattern 'dir'
+		items, err := filepath.Glob(dir)
+		if err != nil {
+			list <- pathInfo{err: err}
+			return
+		}
+
+		for _, item := range items {
+			ok, err := isDir(item)
+			if err != nil {
+				list <- pathInfo{err: err}
+				return
+			}
+			// supply only dirs
+			if ok {
+				list <- pathInfo{path: item}
+			}
+		}
+	}
+	close(list)
+}
+
+func (g *CGroup) generateFiles(dir string, list chan<- pathInfo) {
+	for _, file := range g.Files {
+		// getting all file paths that match the pattern 'dir + file'
+		// path.Base make sure that file variable does not contains part of path
+		items, err := filepath.Glob(path.Join(dir, path.Base(file)))
+		if err != nil {
+			list <- pathInfo{err: err}
+			return
+		}
+
+		for _, item := range items {
+			ok, err := isDir(item)
+			if err != nil {
+				list <- pathInfo{err: err}
+				return
+			}
+			// supply only files not dirs
+			if !ok {
+				list <- pathInfo{path: item}
+			}
+		}
+	}
+	close(list)
+}
+
+// ======================================================================
+
+type fileData struct {
+	data []byte
+	path string
+}
+
+func (fd *fileData) format() (*fileFormat, error) {
+	for _, ff := range fileFormats {
+		ok, err := ff.match(fd.data)
+		if err != nil {
+			return nil, err
+		}
+		if ok {
+			return &ff, nil
+		}
+	}
+
+	return nil, fmt.Errorf("%v: unknown file format", fd.path)
+}
+
+func (fd *fileData) parse(fields map[string]interface{}) error {
+	format, err := fd.format()
+	if err != nil {
+		return err
+	}
+
+	format.parser(filepath.Base(fd.path), fields, fd.data)
+	return nil
+}
+
+// ======================================================================
+
+type fileFormat struct {
+	name    string
+	pattern string
+	parser  func(measurement string, fields map[string]interface{}, b []byte)
+}
+
+const keyPattern = "[[:alpha:]_]+"
+const valuePattern = "[\\d-]+"
+
+var fileFormats = [...]fileFormat{
+	// 	VAL\n
+	fileFormat{
+		name:    "Single value",
+		pattern: "^" + valuePattern + "\n$",
+		parser: func(measurement string, fields map[string]interface{}, b []byte) {
+			re := regexp.MustCompile("^(" + valuePattern + ")\n$")
+			matches := re.FindAllStringSubmatch(string(b), -1)
+			fields[measurement] = numberOrString(matches[0][1])
+		},
+	},
+	// 	VAL0\n
+	// 	VAL1\n
+	// 	...
+	fileFormat{
+		name:    "New line separated values",
+		pattern: "^(" + valuePattern + "\n){2,}$",
+		parser: func(measurement string, fields map[string]interface{}, b []byte) {
+			re := regexp.MustCompile("(" + valuePattern + ")\n")
+			matches := re.FindAllStringSubmatch(string(b), -1)
+			for i, v := range matches {
+				fields[measurement+"."+strconv.Itoa(i)] = numberOrString(v[1])
+			}
+		},
+	},
+	// 	VAL0 VAL1 ...\n
+	fileFormat{
+		name:    "Space separated values",
+		pattern: "^(" + valuePattern + " )+\n$",
+		parser: func(measurement string, fields map[string]interface{}, b []byte) {
+			re := regexp.MustCompile("(" + valuePattern + ") ")
+			matches := re.FindAllStringSubmatch(string(b), -1)
+			for i, v := range matches {
+				fields[measurement+"."+strconv.Itoa(i)] = numberOrString(v[1])
+			}
+		},
+	},
+	// 	KEY0 VAL0\n
+	// 	KEY1 VAL1\n
+	// 	...
+	fileFormat{
+		name:    "New line separated key-space-value's",
+		pattern: "^(" + keyPattern + " " + valuePattern + "\n)+$",
+		parser: func(measurement string, fields map[string]interface{}, b []byte) {
+			re := regexp.MustCompile("(" + keyPattern + ") (" + valuePattern + ")\n")
+			matches := re.FindAllStringSubmatch(string(b), -1)
+			for _, v := range matches {
+				fields[measurement+"."+v[1]] = numberOrString(v[2])
+			}
+		},
+	},
+}
+
+func numberOrString(s string) interface{} {
+	i, err := strconv.Atoi(s)
+	if err == nil {
+		return i
+	}
+
+	return s
+}
+
+func (f fileFormat) match(b []byte) (bool, error) {
+	ok, err := regexp.Match(f.pattern, b)
+	if err != nil {
+		return false, err
+	}
+	if ok {
+		return true, nil
+	}
+	return false, nil
+}

plugins/inputs/cgroup/cgroup_notlinux.go

@@ -0,0 +1,11 @@
+// +build !linux
+
+package cgroup
+
+import (
+	"github.com/influxdata/telegraf"
+)
+
+func (g *CGroup) Gather(acc telegraf.Accumulator) error {
+	return nil
+}

plugins/inputs/cgroup/cgroup_test.go

@@ -0,0 +1,194 @@
+// +build linux
+
+package cgroup
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/influxdata/telegraf/testutil"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"reflect"
+)
+
+var cg1 = &CGroup{
+	Paths: []string{"testdata/memory"},
+	Files: []string{
+		"memory.empty",
+		"memory.max_usage_in_bytes",
+		"memory.limit_in_bytes",
+		"memory.stat",
+		"memory.use_hierarchy",
+		"notify_on_release",
+	},
+}
+
+func assertContainsFields(a *testutil.Accumulator, t *testing.T, measurement string, fieldSet []map[string]interface{}) {
+	a.Lock()
+	defer a.Unlock()
+
+	numEquals := 0
+	for _, p := range a.Metrics {
+		if p.Measurement == measurement {
+			for _, fields := range fieldSet {
+				if reflect.DeepEqual(fields, p.Fields) {
+					numEquals++
+				}
+			}
+		}
+	}
+
+	if numEquals != len(fieldSet) {
+		assert.Fail(t, fmt.Sprintf("only %d of %d are equal", numEquals, len(fieldSet)))
+	}
+}
+
+func TestCgroupStatistics_1(t *testing.T) {
+	var acc testutil.Accumulator
+
+	err := cg1.Gather(&acc)
+	require.NoError(t, err)
+
+	fields := map[string]interface{}{
+		"memory.stat.cache":           1739362304123123123,
+		"memory.stat.rss":             1775325184,
+		"memory.stat.rss_huge":        778043392,
+		"memory.stat.mapped_file":     421036032,
+		"memory.stat.dirty":           -307200,
+		"memory.max_usage_in_bytes.0": 0,
+		"memory.max_usage_in_bytes.1": -1,
+		"memory.max_usage_in_bytes.2": 2,
+		"memory.limit_in_bytes":       223372036854771712,
+		"memory.use_hierarchy":        "12-781",
+		"notify_on_release":           0,
+		"path":                        "testdata/memory",
+	}
+	assertContainsFields(&acc, t, "cgroup", []map[string]interface{}{fields})
+}
+
+// ======================================================================
+
+var cg2 = &CGroup{
+	Paths: []string{"testdata/cpu"},
+	Files: []string{"cpuacct.usage_percpu"},
+}
+
+func TestCgroupStatistics_2(t *testing.T) {
+	var acc testutil.Accumulator
+
+	err := cg2.Gather(&acc)
+	require.NoError(t, err)
+
+	fields := map[string]interface{}{
+		"cpuacct.usage_percpu.0": -1452543795404,
+		"cpuacct.usage_percpu.1": 1376681271659,
+		"cpuacct.usage_percpu.2": 1450950799997,
+		"cpuacct.usage_percpu.3": -1473113374257,
+		"path": "testdata/cpu",
+	}
+	assertContainsFields(&acc, t, "cgroup", []map[string]interface{}{fields})
+}
+
+// ======================================================================
+
+var cg3 = &CGroup{
+	Paths: []string{"testdata/memory/*"},
+	Files: []string{"memory.limit_in_bytes"},
+}
+
+func TestCgroupStatistics_3(t *testing.T) {
+	var acc testutil.Accumulator
+
+	err := cg3.Gather(&acc)
+	require.NoError(t, err)
+
+	fields := map[string]interface{}{
+		"memory.limit_in_bytes": 223372036854771712,
+		"path":                  "testdata/memory/group_1",
+	}
+
+	fieldsTwo := map[string]interface{}{
+		"memory.limit_in_bytes": 223372036854771712,
+		"path":                  "testdata/memory/group_2",
+	}
+	assertContainsFields(&acc, t, "cgroup", []map[string]interface{}{fields, fieldsTwo})
+}
+
+// ======================================================================
+
+var cg4 = &CGroup{
+	Paths: []string{"testdata/memory/*/*", "testdata/memory/group_2"},
+	Files: []string{"memory.limit_in_bytes"},
+}
+
+func TestCgroupStatistics_4(t *testing.T) {
+	var acc testutil.Accumulator
+
+	err := cg4.Gather(&acc)
+	require.NoError(t, err)
+
+	fields := map[string]interface{}{
+		"memory.limit_in_bytes": 223372036854771712,
+		"path":                  "testdata/memory/group_1/group_1_1",
+	}
+
+	fieldsTwo := map[string]interface{}{
+		"memory.limit_in_bytes": 223372036854771712,
+		"path":                  "testdata/memory/group_1/group_1_2",
+	}
+
+	fieldsThree := map[string]interface{}{
+		"memory.limit_in_bytes": 223372036854771712,
+		"path":                  "testdata/memory/group_2",
+	}
+
+	assertContainsFields(&acc, t, "cgroup", []map[string]interface{}{fields, fieldsTwo, fieldsThree})
+}
+
+// ======================================================================
+
+var cg5 = &CGroup{
+	Paths: []string{"testdata/memory/*/group_1_1"},
+	Files: []string{"memory.limit_in_bytes"},
+}
+
+func TestCgroupStatistics_5(t *testing.T) {
+	var acc testutil.Accumulator
+
+	err := cg5.Gather(&acc)
+	require.NoError(t, err)
+
+	fields := map[string]interface{}{
+		"memory.limit_in_bytes": 223372036854771712,
+		"path":                  "testdata/memory/group_1/group_1_1",
+	}
+
+	fieldsTwo := map[string]interface{}{
+		"memory.limit_in_bytes": 223372036854771712,
+		"path":                  "testdata/memory/group_2/group_1_1",
+	}
+	assertContainsFields(&acc, t, "cgroup", []map[string]interface{}{fields, fieldsTwo})
+}
+
+// ======================================================================
+
+var cg6 = &CGroup{
+	Paths: []string{"testdata/memory"},
+	Files: []string{"memory.us*", "*/memory.kmem.*"},
+}
+
+func TestCgroupStatistics_6(t *testing.T) {
+	var acc testutil.Accumulator
+
+	err := cg6.Gather(&acc)
+	require.NoError(t, err)
+
+	fields := map[string]interface{}{
+		"memory.usage_in_bytes":      3513667584,
+		"memory.use_hierarchy":       "12-781",
+		"memory.kmem.limit_in_bytes": 9223372036854771712,
+		"path": "testdata/memory",
+	}
+	assertContainsFields(&acc, t, "cgroup", []map[string]interface{}{fields})
+}

plugins/inputs/cgroup/testdata/blkio/blkio.io_serviced

@@ -0,0 +1 @@
+Total 0

plugins/inputs/cgroup/testdata/blkio/blkio.throttle.io_serviced

@@ -0,0 +1,131 @@
+11:0 Read 0
+11:0 Write 0
+11:0 Sync 0
+11:0 Async 0
+11:0 Total 0
+8:0 Read 49134
+8:0 Write 216703
+8:0 Sync 177906
+8:0 Async 87931
+8:0 Total 265837
+7:7 Read 0
+7:7 Write 0
+7:7 Sync 0
+7:7 Async 0
+7:7 Total 0
+7:6 Read 0
+7:6 Write 0
+7:6 Sync 0
+7:6 Async 0
+7:6 Total 0
+7:5 Read 0
+7:5 Write 0
+7:5 Sync 0
+7:5 Async 0
+7:5 Total 0
+7:4 Read 0
+7:4 Write 0
+7:4 Sync 0
+7:4 Async 0
+7:4 Total 0
+7:3 Read 0
+7:3 Write 0
+7:3 Sync 0
+7:3 Async 0
+7:3 Total 0
+7:2 Read 0
+7:2 Write 0
+7:2 Sync 0
+7:2 Async 0
+7:2 Total 0
+7:1 Read 0
+7:1 Write 0
+7:1 Sync 0
+7:1 Async 0
+7:1 Total 0
+7:0 Read 0
+7:0 Write 0
+7:0 Sync 0
+7:0 Async 0
+7:0 Total 0
+1:15 Read 3
+1:15 Write 0
+1:15 Sync 0
+1:15 Async 3
+1:15 Total 3
+1:14 Read 3
+1:14 Write 0
+1:14 Sync 0
+1:14 Async 3
+1:14 Total 3
+1:13 Read 3
+1:13 Write 0
+1:13 Sync 0
+1:13 Async 3
+1:13 Total 3
+1:12 Read 3
+1:12 Write 0
+1:12 Sync 0
+1:12 Async 3
+1:12 Total 3
+1:11 Read 3
+1:11 Write 0
+1:11 Sync 0
+1:11 Async 3
+1:11 Total 3
+1:10 Read 3
+1:10 Write 0
+1:10 Sync 0
+1:10 Async 3
+1:10 Total 3
+1:9 Read 3
+1:9 Write 0
+1:9 Sync 0
+1:9 Async 3
+1:9 Total 3
+1:8 Read 3
+1:8 Write 0
+1:8 Sync 0
+1:8 Async 3
+1:8 Total 3
+1:7 Read 3
+1:7 Write 0
+1:7 Sync 0
+1:7 Async 3
+1:7 Total 3
+1:6 Read 3
+1:6 Write 0
+1:6 Sync 0
+1:6 Async 3
+1:6 Total 3
+1:5 Read 3
+1:5 Write 0
+1:5 Sync 0
+1:5 Async 3
+1:5 Total 3
+1:4 Read 3
+1:4 Write 0
+1:4 Sync 0
+1:4 Async 3
+1:4 Total 3
+1:3 Read 3
+1:3 Write 0
+1:3 Sync 0
+1:3 Async 3
+1:3 Total 3
+1:2 Read 3
+1:2 Write 0
+1:2 Sync 0
+1:2 Async 3
+1:2 Total 3
+1:1 Read 3
+1:1 Write 0
+1:1 Sync 0
+1:1 Async 3
+1:1 Total 3
+1:0 Read 3
+1:0 Write 0
+1:0 Sync 0
+1:0 Async 3
+1:0 Total 3
+Total 265885

plugins/inputs/cgroup/testdata/cpu/cpu.cfs_quota_us

@@ -0,0 +1 @@
+-1

plugins/inputs/cgroup/testdata/cpu/cpuacct.usage_percpu

@@ -0,0 +1 @@
+-1452543795404 1376681271659 1450950799997 -1473113374257 

plugins/inputs/cgroup/testdata/memory/group_1/group_1_1/memory.limit_in_bytes

@@ -0,0 +1 @@
+223372036854771712

plugins/inputs/cgroup/testdata/memory/group_1/group_1_1/memory.stat

@@ -0,0 +1,5 @@
+cache 1739362304123123123
+rss 1775325184
+rss_huge 778043392
+mapped_file 421036032
+dirty -307200

plugins/inputs/cgroup/testdata/memory/group_1/group_1_2/memory.limit_in_bytes

@@ -0,0 +1 @@
+223372036854771712

plugins/inputs/cgroup/testdata/memory/group_1/group_1_2/memory.stat

@@ -0,0 +1,5 @@
+cache 1739362304123123123
+rss 1775325184
+rss_huge 778043392
+mapped_file 421036032
+dirty -307200

plugins/inputs/cgroup/testdata/memory/group_1/memory.kmem.limit_in_bytes

@@ -0,0 +1 @@
+9223372036854771712

plugins/inputs/cgroup/testdata/memory/group_1/memory.kmem.max_usage_in_bytes

@@ -0,0 +1 @@
+0

plugins/inputs/cgroup/testdata/memory/group_1/memory.limit_in_bytes

@@ -0,0 +1 @@
+223372036854771712

plugins/inputs/cgroup/testdata/memory/group_1/memory.stat

@@ -0,0 +1,5 @@
+cache 1739362304123123123
+rss 1775325184
+rss_huge 778043392
+mapped_file 421036032
+dirty -307200

plugins/inputs/cgroup/testdata/memory/group_2/group_1_1/memory.limit_in_bytes

@@ -0,0 +1 @@
+223372036854771712

plugins/inputs/cgroup/testdata/memory/group_2/group_1_1/memory.stat

@@ -0,0 +1,5 @@
+cache 1739362304123123123
+rss 1775325184
+rss_huge 778043392
+mapped_file 421036032
+dirty -307200

plugins/inputs/cgroup/testdata/memory/group_2/memory.limit_in_bytes

@@ -0,0 +1 @@
+223372036854771712

plugins/inputs/cgroup/testdata/memory/group_2/memory.stat

@@ -0,0 +1,5 @@
+cache 1739362304123123123
+rss 1775325184
+rss_huge 778043392
+mapped_file 421036032
+dirty -307200

plugins/inputs/cgroup/testdata/memory/memory.kmem.limit_in_bytes

@@ -0,0 +1 @@
+9223372036854771712

plugins/inputs/cgroup/testdata/memory/memory.limit_in_bytes

@@ -0,0 +1 @@
+223372036854771712

plugins/inputs/cgroup/testdata/memory/memory.max_usage_in_bytes

@@ -0,0 +1,3 @@
+0
+-1
+2

plugins/inputs/cgroup/testdata/memory/memory.numa_stat

@@ -0,0 +1,8 @@
+total=858067 N0=858067
+file=406254 N0=406254
+anon=451792 N0=451792
+unevictable=21 N0=21
+hierarchical_total=858067 N0=858067
+hierarchical_file=406254 N0=406254
+hierarchical_anon=451792 N0=451792
+hierarchical_unevictable=21 N0=21

plugins/inputs/cgroup/testdata/memory/memory.stat

@@ -0,0 +1,5 @@
+cache 1739362304123123123
+rss 1775325184
+rss_huge 778043392
+mapped_file 421036032
+dirty -307200

plugins/inputs/cgroup/testdata/memory/memory.usage_in_bytes

@@ -0,0 +1 @@
+3513667584

plugins/inputs/cgroup/testdata/memory/memory.use_hierarchy

@@ -0,0 +1 @@
+12-781

plugins/inputs/cgroup/testdata/memory/notify_on_release

@@ -0,0 +1 @@
+0

plugins/inputs/dns_query/dns_query.go

@@ -3,12 +3,14 @@ package dns_query
 import (
 	"errors"
 	"fmt"
-	"github.com/influxdata/telegraf"
-	"github.com/influxdata/telegraf/plugins/inputs"
 	"github.com/miekg/dns"
 	"net"
 	"strconv"
 	"time"
+
+	"github.com/influxdata/telegraf"
+	"github.com/influxdata/telegraf/internal/errchan"
+	"github.com/influxdata/telegraf/plugins/inputs"
 )
 
 type DnsQuery struct {
@@ -55,12 +57,12 @@ func (d *DnsQuery) Description() string {
 }
 func (d *DnsQuery) Gather(acc telegraf.Accumulator) error {
 	d.setDefaultValues()
+
+	errChan := errchan.New(len(d.Domains) * len(d.Servers))
 	for _, domain := range d.Domains {
 		for _, server := range d.Servers {
 			dnsQueryTime, err := d.getDnsQueryTime(domain, server)
-			if err != nil {
-				return err
-			}
+			errChan.C <- err
 			tags := map[string]string{
 				"server":      server,
 				"domain":      domain,
@@ -72,7 +74,7 @@ func (d *DnsQuery) Gather(acc telegraf.Accumulator) error {
 		}
 	}
 
-	return nil
+	return errChan.Error()
 }
 
 func (d *DnsQuery) setDefaultValues() {

plugins/inputs/docker/docker.go

@@ -25,6 +25,8 @@ type Docker struct {
 	Endpoint       string
 	ContainerNames []string
 	Timeout        internal.Duration
+	PerDevice      bool `toml:"perdevice"`
+	Total          bool `toml:"total"`
 
 	client DockerClient
 }
@@ -58,6 +60,13 @@ var sampleConfig = `
   container_names = []
   ## Timeout for docker list, info, and stats commands
   timeout = "5s"
+
+  ## Whether to report for each container per-device blkio (8:0, 8:1...) and
+  ## network (eth0, eth1, ...) stats or not
+  perdevice = true
+  ## Whether to report for each container total blkio and network stats or not
+  total = false
+
 `
 
 // Description returns input description
@@ -207,9 +216,18 @@ func (d *Docker) gatherContainer(
 		cname = strings.TrimPrefix(container.Names[0], "/")
 	}
 
+	// the image name sometimes has a version part.
+	//   ie, rabbitmq:3-management
+	imageParts := strings.Split(container.Image, ":")
+	imageName := imageParts[0]
+	imageVersion := "unknown"
+	if len(imageParts) > 1 {
+		imageVersion = imageParts[1]
+	}
 	tags := map[string]string{
-		"container_name":  cname,
-		"container_image": container.Image,
+		"container_name":    cname,
+		"container_image":   imageName,
+		"container_version": imageVersion,
 	}
 	if len(d.ContainerNames) > 0 {
 		if !sliceContains(cname, d.ContainerNames) {
@@ -237,7 +255,7 @@ func (d *Docker) gatherContainer(
 		tags[k] = label
 	}
 
-	gatherContainerStats(v, acc, tags, container.ID)
+	gatherContainerStats(v, acc, tags, container.ID, d.PerDevice, d.Total)
 
 	return nil
 }
@@ -247,6 +265,8 @@ func gatherContainerStats(
 	acc telegraf.Accumulator,
 	tags map[string]string,
 	id string,
+	perDevice bool,
+	total bool,
 ) {
 	now := stat.Read
 
@@ -314,6 +334,7 @@ func gatherContainerStats(
 		acc.AddFields("docker_container_cpu", fields, percputags, now)
 	}
 
+	totalNetworkStatMap := make(map[string]interface{})
 	for network, netstats := range stat.Networks {
 		netfields := map[string]interface{}{
 			"rx_dropped":   netstats.RxDropped,
@@ -327,12 +348,35 @@ func gatherContainerStats(
 			"container_id": id,
 		}
 		// Create a new network tag dictionary for the "network" tag
-		nettags := copyTags(tags)
-		nettags["network"] = network
-		acc.AddFields("docker_container_net", netfields, nettags, now)
+		if perDevice {
+			nettags := copyTags(tags)
+			nettags["network"] = network
+			acc.AddFields("docker_container_net", netfields, nettags, now)
+		}
+		if total {
+			for field, value := range netfields {
+				if field == "container_id" {
+					continue
+				}
+				_, ok := totalNetworkStatMap[field]
+				if ok {
+					totalNetworkStatMap[field] = totalNetworkStatMap[field].(uint64) + value.(uint64)
+				} else {
+					totalNetworkStatMap[field] = value
+				}
+			}
+		}
 	}
 
-	gatherBlockIOMetrics(stat, acc, tags, now, id)
+	// totalNetworkStatMap could be empty if container is running with --net=host.
+	if total && len(totalNetworkStatMap) != 0 {
+		nettags := copyTags(tags)
+		nettags["network"] = "total"
+		totalNetworkStatMap["container_id"] = id
+		acc.AddFields("docker_container_net", totalNetworkStatMap, nettags, now)
+	}
+
+	gatherBlockIOMetrics(stat, acc, tags, now, id, perDevice, total)
 }
 
 func calculateMemPercent(stat *types.StatsJSON) float64 {
@@ -361,6 +405,8 @@ func gatherBlockIOMetrics(
 	tags map[string]string,
 	now time.Time,
 	id string,
+	perDevice bool,
+	total bool,
 ) {
 	blkioStats := stat.BlkioStats
 	// Make a map of devices to their block io stats
@@ -422,11 +468,33 @@ func gatherBlockIOMetrics(
 		deviceStatMap[device]["sectors_recursive"] = metric.Value
 	}
 
+	totalStatMap := make(map[string]interface{})
 	for device, fields := range deviceStatMap {
-		iotags := copyTags(tags)
-		iotags["device"] = device
 		fields["container_id"] = id
-		acc.AddFields("docker_container_blkio", fields, iotags, now)
+		if perDevice {
+			iotags := copyTags(tags)
+			iotags["device"] = device
+			acc.AddFields("docker_container_blkio", fields, iotags, now)
+		}
+		if total {
+			for field, value := range fields {
+				if field == "container_id" {
+					continue
+				}
+				_, ok := totalStatMap[field]
+				if ok {
+					totalStatMap[field] = totalStatMap[field].(uint64) + value.(uint64)
+				} else {
+					totalStatMap[field] = value
+				}
+			}
+		}
+	}
+	if total {
+		totalStatMap["container_id"] = id
+		iotags := copyTags(tags)
+		iotags["device"] = "total"
+		acc.AddFields("docker_container_blkio", totalStatMap, iotags, now)
 	}
 }
 
@@ -471,7 +539,8 @@ func parseSize(sizeStr string) (int64, error) {
 func init() {
 	inputs.Add("docker", func() telegraf.Input {
 		return &Docker{
-			Timeout: internal.Duration{Duration: time.Second * 5},
+			PerDevice: true,
+			Timeout:   internal.Duration{Duration: time.Second * 5},
 		}
 	})
 }

plugins/inputs/docker/docker_test.go

@@ -24,7 +24,7 @@ func TestDockerGatherContainerStats(t *testing.T) {
 		"container_name":  "redis",
 		"container_image": "redis/image",
 	}
-	gatherContainerStats(stats, &acc, tags, "123456789")
+	gatherContainerStats(stats, &acc, tags, "123456789", true, true)
 
 	// test docker_container_net measurement
 	netfields := map[string]interface{}{
@@ -42,6 +42,21 @@ func TestDockerGatherContainerStats(t *testing.T) {
 	nettags["network"] = "eth0"
 	acc.AssertContainsTaggedFields(t, "docker_container_net", netfields, nettags)
 
+	netfields = map[string]interface{}{
+		"rx_dropped":   uint64(6),
+		"rx_bytes":     uint64(8),
+		"rx_errors":    uint64(10),
+		"tx_packets":   uint64(12),
+		"tx_dropped":   uint64(6),
+		"rx_packets":   uint64(8),
+		"tx_errors":    uint64(10),
+		"tx_bytes":     uint64(12),
+		"container_id": "123456789",
+	}
+	nettags = copyTags(tags)
+	nettags["network"] = "total"
+	acc.AssertContainsTaggedFields(t, "docker_container_net", netfields, nettags)
+
 	// test docker_blkio measurement
 	blkiotags := copyTags(tags)
 	blkiotags["device"] = "6:0"
@@ -52,6 +67,15 @@ func TestDockerGatherContainerStats(t *testing.T) {
 	}
 	acc.AssertContainsTaggedFields(t, "docker_container_blkio", blkiofields, blkiotags)
 
+	blkiotags = copyTags(tags)
+	blkiotags["device"] = "total"
+	blkiofields = map[string]interface{}{
+		"io_service_bytes_recursive_read": uint64(100),
+		"io_serviced_recursive_write":     uint64(302),
+		"container_id":                    "123456789",
+	}
+	acc.AssertContainsTaggedFields(t, "docker_container_blkio", blkiofields, blkiotags)
+
 	// test docker_container_mem measurement
 	memfields := map[string]interface{}{
 		"max_usage":                 uint64(1001),
@@ -186,6 +210,17 @@ func testStats() *types.StatsJSON {
 		TxBytes:   4,
 	}
 
+	stats.Networks["eth1"] = types.NetworkStats{
+		RxDropped: 5,
+		RxBytes:   6,
+		RxErrors:  7,
+		TxPackets: 8,
+		TxDropped: 5,
+		RxPackets: 6,
+		TxErrors:  7,
+		TxBytes:   8,
+	}
+
 	sbr := types.BlkioStatEntry{
 		Major: 6,
 		Minor: 0,
@@ -198,11 +233,19 @@ func testStats() *types.StatsJSON {
 		Op:    "write",
 		Value: 101,
 	}
+	sr2 := types.BlkioStatEntry{
+		Major: 6,
+		Minor: 1,
+		Op:    "write",
+		Value: 201,
+	}
 
 	stats.BlkioStats.IoServiceBytesRecursive = append(
 		stats.BlkioStats.IoServiceBytesRecursive, sbr)
 	stats.BlkioStats.IoServicedRecursive = append(
 		stats.BlkioStats.IoServicedRecursive, sr)
+	stats.BlkioStats.IoServicedRecursive = append(
+		stats.BlkioStats.IoServicedRecursive, sr2)
 
 	return stats
 }
@@ -378,9 +421,10 @@ func TestDockerGatherInfo(t *testing.T) {
 			"container_id": "b7dfbb9478a6ae55e237d4d74f8bbb753f0817192b5081334dc78476296e2173",
 		},
 		map[string]string{
-			"container_name":  "etcd2",
-			"container_image": "quay.io/coreos/etcd:v2.2.2",
-			"cpu":             "cpu3",
+			"container_name":    "etcd2",
+			"container_image":   "quay.io/coreos/etcd",
+			"cpu":               "cpu3",
+			"container_version": "v2.2.2",
 		},
 	)
 	acc.AssertContainsTaggedFields(t,
@@ -423,8 +467,9 @@ func TestDockerGatherInfo(t *testing.T) {
 			"container_id":              "b7dfbb9478a6ae55e237d4d74f8bbb753f0817192b5081334dc78476296e2173",
 		},
 		map[string]string{
-			"container_name":  "etcd2",
-			"container_image": "quay.io/coreos/etcd:v2.2.2",
+			"container_name":    "etcd2",
+			"container_image":   "quay.io/coreos/etcd",
+			"container_version": "v2.2.2",
 		},
 	)
 

plugins/inputs/dovecot/dovecot.go

@@ -12,6 +12,7 @@ import (
 	"time"
 
 	"github.com/influxdata/telegraf"
+	"github.com/influxdata/telegraf/internal/errchan"
 	"github.com/influxdata/telegraf/plugins/inputs"
 )
 
@@ -51,7 +52,6 @@ const defaultPort = "24242"
 
 // Reads stats from all configured servers.
 func (d *Dovecot) Gather(acc telegraf.Accumulator) error {
-
 	if !validQuery[d.Type] {
 		return fmt.Errorf("Error: %s is not a valid query type\n",
 			d.Type)
@@ -61,31 +61,27 @@ func (d *Dovecot) Gather(acc telegraf.Accumulator) error {
 		d.Servers = append(d.Servers, "127.0.0.1:24242")
 	}
 
-	var wg sync.WaitGroup
-
-	var outerr error
-
 	if len(d.Filters) <= 0 {
 		d.Filters = append(d.Filters, "")
 	}
 
-	for _, serv := range d.Servers {
+	var wg sync.WaitGroup
+	errChan := errchan.New(len(d.Servers) * len(d.Filters))
+	for _, server := range d.Servers {
 		for _, filter := range d.Filters {
 			wg.Add(1)
-			go func(serv string, filter string) {
+			go func(s string, f string) {
 				defer wg.Done()
-				outerr = d.gatherServer(serv, acc, d.Type, filter)
-			}(serv, filter)
+				errChan.C <- d.gatherServer(s, acc, d.Type, f)
+			}(server, filter)
 		}
 	}
 
 	wg.Wait()
-
-	return outerr
+	return errChan.Error()
 }
 
 func (d *Dovecot) gatherServer(addr string, acc telegraf.Accumulator, qtype string, filter string) error {
-
 	_, _, err := net.SplitHostPort(addr)
 	if err != nil {
 		return fmt.Errorf("Error: %s on url %s\n", err, addr)

plugins/inputs/elasticsearch/README.md

@@ -11,6 +11,13 @@ and optionally [cluster](https://www.elastic.co/guide/en/elasticsearch/reference
   servers = ["http://localhost:9200"]
   local = true
   cluster_health = true
+
+  ## Optional SSL Config
+  # ssl_ca = "/etc/telegraf/ca.pem"
+  # ssl_cert = "/etc/telegraf/cert.pem"
+  # ssl_key = "/etc/telegraf/key.pem"
+  ## Use SSL but skip chain & host verification
+  # insecure_skip_verify = false
 ```
 
 ### Measurements & Fields:

plugins/inputs/elasticsearch/elasticsearch.go

@@ -8,6 +8,7 @@ import (
 	"time"
 
 	"github.com/influxdata/telegraf"
+	"github.com/influxdata/telegraf/internal"
 	"github.com/influxdata/telegraf/internal/errchan"
 	"github.com/influxdata/telegraf/plugins/inputs"
 	jsonparser "github.com/influxdata/telegraf/plugins/parsers/json"
@@ -67,25 +68,31 @@ const sampleConfig = `
 
   ## set cluster_health to true when you want to also obtain cluster level stats
   cluster_health = false
+
+  ## Optional SSL Config
+  # ssl_ca = "/etc/telegraf/ca.pem"
+  # ssl_cert = "/etc/telegraf/cert.pem"
+  # ssl_key = "/etc/telegraf/key.pem"
+  ## Use SSL but skip chain & host verification
+  # insecure_skip_verify = false
 `
 
 // Elasticsearch is a plugin to read stats from one or many Elasticsearch
 // servers.
 type Elasticsearch struct {
-	Local         bool
-	Servers       []string
-	ClusterHealth bool
-	client        *http.Client
+	Local              bool
+	Servers            []string
+	ClusterHealth      bool
+	SSLCA              string `toml:"ssl_ca"`   // Path to CA file
+	SSLCert            string `toml:"ssl_cert"` // Path to host cert file
+	SSLKey             string `toml:"ssl_key"`  // Path to cert key file
+	InsecureSkipVerify bool   // Use SSL but skip chain & host verification
+	client             *http.Client
 }
 
 // NewElasticsearch return a new instance of Elasticsearch
 func NewElasticsearch() *Elasticsearch {
-	tr := &http.Transport{ResponseHeaderTimeout: time.Duration(3 * time.Second)}
-	client := &http.Client{
-		Transport: tr,
-		Timeout:   time.Duration(4 * time.Second),
-	}
-	return &Elasticsearch{client: client}
+	return &Elasticsearch{}
 }
 
 // SampleConfig returns sample configuration for this plugin.
@@ -101,6 +108,15 @@ func (e *Elasticsearch) Description() string {
 // Gather reads the stats from Elasticsearch and writes it to the
 // Accumulator.
 func (e *Elasticsearch) Gather(acc telegraf.Accumulator) error {
+	if e.client == nil {
+		client, err := e.createHttpClient()
+
+		if err != nil {
+			return err
+		}
+		e.client = client
+	}
+
 	errChan := errchan.New(len(e.Servers))
 	var wg sync.WaitGroup
 	wg.Add(len(e.Servers))
@@ -128,6 +144,23 @@ func (e *Elasticsearch) Gather(acc telegraf.Accumulator) error {
 	return errChan.Error()
 }
 
+func (e *Elasticsearch) createHttpClient() (*http.Client, error) {
+	tlsCfg, err := internal.GetTLSConfig(e.SSLCert, e.SSLKey, e.SSLCA, e.InsecureSkipVerify)
+	if err != nil {
+		return nil, err
+	}
+	tr := &http.Transport{
+		ResponseHeaderTimeout: time.Duration(3 * time.Second),
+		TLSClientConfig:       tlsCfg,
+	}
+	client := &http.Client{
+		Transport: tr,
+		Timeout:   time.Duration(4 * time.Second),
+	}
+
+	return client, nil
+}
+
 func (e *Elasticsearch) gatherNodeStats(url string, acc telegraf.Accumulator) error {
 	nodeStats := &struct {
 		ClusterName string           `json:"cluster_name"`

plugins/inputs/elasticsearch/elasticsearch_test.go

@@ -38,7 +38,7 @@ func (t *transportMock) CancelRequest(_ *http.Request) {
 }
 
 func TestElasticsearch(t *testing.T) {
-	es := NewElasticsearch()
+	es := newElasticsearchWithClient()
 	es.Servers = []string{"http://example.com:9200"}
 	es.client.Transport = newTransportMock(http.StatusOK, statsResponse)
 
@@ -67,7 +67,7 @@ func TestElasticsearch(t *testing.T) {
 }
 
 func TestGatherClusterStats(t *testing.T) {
-	es := NewElasticsearch()
+	es := newElasticsearchWithClient()
 	es.Servers = []string{"http://example.com:9200"}
 	es.ClusterHealth = true
 	es.client.Transport = newTransportMock(http.StatusOK, clusterResponse)
@@ -87,3 +87,9 @@ func TestGatherClusterStats(t *testing.T) {
 		v2IndexExpected,
 		map[string]string{"index": "v2"})
 }
+
+func newElasticsearchWithClient() *Elasticsearch {
+	es := NewElasticsearch()
+	es.client = &http.Client{}
+	return es
+}

plugins/inputs/exec/exec.go

@@ -48,8 +48,6 @@ type Exec struct {
 
 	parser parsers.Parser
 
-	wg sync.WaitGroup
-
 	runner  Runner
 	errChan chan error
 }
@@ -119,8 +117,8 @@ func (c CommandRunner) Run(
 	return out.Bytes(), nil
 }
 
-func (e *Exec) ProcessCommand(command string, acc telegraf.Accumulator) {
-	defer e.wg.Done()
+func (e *Exec) ProcessCommand(command string, acc telegraf.Accumulator, wg *sync.WaitGroup) {
+	defer wg.Done()
 
 	out, err := e.runner.Run(e, command, acc)
 	if err != nil {
@@ -151,6 +149,7 @@ func (e *Exec) SetParser(parser parsers.Parser) {
 }
 
 func (e *Exec) Gather(acc telegraf.Accumulator) error {
+	var wg sync.WaitGroup
 	// Legacy single command support
 	if e.Command != "" {
 		e.Commands = append(e.Commands, e.Command)
@@ -190,11 +189,11 @@ func (e *Exec) Gather(acc telegraf.Accumulator) error {
 	errChan := errchan.New(len(commands))
 	e.errChan = errChan.C
 
-	e.wg.Add(len(commands))
+	wg.Add(len(commands))
 	for _, command := range commands {
-		go e.ProcessCommand(command, acc)
+		go e.ProcessCommand(command, acc, &wg)
 	}
-	e.wg.Wait()
+	wg.Wait()
 	return errChan.Error()
 }
 

plugins/inputs/haproxy/haproxy.go

@@ -92,9 +92,11 @@ type haproxy struct {
 var sampleConfig = `
   ## An array of address to gather stats about. Specify an ip on hostname
   ## with optional port. ie localhost, 10.10.3.33:1936, etc.
-
-  ## If no servers are specified, then default to 127.0.0.1:1936
-  servers = ["http://myhaproxy.com:1936", "http://anotherhaproxy.com:1936"]
+  ## Make sure you specify the complete path to the stats endpoint
+  ## ie 10.10.3.33:1936/haproxy?stats
+  #
+  ## If no servers are specified, then default to 127.0.0.1:1936/haproxy?stats
+  servers = ["http://myhaproxy.com:1936/haproxy?stats"]
   ## Or you can also use local socket
   ## servers = ["socket:/run/haproxy/admin.sock"]
 `
@@ -111,7 +113,7 @@ func (r *haproxy) Description() string {
 // Returns one of the errors encountered while gather stats (if any).
 func (g *haproxy) Gather(acc telegraf.Accumulator) error {
 	if len(g.Servers) == 0 {
-		return g.gatherServer("http://127.0.0.1:1936", acc)
+		return g.gatherServer("http://127.0.0.1:1936/haproxy?stats", acc)
 	}
 
 	var wg sync.WaitGroup
@@ -167,12 +169,16 @@ func (g *haproxy) gatherServer(addr string, acc telegraf.Accumulator) error {
 		g.client = client
 	}
 
+	if !strings.HasSuffix(addr, ";csv") {
+		addr += "/;csv"
+	}
+
 	u, err := url.Parse(addr)
 	if err != nil {
 		return fmt.Errorf("Unable parse server address '%s': %s", addr, err)
 	}
 
-	req, err := http.NewRequest("GET", fmt.Sprintf("%s://%s%s/;csv", u.Scheme, u.Host, u.Path), nil)
+	req, err := http.NewRequest("GET", addr, nil)
 	if u.User != nil {
 		p, _ := u.User.Password()
 		req.SetBasicAuth(u.User.Username(), p)
@@ -184,7 +190,7 @@ func (g *haproxy) gatherServer(addr string, acc telegraf.Accumulator) error {
 	}
 
 	if res.StatusCode != 200 {
-		return fmt.Errorf("Unable to get valid stat result from '%s': %s", addr, err)
+		return fmt.Errorf("Unable to get valid stat result from '%s', http response code : %d", addr, res.StatusCode)
 	}
 
 	return importCsvResult(res.Body, acc, u.Host)

plugins/inputs/haproxy/haproxy_test.go

@@ -243,7 +243,7 @@ func TestHaproxyDefaultGetFromLocalhost(t *testing.T) {
 
 	err := r.Gather(&acc)
 	require.Error(t, err)
-	assert.Contains(t, err.Error(), "127.0.0.1:1936/;csv")
+	assert.Contains(t, err.Error(), "127.0.0.1:1936/haproxy?stats/;csv")
 }
 
 const csvOutputSample = `

plugins/inputs/hddtemp/README.md

@@ -0,0 +1,22 @@
+# Hddtemp Input Plugin
+
+This plugin reads data from hddtemp daemon
+
+## Requirements
+
+Hddtemp should be installed and its daemon running
+
+## Configuration
+
+```
+[[inputs.hddtemp]]
+## By default, telegraf gathers temps data from all disks detected by the
+## hddtemp.
+##
+## Only collect temps from the selected disks.
+##
+## A * as the device name will return the temperature values of all disks.
+##
+# address = "127.0.0.1:7634"
+# devices = ["sda", "*"]
+```

plugins/inputs/hddtemp/go-hddtemp/LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2016 Mendelson Gusmão
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

plugins/inputs/hddtemp/go-hddtemp/hddtemp.go

@@ -0,0 +1,61 @@
+package hddtemp
+
+import (
+	"bytes"
+	"io"
+	"net"
+	"strconv"
+	"strings"
+)
+
+type disk struct {
+	DeviceName  string
+	Model       string
+	Temperature int32
+	Unit        string
+	Status      string
+}
+
+func Fetch(address string) ([]disk, error) {
+	var (
+		err    error
+		conn   net.Conn
+		buffer bytes.Buffer
+		disks  []disk
+	)
+
+	if conn, err = net.Dial("tcp", address); err != nil {
+		return nil, err
+	}
+
+	if _, err = io.Copy(&buffer, conn); err != nil {
+		return nil, err
+	}
+
+	fields := strings.Split(buffer.String(), "|")
+
+	for index := 0; index < len(fields)/5; index++ {
+		status := ""
+		offset := index * 5
+		device := fields[offset+1]
+		device = device[strings.LastIndex(device, "/")+1:]
+
+		temperatureField := fields[offset+3]
+		temperature, err := strconv.ParseInt(temperatureField, 10, 32)
+
+		if err != nil {
+			temperature = 0
+			status = temperatureField
+		}
+
+		disks = append(disks, disk{
+			DeviceName:  device,
+			Model:       fields[offset+2],
+			Temperature: int32(temperature),
+			Unit:        fields[offset+4],
+			Status:      status,
+		})
+	}
+
+	return disks, nil
+}

plugins/inputs/hddtemp/go-hddtemp/hddtemp_test.go

@@ -0,0 +1,116 @@
+package hddtemp
+
+import (
+	"net"
+	"reflect"
+	"testing"
+)
+
+func TestFetch(t *testing.T) {
+	l := serve(t, []byte("|/dev/sda|foobar|36|C|"))
+	defer l.Close()
+
+	disks, err := Fetch(l.Addr().String())
+
+	if err != nil {
+		t.Error("expecting err to be nil")
+	}
+
+	expected := []disk{
+		{
+			DeviceName:  "sda",
+			Model:       "foobar",
+			Temperature: 36,
+			Unit:        "C",
+		},
+	}
+
+	if !reflect.DeepEqual(expected, disks) {
+		t.Error("disks' slice is different from expected")
+	}
+}
+
+func TestFetchWrongAddress(t *testing.T) {
+	_, err := Fetch("127.0.0.1:1")
+
+	if err == nil {
+		t.Error("expecting err to be non-nil")
+	}
+}
+
+func TestFetchStatus(t *testing.T) {
+	l := serve(t, []byte("|/dev/sda|foobar|SLP|C|"))
+	defer l.Close()
+
+	disks, err := Fetch(l.Addr().String())
+
+	if err != nil {
+		t.Error("expecting err to be nil")
+	}
+
+	expected := []disk{
+		{
+			DeviceName:  "sda",
+			Model:       "foobar",
+			Temperature: 0,
+			Unit:        "C",
+			Status:      "SLP",
+		},
+	}
+
+	if !reflect.DeepEqual(expected, disks) {
+		t.Error("disks' slice is different from expected")
+	}
+}
+
+func TestFetchTwoDisks(t *testing.T) {
+	l := serve(t, []byte("|/dev/hda|ST380011A|46|C||/dev/hdd|ST340016A|SLP|*|"))
+	defer l.Close()
+
+	disks, err := Fetch(l.Addr().String())
+
+	if err != nil {
+		t.Error("expecting err to be nil")
+	}
+
+	expected := []disk{
+		{
+			DeviceName:  "hda",
+			Model:       "ST380011A",
+			Temperature: 46,
+			Unit:        "C",
+		},
+		{
+			DeviceName:  "hdd",
+			Model:       "ST340016A",
+			Temperature: 0,
+			Unit:        "*",
+			Status:      "SLP",
+		},
+	}
+
+	if !reflect.DeepEqual(expected, disks) {
+		t.Error("disks' slice is different from expected")
+	}
+}
+
+func serve(t *testing.T, data []byte) net.Listener {
+	l, err := net.Listen("tcp", "127.0.0.1:0")
+
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	go func(t *testing.T) {
+		conn, err := l.Accept()
+
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		conn.Write(data)
+		conn.Close()
+	}(t)
+
+	return l
+}

plugins/inputs/hddtemp/hddtemp.go

@@ -0,0 +1,74 @@
+// +build linux
+
+package hddtemp
+
+import (
+	"github.com/influxdata/telegraf"
+	"github.com/influxdata/telegraf/plugins/inputs"
+	gohddtemp "github.com/influxdata/telegraf/plugins/inputs/hddtemp/go-hddtemp"
+)
+
+const defaultAddress = "127.0.0.1:7634"
+
+type HDDTemp struct {
+	Address string
+	Devices []string
+}
+
+func (_ *HDDTemp) Description() string {
+	return "Monitor disks' temperatures using hddtemp"
+}
+
+var hddtempSampleConfig = `
+  ## By default, telegraf gathers temps data from all disks detected by the
+  ## hddtemp.
+  ##
+  ## Only collect temps from the selected disks.
+  ##
+  ## A * as the device name will return the temperature values of all disks.
+  ##
+  # address = "127.0.0.1:7634"
+  # devices = ["sda", "*"]
+`
+
+func (_ *HDDTemp) SampleConfig() string {
+	return hddtempSampleConfig
+}
+
+func (h *HDDTemp) Gather(acc telegraf.Accumulator) error {
+	disks, err := gohddtemp.Fetch(h.Address)
+
+	if err != nil {
+		return err
+	}
+
+	for _, disk := range disks {
+		for _, chosenDevice := range h.Devices {
+			if chosenDevice == "*" || chosenDevice == disk.DeviceName {
+				tags := map[string]string{
+					"device": disk.DeviceName,
+					"model":  disk.Model,
+					"unit":   disk.Unit,
+					"status": disk.Status,
+				}
+
+				fields := map[string]interface{}{
+					disk.DeviceName: disk.Temperature,
+				}
+
+				acc.AddFields("hddtemp", fields, tags)
+			}
+		}
+	}
+
+	return nil
+}
+
+func init() {
+	inputs.Add("hddtemp", func() telegraf.Input {
+		return &HDDTemp{
+			Address: defaultAddress,
+			Devices: []string{"*"},
+		}
+	})
+}

plugins/inputs/hddtemp/hddtemp_nocompile.go

@@ -0,0 +1,3 @@
+// +build !linux
+
+package hddtemp

plugins/inputs/jolokia/jolokia.go

@@ -249,7 +249,14 @@ func (j *Jolokia) Gather(acc telegraf.Accumulator) error {
 					switch t := values.(type) {
 					case map[string]interface{}:
 						for k, v := range t {
-							fields[measurement+"_"+k] = v
+							switch t2 := v.(type) {
+							case map[string]interface{}:
+								for k2, v2 := range t2 {
+									fields[measurement+"_"+k+"_"+k2] = v2
+								}
+							case interface{}:
+								fields[measurement+"_"+k] = t2
+							}
 						}
 					case interface{}:
 						fields[measurement] = t

plugins/inputs/kafka_consumer/README.md

@@ -22,7 +22,7 @@ from the same topic in parallel.
   ## Offset (must be either "oldest" or "newest")
   offset = "oldest"
 
-  ## Data format to consume. 
+  ## Data format to consume.
 
   ## Each data format has it's own unique set of configuration options, read
   ## more about them here:
@@ -32,11 +32,5 @@ from the same topic in parallel.
 
 ## Testing
 
-Running integration tests requires running Zookeeper & Kafka. The following
-commands assume you're on OS X & using [boot2docker](http://boot2docker.io/) or docker-machine through [Docker Toolbox](https://www.docker.com/docker-toolbox).
-
-To start Kafka & Zookeeper:
-
-```
-docker run -d -p 2181:2181 -p 9092:9092 --env ADVERTISED_HOST=`boot2docker ip || docker-machine ip <your_machine_name>` --env ADVERTISED_PORT=9092 spotify/kafka
-```
+Running integration tests requires running Zookeeper & Kafka. See Makefile
+for kafka container command.

plugins/inputs/logparser/README.md

@@ -14,17 +14,22 @@ regex patterns.
   ##   /var/log/**.log     -> recursively find all .log files in /var/log
   ##   /var/log/*/*.log    -> find all .log files with a parent dir in /var/log
   ##   /var/log/apache.log -> only tail the apache log file
-  files = ["/var/log/influxdb/influxdb.log"]
+  files = ["/var/log/apache/access.log"]
   ## Read file from beginning.
   from_beginning = false
 
   ## Parse logstash-style "grok" patterns:
-  ##   Telegraf builtin parsing patterns: https://goo.gl/dkay10
+  ##   Telegraf built-in parsing patterns: https://goo.gl/dkay10
   [inputs.logparser.grok]
     ## This is a list of patterns to check the given log file(s) for.
     ## Note that adding patterns here increases processing time. The most
-    ## efficient configuration is to have one file & pattern per logparser.
-    patterns = ["%{INFLUXDB_HTTPD_LOG}"]
+    ## efficient configuration is to have one pattern per logparser.
+    ## Other common built-in patterns are:
+    ##   %{COMMON_LOG_FORMAT}   (plain apache & nginx access logs)
+    ##   %{COMBINED_LOG_FORMAT} (access logs + referrer & agent)
+    patterns = ["%{COMBINED_LOG_FORMAT}"]
+    ## Name of the outputted measurement name.
+    measurement = "apache_access_log"
     ## Full path(s) to custom pattern files.
     custom_pattern_files = []
     ## Custom patterns can also be defined here. Put one pattern per line.
@@ -67,6 +72,7 @@ Timestamp modifiers can be used to convert captures to the timestamp of the
   - tag      (converts the field into a tag)
   - drop     (drops the field completely)
 - Timestamp modifiers:
+  - ts               (This will auto-learn the timestamp format)
   - ts-ansic         ("Mon Jan _2 15:04:05 2006")
   - ts-unix          ("Mon Jan _2 15:04:05 MST 2006")
   - ts-ruby          ("Mon Jan 02 15:04:05 -0700 2006")

plugins/inputs/logparser/grok/grok.go

@@ -15,7 +15,7 @@ import (
 	"github.com/influxdata/telegraf"
 )
 
-var timeFormats = map[string]string{
+var timeLayouts = map[string]string{
 	"ts-ansic":       "Mon Jan _2 15:04:05 2006",
 	"ts-unix":        "Mon Jan _2 15:04:05 MST 2006",
 	"ts-ruby":        "Mon Jan 02 15:04:05 -0700 2006",
@@ -27,35 +27,47 @@ var timeFormats = map[string]string{
 	"ts-rfc3339":     "2006-01-02T15:04:05Z07:00",
 	"ts-rfc3339nano": "2006-01-02T15:04:05.999999999Z07:00",
 	"ts-httpd":       "02/Jan/2006:15:04:05 -0700",
-	"ts-epoch":       "EPOCH",
-	"ts-epochnano":   "EPOCH_NANO",
+	// These three are not exactly "layouts", but they are special cases that
+	// will get handled in the ParseLine function.
+	"ts-epoch":     "EPOCH",
+	"ts-epochnano": "EPOCH_NANO",
+	"ts":           "GENERIC_TIMESTAMP", // try parsing all known timestamp layouts.
 }
 
 const (
-	INT      = "int"
-	TAG      = "tag"
-	FLOAT    = "float"
-	STRING   = "string"
-	DURATION = "duration"
-	DROP     = "drop"
+	INT               = "int"
+	TAG               = "tag"
+	FLOAT             = "float"
+	STRING            = "string"
+	DURATION          = "duration"
+	DROP              = "drop"
+	EPOCH             = "EPOCH"
+	EPOCH_NANO        = "EPOCH_NANO"
+	GENERIC_TIMESTAMP = "GENERIC_TIMESTAMP"
 )
 
 var (
-	// matches named captures that contain a type.
+	// matches named captures that contain a modifier.
 	//   ie,
 	//     %{NUMBER:bytes:int}
 	//     %{IPORHOST:clientip:tag}
 	//     %{HTTPDATE:ts1:ts-http}
 	//     %{HTTPDATE:ts2:ts-"02 Jan 06 15:04"}
-	typedRe = regexp.MustCompile(`%{\w+:(\w+):(ts-".+"|t?s?-?\w+)}`)
+	modifierRe = regexp.MustCompile(`%{\w+:(\w+):(ts-".+"|t?s?-?\w+)}`)
 	// matches a plain pattern name. ie, %{NUMBER}
 	patternOnlyRe = regexp.MustCompile(`%{(\w+)}`)
 )
 
 type Parser struct {
-	Patterns           []string
+	Patterns []string
+	// namedPatterns is a list of internally-assigned names to the patterns
+	// specified by the user in Patterns.
+	// They will look like:
+	//   GROK_INTERNAL_PATTERN_0, GROK_INTERNAL_PATTERN_1, etc.
+	namedPatterns      []string
 	CustomPatterns     string
 	CustomPatternFiles []string
+	Measurement        string
 
 	// typeMap is a map of patterns -> capture name -> modifier,
 	//   ie, {
@@ -81,6 +93,12 @@ type Parser struct {
 	//          "RESPONSE_CODE": "%{NUMBER:rc:tag}"
 	//       }
 	patterns map[string]string
+	// foundTsLayouts is a slice of timestamp patterns that have been found
+	// in the log lines. This slice gets updated if the user uses the generic
+	// 'ts' modifier for timestamps. This slice is checked first for matches,
+	// so that previously-matched layouts get priority over all other timestamp
+	// layouts.
+	foundTsLayouts []string
 
 	g        *grok.Grok
 	tsModder *tsModder
@@ -97,13 +115,24 @@ func (p *Parser) Compile() error {
 		return err
 	}
 
-	p.CustomPatterns = DEFAULT_PATTERNS + p.CustomPatterns
+	// Give Patterns fake names so that they can be treated as named
+	// "custom patterns"
+	p.namedPatterns = make([]string, len(p.Patterns))
+	for i, pattern := range p.Patterns {
+		name := fmt.Sprintf("GROK_INTERNAL_PATTERN_%d", i)
+		p.CustomPatterns += "\n" + name + " " + pattern + "\n"
+		p.namedPatterns[i] = "%{" + name + "}"
+	}
 
+	// Combine user-supplied CustomPatterns with DEFAULT_PATTERNS and parse
+	// them together as the same type of pattern.
+	p.CustomPatterns = DEFAULT_PATTERNS + p.CustomPatterns
 	if len(p.CustomPatterns) != 0 {
 		scanner := bufio.NewScanner(strings.NewReader(p.CustomPatterns))
 		p.addCustomPatterns(scanner)
 	}
 
+	// Parse any custom pattern files supplied.
 	for _, filename := range p.CustomPatternFiles {
 		file, err := os.Open(filename)
 		if err != nil {
@@ -114,15 +143,20 @@ func (p *Parser) Compile() error {
 		p.addCustomPatterns(scanner)
 	}
 
+	if p.Measurement == "" {
+		p.Measurement = "logparser_grok"
+	}
+
 	return p.compileCustomPatterns()
 }
 
 func (p *Parser) ParseLine(line string) (telegraf.Metric, error) {
 	var err error
+	// values are the parsed fields from the log line
 	var values map[string]string
 	// the matching pattern string
 	var patternName string
-	for _, pattern := range p.Patterns {
+	for _, pattern := range p.namedPatterns {
 		if values, err = p.g.Parse(pattern, line); err != nil {
 			return nil, err
 		}
@@ -144,6 +178,7 @@ func (p *Parser) ParseLine(line string) (telegraf.Metric, error) {
 			continue
 		}
 
+		// t is the modifier of the field
 		var t string
 		// check if pattern has some modifiers
 		if types, ok := p.typeMap[patternName]; ok {
@@ -189,20 +224,50 @@ func (p *Parser) ParseLine(line string) (telegraf.Metric, error) {
 			tags[k] = v
 		case STRING:
 			fields[k] = strings.Trim(v, `"`)
-		case "EPOCH":
+		case EPOCH:
 			iv, err := strconv.ParseInt(v, 10, 64)
 			if err != nil {
 				log.Printf("ERROR parsing %s to int: %s", v, err)
 			} else {
 				timestamp = time.Unix(iv, 0)
 			}
-		case "EPOCH_NANO":
+		case EPOCH_NANO:
 			iv, err := strconv.ParseInt(v, 10, 64)
 			if err != nil {
 				log.Printf("ERROR parsing %s to int: %s", v, err)
 			} else {
 				timestamp = time.Unix(0, iv)
 			}
+		case GENERIC_TIMESTAMP:
+			var foundTs bool
+			// first try timestamp layouts that we've already found
+			for _, layout := range p.foundTsLayouts {
+				ts, err := time.Parse(layout, v)
+				if err == nil {
+					timestamp = ts
+					foundTs = true
+					break
+				}
+			}
+			// if we haven't found a timestamp layout yet, try all timestamp
+			// layouts.
+			if !foundTs {
+				for _, layout := range timeLayouts {
+					ts, err := time.Parse(layout, v)
+					if err == nil {
+						timestamp = ts
+						foundTs = true
+						p.foundTsLayouts = append(p.foundTsLayouts, layout)
+						break
+					}
+				}
+			}
+			// if we still haven't found a timestamp layout, log it and we will
+			// just use time.Now()
+			if !foundTs {
+				log.Printf("ERROR parsing timestamp [%s], could not find any "+
+					"suitable time layouts.", v)
+			}
 		case DROP:
 		// goodbye!
 		default:
@@ -215,7 +280,7 @@ func (p *Parser) ParseLine(line string) (telegraf.Metric, error) {
 		}
 	}
 
-	return telegraf.NewMetric("logparser_grok", tags, fields, p.tsModder.tsMod(timestamp))
+	return telegraf.NewMetric(p.Measurement, tags, fields, p.tsModder.tsMod(timestamp))
 }
 
 func (p *Parser) addCustomPatterns(scanner *bufio.Scanner) {
@@ -246,7 +311,7 @@ func (p *Parser) compileCustomPatterns() error {
 
 	// check if pattern contains modifiers. Parse them out if it does.
 	for name, pattern := range p.patterns {
-		if typedRe.MatchString(pattern) {
+		if modifierRe.MatchString(pattern) {
 			// this pattern has modifiers, so parse out the modifiers
 			pattern, err = p.parseTypedCaptures(name, pattern)
 			if err != nil {
@@ -259,13 +324,13 @@ func (p *Parser) compileCustomPatterns() error {
 	return p.g.AddPatternsFromMap(p.patterns)
 }
 
-// parseTypedCaptures parses the capture types, and then deletes the type from
-// the line so that it is a valid "grok" pattern again.
+// parseTypedCaptures parses the capture modifiers, and then deletes the
+// modifier from the line so that it is a valid "grok" pattern again.
 //   ie,
 //     %{NUMBER:bytes:int}      => %{NUMBER:bytes}      (stores %{NUMBER}->bytes->int)
 //     %{IPORHOST:clientip:tag} => %{IPORHOST:clientip} (stores %{IPORHOST}->clientip->tag)
 func (p *Parser) parseTypedCaptures(name, pattern string) (string, error) {
-	matches := typedRe.FindAllStringSubmatch(pattern, -1)
+	matches := modifierRe.FindAllStringSubmatch(pattern, -1)
 
 	// grab the name of the capture pattern
 	patternName := "%{" + name + "}"
@@ -277,16 +342,18 @@ func (p *Parser) parseTypedCaptures(name, pattern string) (string, error) {
 	hasTimestamp := false
 	for _, match := range matches {
 		// regex capture 1 is the name of the capture
-		// regex capture 2 is the type of the capture
-		if strings.HasPrefix(match[2], "ts-") {
+		// regex capture 2 is the modifier of the capture
+		if strings.HasPrefix(match[2], "ts") {
 			if hasTimestamp {
 				return pattern, fmt.Errorf("logparser pattern compile error: "+
 					"Each pattern is allowed only one named "+
 					"timestamp data type. pattern: %s", pattern)
 			}
-			if f, ok := timeFormats[match[2]]; ok {
-				p.tsMap[patternName][match[1]] = f
+			if layout, ok := timeLayouts[match[2]]; ok {
+				// built-in time format
+				p.tsMap[patternName][match[1]] = layout
 			} else {
+				// custom time format
 				p.tsMap[patternName][match[1]] = strings.TrimSuffix(strings.TrimPrefix(match[2], `ts-"`), `"`)
 			}
 			hasTimestamp = true

plugins/inputs/logparser/grok/grok_test.go

@@ -38,32 +38,6 @@ func Benchmark_ParseLine_CombinedLogFormat(b *testing.B) {
 	benchM = m
 }
 
-func Benchmark_ParseLine_InfluxLog(b *testing.B) {
-	p := &Parser{
-		Patterns: []string{"%{INFLUXDB_HTTPD_LOG}"},
-	}
-	p.Compile()
-
-	var m telegraf.Metric
-	for n := 0; n < b.N; n++ {
-		m, _ = p.ParseLine(`[httpd] 192.168.1.1 - - [14/Jun/2016:11:33:29 +0100] "POST /write?consistency=any&db=telegraf&precision=ns&rp= HTTP/1.1" 204 0 "-" "InfluxDBClient" 6f61bc44-321b-11e6-8050-000000000000 2513`)
-	}
-	benchM = m
-}
-
-func Benchmark_ParseLine_InfluxLog_NoMatch(b *testing.B) {
-	p := &Parser{
-		Patterns: []string{"%{INFLUXDB_HTTPD_LOG}"},
-	}
-	p.Compile()
-
-	var m telegraf.Metric
-	for n := 0; n < b.N; n++ {
-		m, _ = p.ParseLine(`[retention] 2016/06/14 14:38:24 retention policy shard deletion check commencing`)
-	}
-	benchM = m
-}
-
 func Benchmark_ParseLine_CustomPattern(b *testing.B) {
 	p := &Parser{
 		Patterns: []string{"%{TEST_LOG_A}", "%{TEST_LOG_B}"},
@@ -83,9 +57,34 @@ func Benchmark_ParseLine_CustomPattern(b *testing.B) {
 	benchM = m
 }
 
-func TestBuiltinInfluxdbHttpd(t *testing.T) {
+func TestMeasurementName(t *testing.T) {
 	p := &Parser{
-		Patterns: []string{"%{INFLUXDB_HTTPD_LOG}"},
+		Measurement: "my_web_log",
+		Patterns:    []string{"%{COMMON_LOG_FORMAT}"},
+	}
+	assert.NoError(t, p.Compile())
+
+	// Parse an influxdb POST request
+	m, err := p.ParseLine(`127.0.0.1 user-identifier frank [10/Oct/2000:13:55:36 -0700] "GET /apache_pb.gif HTTP/1.0" 200 2326`)
+	require.NotNil(t, m)
+	assert.NoError(t, err)
+	assert.Equal(t,
+		map[string]interface{}{
+			"resp_bytes":   int64(2326),
+			"auth":         "frank",
+			"client_ip":    "127.0.0.1",
+			"http_version": float64(1.0),
+			"ident":        "user-identifier",
+			"request":      "/apache_pb.gif",
+		},
+		m.Fields())
+	assert.Equal(t, map[string]string{"verb": "GET", "resp_code": "200"}, m.Tags())
+	assert.Equal(t, "my_web_log", m.Name())
+}
+
+func TestCustomInfluxdbHttpd(t *testing.T) {
+	p := &Parser{
+		Patterns: []string{`\[httpd\] %{COMBINED_LOG_FORMAT} %{UUID:uuid:drop} %{NUMBER:response_time_us:int}`},
 	}
 	assert.NoError(t, p.Compile())
 
@@ -98,7 +97,6 @@ func TestBuiltinInfluxdbHttpd(t *testing.T) {
 			"resp_bytes":       int64(0),
 			"auth":             "-",
 			"client_ip":        "::1",
-			"resp_code":        int64(204),
 			"http_version":     float64(1.1),
 			"ident":            "-",
 			"referrer":         "-",
@@ -107,7 +105,7 @@ func TestBuiltinInfluxdbHttpd(t *testing.T) {
 			"agent":            "InfluxDBClient",
 		},
 		m.Fields())
-	assert.Equal(t, map[string]string{"verb": "POST"}, m.Tags())
+	assert.Equal(t, map[string]string{"verb": "POST", "resp_code": "204"}, m.Tags())
 
 	// Parse an influxdb GET request
 	m, err = p.ParseLine(`[httpd] ::1 - - [14/Jun/2016:12:10:02 +0100] "GET /query?db=telegraf&q=SELECT+bytes%2Cresponse_time_us+FROM+logparser_grok+WHERE+http_method+%3D+%27GET%27+AND+response_time_us+%3E+0+AND+time+%3E+now%28%29+-+1h HTTP/1.1" 200 578 "http://localhost:8083/" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.84 Safari/537.36" 8a3806f1-3220-11e6-8006-000000000000 988`)
@@ -118,7 +116,6 @@ func TestBuiltinInfluxdbHttpd(t *testing.T) {
 			"resp_bytes":       int64(578),
 			"auth":             "-",
 			"client_ip":        "::1",
-			"resp_code":        int64(200),
 			"http_version":     float64(1.1),
 			"ident":            "-",
 			"referrer":         "http://localhost:8083/",
@@ -127,7 +124,7 @@ func TestBuiltinInfluxdbHttpd(t *testing.T) {
 			"agent":            "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.84 Safari/537.36",
 		},
 		m.Fields())
-	assert.Equal(t, map[string]string{"verb": "GET"}, m.Tags())
+	assert.Equal(t, map[string]string{"verb": "GET", "resp_code": "200"}, m.Tags())
 }
 
 // common log format
@@ -147,13 +144,12 @@ func TestBuiltinCommonLogFormat(t *testing.T) {
 			"resp_bytes":   int64(2326),
 			"auth":         "frank",
 			"client_ip":    "127.0.0.1",
-			"resp_code":    int64(200),
 			"http_version": float64(1.0),
 			"ident":        "user-identifier",
 			"request":      "/apache_pb.gif",
 		},
 		m.Fields())
-	assert.Equal(t, map[string]string{"verb": "GET"}, m.Tags())
+	assert.Equal(t, map[string]string{"verb": "GET", "resp_code": "200"}, m.Tags())
 }
 
 // combined log format
@@ -173,7 +169,6 @@ func TestBuiltinCombinedLogFormat(t *testing.T) {
 			"resp_bytes":   int64(2326),
 			"auth":         "frank",
 			"client_ip":    "127.0.0.1",
-			"resp_code":    int64(200),
 			"http_version": float64(1.0),
 			"ident":        "user-identifier",
 			"request":      "/apache_pb.gif",
@@ -181,12 +176,12 @@ func TestBuiltinCombinedLogFormat(t *testing.T) {
 			"agent":        "Mozilla",
 		},
 		m.Fields())
-	assert.Equal(t, map[string]string{"verb": "GET"}, m.Tags())
+	assert.Equal(t, map[string]string{"verb": "GET", "resp_code": "200"}, m.Tags())
 }
 
 func TestCompileStringAndParse(t *testing.T) {
 	p := &Parser{
-		Patterns: []string{"%{TEST_LOG_A}", "%{TEST_LOG_B}"},
+		Patterns: []string{"%{TEST_LOG_A}"},
 		CustomPatterns: `
 			DURATION %{NUMBER}[nuµm]?s
 			RESPONSE_CODE %{NUMBER:response_code:tag}
@@ -209,6 +204,41 @@ func TestCompileStringAndParse(t *testing.T) {
 	assert.Equal(t, map[string]string{"response_code": "200"}, metricA.Tags())
 }
 
+func TestCompileErrorsOnInvalidPattern(t *testing.T) {
+	p := &Parser{
+		Patterns: []string{"%{TEST_LOG_A}", "%{TEST_LOG_B}"},
+		CustomPatterns: `
+			DURATION %{NUMBER}[nuµm]?s
+			RESPONSE_CODE %{NUMBER:response_code:tag}
+			RESPONSE_TIME %{DURATION:response_time:duration}
+			TEST_LOG_A %{NUMBER:myfloat:float} %{RESPONSE_CODE} %{IPORHOST:clientip} %{RESPONSE_TIME}
+		`,
+	}
+	assert.Error(t, p.Compile())
+
+	metricA, _ := p.ParseLine(`1.25 200 192.168.1.1 5.432µs`)
+	require.Nil(t, metricA)
+}
+
+func TestParsePatternsWithoutCustom(t *testing.T) {
+	p := &Parser{
+		Patterns: []string{"%{POSINT:ts:ts-epochnano} response_time=%{POSINT:response_time:int} mymetric=%{NUMBER:metric:float}"},
+	}
+	assert.NoError(t, p.Compile())
+
+	metricA, err := p.ParseLine(`1466004605359052000 response_time=20821 mymetric=10890.645`)
+	require.NotNil(t, metricA)
+	assert.NoError(t, err)
+	assert.Equal(t,
+		map[string]interface{}{
+			"response_time": int64(20821),
+			"metric":        float64(10890.645),
+		},
+		metricA.Fields())
+	assert.Equal(t, map[string]string{}, metricA.Tags())
+	assert.Equal(t, time.Unix(0, 1466004605359052000), metricA.Time())
+}
+
 func TestParseEpochNano(t *testing.T) {
 	p := &Parser{
 		Patterns: []string{"%{MYAPP}"},
@@ -277,6 +307,55 @@ func TestParseEpochErrors(t *testing.T) {
 	assert.NoError(t, err)
 }
 
+func TestParseGenericTimestamp(t *testing.T) {
+	p := &Parser{
+		Patterns: []string{`\[%{HTTPDATE:ts:ts}\] response_time=%{POSINT:response_time:int} mymetric=%{NUMBER:metric:float}`},
+	}
+	assert.NoError(t, p.Compile())
+
+	metricA, err := p.ParseLine(`[09/Jun/2016:03:37:03 +0000] response_time=20821 mymetric=10890.645`)
+	require.NotNil(t, metricA)
+	assert.NoError(t, err)
+	assert.Equal(t,
+		map[string]interface{}{
+			"response_time": int64(20821),
+			"metric":        float64(10890.645),
+		},
+		metricA.Fields())
+	assert.Equal(t, map[string]string{}, metricA.Tags())
+	assert.Equal(t, time.Unix(1465443423, 0).UTC(), metricA.Time().UTC())
+
+	metricB, err := p.ParseLine(`[09/Jun/2016:03:37:04 +0000] response_time=20821 mymetric=10890.645`)
+	require.NotNil(t, metricB)
+	assert.NoError(t, err)
+	assert.Equal(t,
+		map[string]interface{}{
+			"response_time": int64(20821),
+			"metric":        float64(10890.645),
+		},
+		metricB.Fields())
+	assert.Equal(t, map[string]string{}, metricB.Tags())
+	assert.Equal(t, time.Unix(1465443424, 0).UTC(), metricB.Time().UTC())
+}
+
+func TestParseGenericTimestampNotFound(t *testing.T) {
+	p := &Parser{
+		Patterns: []string{`\[%{NOTSPACE:ts:ts}\] response_time=%{POSINT:response_time:int} mymetric=%{NUMBER:metric:float}`},
+	}
+	assert.NoError(t, p.Compile())
+
+	metricA, err := p.ParseLine(`[foobar] response_time=20821 mymetric=10890.645`)
+	require.NotNil(t, metricA)
+	assert.NoError(t, err)
+	assert.Equal(t,
+		map[string]interface{}{
+			"response_time": int64(20821),
+			"metric":        float64(10890.645),
+		},
+		metricA.Fields())
+	assert.Equal(t, map[string]string{}, metricA.Tags())
+}
+
 func TestCompileFileAndParse(t *testing.T) {
 	p := &Parser{
 		Patterns:           []string{"%{TEST_LOG_A}", "%{TEST_LOG_B}"},
@@ -392,7 +471,7 @@ func TestParseErrors(t *testing.T) {
 			TEST_LOG_A %{HTTPDATE:ts:ts-httpd} %{WORD:myword:int} %{}
 		`,
 	}
-	assert.NoError(t, p.Compile())
+	assert.Error(t, p.Compile())
 	_, err := p.ParseLine(`[04/Jun/2016:12:41:45 +0100] notnumber 200 192.168.1.1 5.432µs 101`)
 	assert.Error(t, err)
 

plugins/inputs/logparser/grok/influx_patterns.go

@@ -55,18 +55,16 @@ EXAMPLE_LOG \[%{HTTPDATE:ts:ts-httpd}\] %{NUMBER:myfloat:float} %{RESPONSE_CODE}
 # Wider-ranging username matching vs. logstash built-in %{USER}
 NGUSERNAME [a-zA-Z\.\@\-\+_%]+
 NGUSER %{NGUSERNAME}
+# Wider-ranging client IP matching
+CLIENT (?:%{IPORHOST}|%{HOSTPORT}|::1)
 
 ##
 ## COMMON LOG PATTERNS
 ##
 
-# InfluxDB log patterns
-CLIENT (?:%{IPORHOST}|%{HOSTPORT}|::1)
-INFLUXDB_HTTPD_LOG \[httpd\] %{COMBINED_LOG_FORMAT} %{UUID:uuid:drop} %{NUMBER:response_time_us:int}
-
 # apache & nginx logs, this is also known as the "common log format"
 #   see https://en.wikipedia.org/wiki/Common_Log_Format
-COMMON_LOG_FORMAT %{CLIENT:client_ip} %{NGUSER:ident} %{NGUSER:auth} \[%{HTTPDATE:ts:ts-httpd}\] "(?:%{WORD:verb:tag} %{NOTSPACE:request}(?: HTTP/%{NUMBER:http_version:float})?|%{DATA})" %{NUMBER:resp_code:int} (?:%{NUMBER:resp_bytes:int}|-)
+COMMON_LOG_FORMAT %{CLIENT:client_ip} %{NGUSER:ident} %{NGUSER:auth} \[%{HTTPDATE:ts:ts-httpd}\] "(?:%{WORD:verb:tag} %{NOTSPACE:request}(?: HTTP/%{NUMBER:http_version:float})?|%{DATA})" %{NUMBER:resp_code:tag} (?:%{NUMBER:resp_bytes:int}|-)
 
 # Combined log format is the same as the common log format but with the addition
 # of two quoted strings at the end for "referrer" and "agent"

plugins/inputs/logparser/grok/patterns/influx-patterns

@@ -51,18 +51,16 @@ EXAMPLE_LOG \[%{HTTPDATE:ts:ts-httpd}\] %{NUMBER:myfloat:float} %{RESPONSE_CODE}
 # Wider-ranging username matching vs. logstash built-in %{USER}
 NGUSERNAME [a-zA-Z\.\@\-\+_%]+
 NGUSER %{NGUSERNAME}
+# Wider-ranging client IP matching
+CLIENT (?:%{IPORHOST}|%{HOSTPORT}|::1)
 
 ##
 ## COMMON LOG PATTERNS
 ##
 
-# InfluxDB log patterns
-CLIENT (?:%{IPORHOST}|%{HOSTPORT}|::1)
-INFLUXDB_HTTPD_LOG \[httpd\] %{COMBINED_LOG_FORMAT} %{UUID:uuid:drop} %{NUMBER:response_time_us:int}
-
 # apache & nginx logs, this is also known as the "common log format"
 #   see https://en.wikipedia.org/wiki/Common_Log_Format
-COMMON_LOG_FORMAT %{CLIENT:client_ip} %{NGUSER:ident} %{NGUSER:auth} \[%{HTTPDATE:ts:ts-httpd}\] "(?:%{WORD:verb:tag} %{NOTSPACE:request}(?: HTTP/%{NUMBER:http_version:float})?|%{DATA})" %{NUMBER:resp_code:int} (?:%{NUMBER:resp_bytes:int}|-)
+COMMON_LOG_FORMAT %{CLIENT:client_ip} %{NGUSER:ident} %{NGUSER:auth} \[%{HTTPDATE:ts:ts-httpd}\] "(?:%{WORD:verb:tag} %{NOTSPACE:request}(?: HTTP/%{NUMBER:http_version:float})?|%{DATA})" %{NUMBER:resp_code:tag} (?:%{NUMBER:resp_bytes:int}|-)
 
 # Combined log format is the same as the common log format but with the addition
 # of two quoted strings at the end for "referrer" and "agent"

plugins/inputs/logparser/logparser.go

@@ -9,6 +9,7 @@ import (
 	"github.com/hpcloud/tail"
 
 	"github.com/influxdata/telegraf"
+	"github.com/influxdata/telegraf/internal/errchan"
 	"github.com/influxdata/telegraf/internal/globpath"
 	"github.com/influxdata/telegraf/plugins/inputs"
 
@@ -44,7 +45,7 @@ const sampleConfig = `
   ##   /var/log/**.log     -> recursively find all .log files in /var/log
   ##   /var/log/*/*.log    -> find all .log files with a parent dir in /var/log
   ##   /var/log/apache.log -> only tail the apache log file
-  files = ["/var/log/influxdb/influxdb.log"]
+  files = ["/var/log/apache/access.log"]
   ## Read file from beginning.
   from_beginning = false
 
@@ -57,7 +58,9 @@ const sampleConfig = `
     ## Other common built-in patterns are:
     ##   %{COMMON_LOG_FORMAT}   (plain apache & nginx access logs)
     ##   %{COMBINED_LOG_FORMAT} (access logs + referrer & agent)
-    patterns = ["%{INFLUXDB_HTTPD_LOG}"]
+    patterns = ["%{COMBINED_LOG_FORMAT}"]
+    ## Name of the outputted measurement name.
+    measurement = "apache_access_log"
     ## Full path(s) to custom pattern files.
     custom_pattern_files = []
     ## Custom patterns can also be defined here. Put one pattern per line.
@@ -108,11 +111,15 @@ func (l *LogParserPlugin) Start(acc telegraf.Accumulator) error {
 	}
 
 	// compile log parser patterns:
+	errChan := errchan.New(len(l.parsers))
 	for _, parser := range l.parsers {
 		if err := parser.Compile(); err != nil {
-			return err
+			errChan.C <- err
 		}
 	}
+	if err := errChan.Error(); err != nil {
+		return err
+	}
 
 	var seek tail.SeekInfo
 	if !l.FromBeginning {
@@ -123,24 +130,25 @@ func (l *LogParserPlugin) Start(acc telegraf.Accumulator) error {
 	l.wg.Add(1)
 	go l.parser()
 
-	var errS string
 	// Create a "tailer" for each file
 	for _, filepath := range l.Files {
 		g, err := globpath.Compile(filepath)
 		if err != nil {
 			log.Printf("ERROR Glob %s failed to compile, %s", filepath, err)
+			continue
 		}
-		for file, _ := range g.Match() {
+		files := g.Match()
+		errChan = errchan.New(len(files))
+		for file, _ := range files {
 			tailer, err := tail.TailFile(file,
 				tail.Config{
-					ReOpen:   true,
-					Follow:   true,
-					Location: &seek,
+					ReOpen:    true,
+					Follow:    true,
+					Location:  &seek,
+					MustExist: true,
 				})
-			if err != nil {
-				errS += err.Error() + " "
-				continue
-			}
+			errChan.C <- err
+
 			// create a goroutine for each "tailer"
 			l.wg.Add(1)
 			go l.receiver(tailer)
@@ -148,10 +156,7 @@ func (l *LogParserPlugin) Start(acc telegraf.Accumulator) error {
 		}
 	}
 
-	if errS != "" {
-		return fmt.Errorf(errS)
-	}
-	return nil
+	return errChan.Error()
 }
 
 // receiver is launched as a goroutine to continuously watch a tailed logfile
@@ -199,8 +204,6 @@ func (l *LogParserPlugin) parser() {
 				if m != nil {
 					l.acc.AddFields(m.Name(), m.Fields(), m.Tags(), m.Time())
 				}
-			} else {
-				log.Printf("Malformed log line in [%s], Error: %s\n", line, err)
 			}
 		}
 	}

plugins/inputs/logparser/logparser_test.go

@@ -37,7 +37,7 @@ func TestGrokParseLogFilesNonExistPattern(t *testing.T) {
 	}
 
 	acc := testutil.Accumulator{}
-	assert.NoError(t, logparser.Start(&acc))
+	assert.Error(t, logparser.Start(&acc))
 
 	time.Sleep(time.Millisecond * 500)
 	logparser.Stop()
@@ -80,6 +80,8 @@ func TestGrokParseLogFiles(t *testing.T) {
 		map[string]string{})
 }
 
+// Test that test_a.log line gets parsed even though we don't have the correct
+// pattern available for test_b.log
 func TestGrokParseLogFilesOneBad(t *testing.T) {
 	thisdir := getCurrentDir()
 	p := &grok.Parser{
@@ -90,11 +92,12 @@ func TestGrokParseLogFilesOneBad(t *testing.T) {
 
 	logparser := &LogParserPlugin{
 		FromBeginning: true,
-		Files:         []string{thisdir + "grok/testdata/*.log"},
+		Files:         []string{thisdir + "grok/testdata/test_a.log"},
 		GrokParser:    p,
 	}
 
 	acc := testutil.Accumulator{}
+	acc.SetDebug(true)
 	assert.NoError(t, logparser.Start(&acc))
 
 	time.Sleep(time.Millisecond * 500)

plugins/inputs/memcached/memcached.go

@@ -9,6 +9,7 @@ import (
 	"time"
 
 	"github.com/influxdata/telegraf"
+	"github.com/influxdata/telegraf/internal/errchan"
 	"github.com/influxdata/telegraf/plugins/inputs"
 )
 
@@ -73,19 +74,16 @@ func (m *Memcached) Gather(acc telegraf.Accumulator) error {
 		return m.gatherServer(":11211", false, acc)
 	}
 
+	errChan := errchan.New(len(m.Servers) + len(m.UnixSockets))
 	for _, serverAddress := range m.Servers {
-		if err := m.gatherServer(serverAddress, false, acc); err != nil {
-			return err
-		}
+		errChan.C <- m.gatherServer(serverAddress, false, acc)
 	}
 
 	for _, unixAddress := range m.UnixSockets {
-		if err := m.gatherServer(unixAddress, true, acc); err != nil {
-			return err
-		}
+		errChan.C <- m.gatherServer(unixAddress, true, acc)
 	}
 
-	return nil
+	return errChan.Error()
 }
 
 func (m *Memcached) gatherServer(

plugins/inputs/mesos/README.md

@@ -1,6 +1,6 @@
 # Mesos Input Plugin
 
-This input plugin gathers metrics from Mesos (*currently only Mesos masters*).
+This input plugin gathers metrics from Mesos.
 For more information, please check the [Mesos Observability Metrics](http://mesos.apache.org/documentation/latest/monitoring/) page.
 
 ### Configuration:
@@ -8,14 +8,41 @@ For more information, please check the [Mesos Observability Metrics](http://meso
 ```toml
 # Telegraf plugin for gathering metrics from N Mesos masters
 [[inputs.mesos]]
-  # Timeout, in ms.
+  ## Timeout, in ms.
   timeout = 100
-  # A list of Mesos masters, default value is localhost:5050.
+  ## A list of Mesos masters.
   masters = ["localhost:5050"]
-  # Metrics groups to be collected, by default, all enabled.
-  master_collections = ["resources","master","system","slaves","frameworks","messages","evqueue","registrar"]
+  ## Master metrics groups to be collected, by default, all enabled.
+  master_collections = [
+    "resources",
+    "master",
+    "system",
+    "agents",
+    "frameworks",
+    "tasks",
+    "messages",
+    "evqueue",
+    "registrar",
+  ]
+  ## A list of Mesos slaves, default is []
+  # slaves = []
+  ## Slave metrics groups to be collected, by default, all enabled.
+  # slave_collections = [
+  #   "resources",
+  #   "agent",
+  #   "system",
+  #   "executors",
+  #   "tasks",
+  #   "messages",
+  # ]
+  ## Include mesos tasks statistics, default is false
+  # slave_tasks = true
 ```
 
+By default this plugin is not configured to gather metrics from mesos. Since a mesos cluster can be deployed in numerous ways it does not provide any default
+values. User needs to specify master/slave nodes this plugin will gather metrics from. Additionally, enabling `slave_tasks` will allow
+gathering metrics from tasks running on specified slaves (this option is disabled by default).
+
 ### Measurements & Fields:
 
 Mesos master metric groups
@@ -33,6 +60,12 @@ Mesos master metric groups
     - master/disk_revocable_percent
     - master/disk_revocable_total
     - master/disk_revocable_used
+    - master/gpus_percent
+    - master/gpus_used
+    - master/gpus_total
+    - master/gpus_revocable_percent
+    - master/gpus_revocable_total
+    - master/gpus_revocable_used
     - master/mem_percent
     - master/mem_used
     - master/mem_total
@@ -136,17 +169,111 @@ Mesos master metric groups
     - registrar/state_store_ms/p999
     - registrar/state_store_ms/p9999
 
+Mesos slave metric groups
+- resources
+    - slave/cpus_percent
+    - slave/cpus_used
+    - slave/cpus_total
+    - slave/cpus_revocable_percent
+    - slave/cpus_revocable_total
+    - slave/cpus_revocable_used
+    - slave/disk_percent
+    - slave/disk_used
+    - slave/disk_total
+    - slave/disk_revocable_percent
+    - slave/disk_revocable_total
+    - slave/disk_revocable_used
+    - slave/gpus_percent
+    - slave/gpus_used
+    - slave/gpus_total,
+    - slave/gpus_revocable_percent
+    - slave/gpus_revocable_total
+    - slave/gpus_revocable_used
+    - slave/mem_percent
+    - slave/mem_used
+    - slave/mem_total
+    - slave/mem_revocable_percent
+    - slave/mem_revocable_total
+    - slave/mem_revocable_used
+
+- agent
+    - slave/registered
+    - slave/uptime_secs
+
+- system
+    - system/cpus_total
+    - system/load_15min
+    - system/load_5min
+    - system/load_1min
+    - system/mem_free_bytes
+    - system/mem_total_bytes
+
+- executors
+    - containerizer/mesos/container_destroy_errors
+    - slave/container_launch_errors
+    - slave/executors_preempted
+    - slave/frameworks_active
+    - slave/executor_directory_max_allowed_age_secs
+    - slave/executors_registering
+    - slave/executors_running
+    - slave/executors_terminated
+    - slave/executors_terminating
+    - slave/recovery_errors
+
+- tasks
+    - slave/tasks_failed
+    - slave/tasks_finished
+    - slave/tasks_killed
+    - slave/tasks_lost
+    - slave/tasks_running
+    - slave/tasks_staging
+    - slave/tasks_starting
+
+- messages
+    - slave/invalid_framework_messages
+    - slave/invalid_status_updates
+    - slave/valid_framework_messages
+    - slave/valid_status_updates
+
+Mesos tasks metric groups
+
+- executor_id
+- executor_name
+- framework_id
+- source
+- statistics (all metrics below will have `statistics_` prefix included in their names
+    - cpus_limit
+    - cpus_system_time_secs
+    - cpus_user_time_secs
+    - mem_anon_bytes
+    - mem_cache_bytes
+    - mem_critical_pressure_counter
+    - mem_file_bytes
+    - mem_limit_bytes
+    - mem_low_pressure_counter
+    - mem_mapped_file_bytes
+    - mem_medium_pressure_counter
+    - mem_rss_bytes
+    - mem_swap_bytes
+    - mem_total_bytes
+    - mem_total_memsw_bytes
+    - mem_unevictable_bytes
+    - timestamp
+
 ### Tags:
 
-- All measurements have the following tags:
+- All master/slave measurements have the following tags:
+    - server
+    - role (master/slave)
+
+- Tasks  measurements have the following tags:
     - server
 
 ### Example Output:
-
 ```
 $ telegraf -config ~/mesos.conf -input-filter mesos -test
 * Plugin: mesos, Collection 1
-mesos,server=172.17.8.101 allocator/event_queue_dispatches=0,master/cpus_percent=0,
+mesos,host=172.17.8.102,server=172.17.8.101 allocator/event_queue_dispatches=0,master/cpus_percent=0,
 master/cpus_revocable_percent=0,master/cpus_revocable_total=0,
 master/cpus_revocable_used=0,master/cpus_total=2,
 master/cpus_used=0,master/disk_percent=0,master/disk_revocable_percent=0,
@@ -163,3 +290,16 @@ master/mem_revocable_used=0,master/mem_total=1002,
 master/mem_used=0,master/messages_authenticate=0,
 master/messages_deactivate_framework=0 ...
 ```
+
+Meoso tasks metrics (if enabled):
+```
+mesos-tasks,host=172.17.8.102,server=172.17.8.101,task_id=hello-world.e4b5b497-2ccd-11e6-a659-0242fb222ce2
+statistics_cpus_limit=0.2,statistics_cpus_system_time_secs=142.49,statistics_cpus_user_time_secs=388.14,
+statistics_mem_anon_bytes=359129088,statistics_mem_cache_bytes=3964928,
+statistics_mem_critical_pressure_counter=0,statistics_mem_file_bytes=3964928,
+statistics_mem_limit_bytes=767557632,statistics_mem_low_pressure_counter=0,
+statistics_mem_mapped_file_bytes=114688,statistics_mem_medium_pressure_counter=0,
+statistics_mem_rss_bytes=359129088,statistics_mem_swap_bytes=0,statistics_mem_total_bytes=363094016,
+statistics_mem_total_memsw_bytes=363094016,statistics_mem_unevictable_bytes=0,
+statistics_timestamp=1465486052.70525 1465486053052811792...
+```

plugins/inputs/mesos/mesos.go

@@ -17,33 +17,57 @@ import (
 	jsonparser "github.com/influxdata/telegraf/plugins/parsers/json"
 )
 
+type Role string
+
+const (
+	MASTER Role = "master"
+	SLAVE       = "slave"
+)
+
 type Mesos struct {
 	Timeout    int
 	Masters    []string
 	MasterCols []string `toml:"master_collections"`
+	Slaves     []string
+	SlaveCols  []string `toml:"slave_collections"`
+	SlaveTasks bool
 }
 
-var defaultMetrics = []string{
-	"resources", "master", "system", "slaves", "frameworks",
-	"tasks", "messages", "evqueue", "messages", "registrar",
+var allMetrics = map[Role][]string{
+	MASTER: []string{"resources", "master", "system", "agents", "frameworks", "tasks", "messages", "evqueue", "registrar"},
+	SLAVE:  []string{"resources", "agent", "system", "executors", "tasks", "messages"},
 }
 
 var sampleConfig = `
-  # Timeout, in ms.
+  ## Timeout, in ms.
   timeout = 100
-  # A list of Mesos masters, default value is localhost:5050.
+  ## A list of Mesos masters.
   masters = ["localhost:5050"]
-  # Metrics groups to be collected, by default, all enabled.
+  ## Master metrics groups to be collected, by default, all enabled.
   master_collections = [
     "resources",
     "master",
     "system",
-    "slaves",
+    "agents",
     "frameworks",
+    "tasks",
     "messages",
     "evqueue",
     "registrar",
   ]
+  ## A list of Mesos slaves, default is []
+  # slaves = []
+  ## Slave metrics groups to be collected, by default, all enabled.
+  # slave_collections = [
+  #   "resources",
+  #   "agent",
+  #   "system",
+  #   "executors",
+  #   "tasks",
+  #   "messages",
+  # ]
+  ## Include mesos tasks statistics, default is false
+  # slave_tasks = true
 `
 
 // SampleConfig returns a sample configuration block
@@ -56,21 +80,54 @@ func (m *Mesos) Description() string {
 	return "Telegraf plugin for gathering metrics from N Mesos masters"
 }
 
+func (m *Mesos) SetDefaults() {
+	if len(m.MasterCols) == 0 {
+		m.MasterCols = allMetrics[MASTER]
+	}
+
+	if len(m.SlaveCols) == 0 {
+		m.SlaveCols = allMetrics[SLAVE]
+	}
+
+	if m.Timeout == 0 {
+		log.Println("[mesos] Missing timeout value, setting default value (100ms)")
+		m.Timeout = 100
+	}
+}
+
 // Gather() metrics from given list of Mesos Masters
 func (m *Mesos) Gather(acc telegraf.Accumulator) error {
 	var wg sync.WaitGroup
 	var errorChannel chan error
 
-	if len(m.Masters) == 0 {
-		m.Masters = []string{"localhost:5050"}
-	}
+	m.SetDefaults()
 
-	errorChannel = make(chan error, len(m.Masters)*2)
+	errorChannel = make(chan error, len(m.Masters)+2*len(m.Slaves))
 
 	for _, v := range m.Masters {
 		wg.Add(1)
 		go func(c string) {
-			errorChannel <- m.gatherMetrics(c, acc)
+			errorChannel <- m.gatherMainMetrics(c, ":5050", MASTER, acc)
+			wg.Done()
+			return
+		}(v)
+	}
+
+	for _, v := range m.Slaves {
+		wg.Add(1)
+		go func(c string) {
+			errorChannel <- m.gatherMainMetrics(c, ":5051", MASTER, acc)
+			wg.Done()
+			return
+		}(v)
+
+		if !m.SlaveTasks {
+			continue
+		}
+
+		wg.Add(1)
+		go func(c string) {
+			errorChannel <- m.gatherSlaveTaskMetrics(c, ":5051", acc)
 			wg.Done()
 			return
 		}(v)
@@ -94,7 +151,7 @@ func (m *Mesos) Gather(acc telegraf.Accumulator) error {
 }
 
 // metricsDiff() returns set names for removal
-func metricsDiff(w []string) []string {
+func metricsDiff(role Role, w []string) []string {
 	b := []string{}
 	s := make(map[string]bool)
 
@@ -106,7 +163,7 @@ func metricsDiff(w []string) []string {
 		s[v] = true
 	}
 
-	for _, d := range defaultMetrics {
+	for _, d := range allMetrics[role] {
 		if _, ok := s[d]; !ok {
 			b = append(b, d)
 		}
@@ -116,156 +173,239 @@ func metricsDiff(w []string) []string {
 }
 
 // masterBlocks serves as kind of metrics registry groupping them in sets
-func masterBlocks(g string) []string {
+func getMetrics(role Role, group string) []string {
 	var m map[string][]string
 
 	m = make(map[string][]string)
 
-	m["resources"] = []string{
-		"master/cpus_percent",
-		"master/cpus_used",
-		"master/cpus_total",
-		"master/cpus_revocable_percent",
-		"master/cpus_revocable_total",
-		"master/cpus_revocable_used",
-		"master/disk_percent",
-		"master/disk_used",
-		"master/disk_total",
-		"master/disk_revocable_percent",
-		"master/disk_revocable_total",
-		"master/disk_revocable_used",
-		"master/mem_percent",
-		"master/mem_used",
-		"master/mem_total",
-		"master/mem_revocable_percent",
-		"master/mem_revocable_total",
-		"master/mem_revocable_used",
+	if role == MASTER {
+		m["resources"] = []string{
+			"master/cpus_percent",
+			"master/cpus_used",
+			"master/cpus_total",
+			"master/cpus_revocable_percent",
+			"master/cpus_revocable_total",
+			"master/cpus_revocable_used",
+			"master/disk_percent",
+			"master/disk_used",
+			"master/disk_total",
+			"master/disk_revocable_percent",
+			"master/disk_revocable_total",
+			"master/disk_revocable_used",
+			"master/gpus_percent",
+			"master/gpus_used",
+			"master/gpus_total",
+			"master/gpus_revocable_percent",
+			"master/gpus_revocable_total",
+			"master/gpus_revocable_used",
+			"master/mem_percent",
+			"master/mem_used",
+			"master/mem_total",
+			"master/mem_revocable_percent",
+			"master/mem_revocable_total",
+			"master/mem_revocable_used",
+		}
+
+		m["master"] = []string{
+			"master/elected",
+			"master/uptime_secs",
+		}
+
+		m["system"] = []string{
+			"system/cpus_total",
+			"system/load_15min",
+			"system/load_5min",
+			"system/load_1min",
+			"system/mem_free_bytes",
+			"system/mem_total_bytes",
+		}
+
+		m["agents"] = []string{
+			"master/slave_registrations",
+			"master/slave_removals",
+			"master/slave_reregistrations",
+			"master/slave_shutdowns_scheduled",
+			"master/slave_shutdowns_canceled",
+			"master/slave_shutdowns_completed",
+			"master/slaves_active",
+			"master/slaves_connected",
+			"master/slaves_disconnected",
+			"master/slaves_inactive",
+		}
+
+		m["frameworks"] = []string{
+			"master/frameworks_active",
+			"master/frameworks_connected",
+			"master/frameworks_disconnected",
+			"master/frameworks_inactive",
+			"master/outstanding_offers",
+		}
+
+		m["tasks"] = []string{
+			"master/tasks_error",
+			"master/tasks_failed",
+			"master/tasks_finished",
+			"master/tasks_killed",
+			"master/tasks_lost",
+			"master/tasks_running",
+			"master/tasks_staging",
+			"master/tasks_starting",
+		}
+
+		m["messages"] = []string{
+			"master/invalid_executor_to_framework_messages",
+			"master/invalid_framework_to_executor_messages",
+			"master/invalid_status_update_acknowledgements",
+			"master/invalid_status_updates",
+			"master/dropped_messages",
+			"master/messages_authenticate",
+			"master/messages_deactivate_framework",
+			"master/messages_decline_offers",
+			"master/messages_executor_to_framework",
+			"master/messages_exited_executor",
+			"master/messages_framework_to_executor",
+			"master/messages_kill_task",
+			"master/messages_launch_tasks",
+			"master/messages_reconcile_tasks",
+			"master/messages_register_framework",
+			"master/messages_register_slave",
+			"master/messages_reregister_framework",
+			"master/messages_reregister_slave",
+			"master/messages_resource_request",
+			"master/messages_revive_offers",
+			"master/messages_status_update",
+			"master/messages_status_update_acknowledgement",
+			"master/messages_unregister_framework",
+			"master/messages_unregister_slave",
+			"master/messages_update_slave",
+			"master/recovery_slave_removals",
+			"master/slave_removals/reason_registered",
+			"master/slave_removals/reason_unhealthy",
+			"master/slave_removals/reason_unregistered",
+			"master/valid_framework_to_executor_messages",
+			"master/valid_status_update_acknowledgements",
+			"master/valid_status_updates",
+			"master/task_lost/source_master/reason_invalid_offers",
+			"master/task_lost/source_master/reason_slave_removed",
+			"master/task_lost/source_slave/reason_executor_terminated",
+			"master/valid_executor_to_framework_messages",
+		}
+
+		m["evqueue"] = []string{
+			"master/event_queue_dispatches",
+			"master/event_queue_http_requests",
+			"master/event_queue_messages",
+		}
+
+		m["registrar"] = []string{
+			"registrar/state_fetch_ms",
+			"registrar/state_store_ms",
+			"registrar/state_store_ms/max",
+			"registrar/state_store_ms/min",
+			"registrar/state_store_ms/p50",
+			"registrar/state_store_ms/p90",
+			"registrar/state_store_ms/p95",
+			"registrar/state_store_ms/p99",
+			"registrar/state_store_ms/p999",
+			"registrar/state_store_ms/p9999",
+		}
+	} else if role == SLAVE {
+		m["resources"] = []string{
+			"slave/cpus_percent",
+			"slave/cpus_used",
+			"slave/cpus_total",
+			"slave/cpus_revocable_percent",
+			"slave/cpus_revocable_total",
+			"slave/cpus_revocable_used",
+			"slave/disk_percent",
+			"slave/disk_used",
+			"slave/disk_total",
+			"slave/disk_revocable_percent",
+			"slave/disk_revocable_total",
+			"slave/disk_revocable_used",
+			"slave/gpus_percent",
+			"slave/gpus_used",
+			"slave/gpus_total",
+			"slave/gpus_revocable_percent",
+			"slave/gpus_revocable_total",
+			"slave/gpus_revocable_used",
+			"slave/mem_percent",
+			"slave/mem_used",
+			"slave/mem_total",
+			"slave/mem_revocable_percent",
+			"slave/mem_revocable_total",
+			"slave/mem_revocable_used",
+		}
+
+		m["agent"] = []string{
+			"slave/registered",
+			"slave/uptime_secs",
+		}
+
+		m["system"] = []string{
+			"system/cpus_total",
+			"system/load_15min",
+			"system/load_5min",
+			"system/load_1min",
+			"system/mem_free_bytes",
+			"system/mem_total_bytes",
+		}
+
+		m["executors"] = []string{
+			"containerizer/mesos/container_destroy_errors",
+			"slave/container_launch_errors",
+			"slave/executors_preempted",
+			"slave/frameworks_active",
+			"slave/executor_directory_max_allowed_age_secs",
+			"slave/executors_registering",
+			"slave/executors_running",
+			"slave/executors_terminated",
+			"slave/executors_terminating",
+			"slave/recovery_errors",
+		}
+
+		m["tasks"] = []string{
+			"slave/tasks_failed",
+			"slave/tasks_finished",
+			"slave/tasks_killed",
+			"slave/tasks_lost",
+			"slave/tasks_running",
+			"slave/tasks_staging",
+			"slave/tasks_starting",
+		}
+
+		m["messages"] = []string{
+			"slave/invalid_framework_messages",
+			"slave/invalid_status_updates",
+			"slave/valid_framework_messages",
+			"slave/valid_status_updates",
+		}
 	}
 
-	m["master"] = []string{
-		"master/elected",
-		"master/uptime_secs",
-	}
-
-	m["system"] = []string{
-		"system/cpus_total",
-		"system/load_15min",
-		"system/load_5min",
-		"system/load_1min",
-		"system/mem_free_bytes",
-		"system/mem_total_bytes",
-	}
-
-	m["slaves"] = []string{
-		"master/slave_registrations",
-		"master/slave_removals",
-		"master/slave_reregistrations",
-		"master/slave_shutdowns_scheduled",
-		"master/slave_shutdowns_canceled",
-		"master/slave_shutdowns_completed",
-		"master/slaves_active",
-		"master/slaves_connected",
-		"master/slaves_disconnected",
-		"master/slaves_inactive",
-	}
-
-	m["frameworks"] = []string{
-		"master/frameworks_active",
-		"master/frameworks_connected",
-		"master/frameworks_disconnected",
-		"master/frameworks_inactive",
-		"master/outstanding_offers",
-	}
-
-	m["tasks"] = []string{
-		"master/tasks_error",
-		"master/tasks_failed",
-		"master/tasks_finished",
-		"master/tasks_killed",
-		"master/tasks_lost",
-		"master/tasks_running",
-		"master/tasks_staging",
-		"master/tasks_starting",
-	}
-
-	m["messages"] = []string{
-		"master/invalid_executor_to_framework_messages",
-		"master/invalid_framework_to_executor_messages",
-		"master/invalid_status_update_acknowledgements",
-		"master/invalid_status_updates",
-		"master/dropped_messages",
-		"master/messages_authenticate",
-		"master/messages_deactivate_framework",
-		"master/messages_decline_offers",
-		"master/messages_executor_to_framework",
-		"master/messages_exited_executor",
-		"master/messages_framework_to_executor",
-		"master/messages_kill_task",
-		"master/messages_launch_tasks",
-		"master/messages_reconcile_tasks",
-		"master/messages_register_framework",
-		"master/messages_register_slave",
-		"master/messages_reregister_framework",
-		"master/messages_reregister_slave",
-		"master/messages_resource_request",
-		"master/messages_revive_offers",
-		"master/messages_status_update",
-		"master/messages_status_update_acknowledgement",
-		"master/messages_unregister_framework",
-		"master/messages_unregister_slave",
-		"master/messages_update_slave",
-		"master/recovery_slave_removals",
-		"master/slave_removals/reason_registered",
-		"master/slave_removals/reason_unhealthy",
-		"master/slave_removals/reason_unregistered",
-		"master/valid_framework_to_executor_messages",
-		"master/valid_status_update_acknowledgements",
-		"master/valid_status_updates",
-		"master/task_lost/source_master/reason_invalid_offers",
-		"master/task_lost/source_master/reason_slave_removed",
-		"master/task_lost/source_slave/reason_executor_terminated",
-		"master/valid_executor_to_framework_messages",
-	}
-
-	m["evqueue"] = []string{
-		"master/event_queue_dispatches",
-		"master/event_queue_http_requests",
-		"master/event_queue_messages",
-	}
-
-	m["registrar"] = []string{
-		"registrar/state_fetch_ms",
-		"registrar/state_store_ms",
-		"registrar/state_store_ms/max",
-		"registrar/state_store_ms/min",
-		"registrar/state_store_ms/p50",
-		"registrar/state_store_ms/p90",
-		"registrar/state_store_ms/p95",
-		"registrar/state_store_ms/p99",
-		"registrar/state_store_ms/p999",
-		"registrar/state_store_ms/p9999",
-	}
-
-	ret, ok := m[g]
+	ret, ok := m[group]
 
 	if !ok {
-		log.Println("[mesos] Unkown metrics group: ", g)
+		log.Printf("[mesos] Unkown %s metrics group: %s\n", role, group)
 		return []string{}
 	}
 
 	return ret
 }
 
-// removeGroup(), remove unwanted sets
-func (m *Mesos) removeGroup(j *map[string]interface{}) {
+func (m *Mesos) filterMetrics(role Role, metrics *map[string]interface{}) {
 	var ok bool
+	var selectedMetrics []string
 
-	b := metricsDiff(m.MasterCols)
+	if role == MASTER {
+		selectedMetrics = m.MasterCols
+	} else if role == SLAVE {
+		selectedMetrics = m.SlaveCols
+	}
 
-	for _, k := range b {
-		for _, v := range masterBlocks(k) {
-			if _, ok = (*j)[v]; ok {
-				delete((*j), v)
+	for _, k := range metricsDiff(role, selectedMetrics) {
+		for _, v := range getMetrics(role, k) {
+			if _, ok = (*metrics)[v]; ok {
+				delete((*metrics), v)
 			}
 		}
 	}
@@ -280,23 +420,66 @@ var client = &http.Client{
 	Timeout:   time.Duration(4 * time.Second),
 }
 
-// This should not belong to the object
-func (m *Mesos) gatherMetrics(a string, acc telegraf.Accumulator) error {
-	var jsonOut map[string]interface{}
+func (m *Mesos) gatherSlaveTaskMetrics(address string, defaultPort string, acc telegraf.Accumulator) error {
+	var metrics []map[string]interface{}
 
-	host, _, err := net.SplitHostPort(a)
+	host, _, err := net.SplitHostPort(address)
 	if err != nil {
-		host = a
-		a = a + ":5050"
+		host = address
+		address = address + defaultPort
 	}
 
 	tags := map[string]string{
 		"server": host,
 	}
 
-	if m.Timeout == 0 {
-		log.Println("[mesos] Missing timeout value, setting default value (100ms)")
-		m.Timeout = 100
+	ts := strconv.Itoa(m.Timeout) + "ms"
+
+	resp, err := client.Get("http://" + address + "/monitor/statistics?timeout=" + ts)
+
+	if err != nil {
+		return err
+	}
+
+	data, err := ioutil.ReadAll(resp.Body)
+	resp.Body.Close()
+	if err != nil {
+		return err
+	}
+
+	if err = json.Unmarshal([]byte(data), &metrics); err != nil {
+		return errors.New("Error decoding JSON response")
+	}
+
+	for _, task := range metrics {
+		tags["task_id"] = task["executor_id"].(string)
+
+		jf := jsonparser.JSONFlattener{}
+		err = jf.FlattenJSON("", task)
+
+		if err != nil {
+			return err
+		}
+
+		acc.AddFields("mesos-tasks", jf.Fields, tags)
+	}
+
+	return nil
+}
+
+// This should not belong to the object
+func (m *Mesos) gatherMainMetrics(a string, defaultPort string, role Role, acc telegraf.Accumulator) error {
+	var jsonOut map[string]interface{}
+
+	host, _, err := net.SplitHostPort(a)
+	if err != nil {
+		host = a
+		a = a + defaultPort
+	}
+
+	tags := map[string]string{
+		"server": host,
+		"role":   string(role),
 	}
 
 	ts := strconv.Itoa(m.Timeout) + "ms"
@@ -317,7 +500,7 @@ func (m *Mesos) gatherMetrics(a string, acc telegraf.Accumulator) error {
 		return errors.New("Error decoding JSON response")
 	}
 
-	m.removeGroup(&jsonOut)
+	m.filterMetrics(role, &jsonOut)
 
 	jf := jsonparser.JSONFlattener{}
 

plugins/inputs/mesos/mesos_test.go

@@ -2,70 +2,275 @@ package mesos
 
 import (
 	"encoding/json"
+	"fmt"
 	"math/rand"
 	"net/http"
 	"net/http/httptest"
 	"os"
 	"testing"
 
+	jsonparser "github.com/influxdata/telegraf/plugins/parsers/json"
 	"github.com/influxdata/telegraf/testutil"
 )
 
-var mesosMetrics map[string]interface{}
-var ts *httptest.Server
+var masterMetrics map[string]interface{}
+var masterTestServer *httptest.Server
+var slaveMetrics map[string]interface{}
+var slaveTaskMetrics map[string]interface{}
+var slaveTestServer *httptest.Server
+
+func randUUID() string {
+	b := make([]byte, 16)
+	rand.Read(b)
+	return fmt.Sprintf("%x-%x-%x-%x-%x", b[0:4], b[4:6], b[6:8], b[8:10], b[10:])
+}
 
 func generateMetrics() {
-	mesosMetrics = make(map[string]interface{})
+	masterMetrics = make(map[string]interface{})
 
-	metricNames := []string{"master/cpus_percent", "master/cpus_used", "master/cpus_total",
-		"master/cpus_revocable_percent", "master/cpus_revocable_total", "master/cpus_revocable_used",
-		"master/disk_percent", "master/disk_used", "master/disk_total", "master/disk_revocable_percent",
-		"master/disk_revocable_total", "master/disk_revocable_used", "master/mem_percent",
-		"master/mem_used", "master/mem_total", "master/mem_revocable_percent", "master/mem_revocable_total",
-		"master/mem_revocable_used", "master/elected", "master/uptime_secs", "system/cpus_total",
-		"system/load_15min", "system/load_5min", "system/load_1min", "system/mem_free_bytes",
-		"system/mem_total_bytes", "master/slave_registrations", "master/slave_removals",
-		"master/slave_reregistrations", "master/slave_shutdowns_scheduled", "master/slave_shutdowns_canceled",
-		"master/slave_shutdowns_completed", "master/slaves_active", "master/slaves_connected",
-		"master/slaves_disconnected", "master/slaves_inactive", "master/frameworks_active",
-		"master/frameworks_connected", "master/frameworks_disconnected", "master/frameworks_inactive",
-		"master/outstanding_offers", "master/tasks_error", "master/tasks_failed", "master/tasks_finished",
-		"master/tasks_killed", "master/tasks_lost", "master/tasks_running", "master/tasks_staging",
-		"master/tasks_starting", "master/invalid_executor_to_framework_messages", "master/invalid_framework_to_executor_messages",
-		"master/invalid_status_update_acknowledgements", "master/invalid_status_updates",
-		"master/dropped_messages", "master/messages_authenticate", "master/messages_deactivate_framework",
-		"master/messages_decline_offers", "master/messages_executor_to_framework", "master/messages_exited_executor",
-		"master/messages_framework_to_executor", "master/messages_kill_task", "master/messages_launch_tasks",
-		"master/messages_reconcile_tasks", "master/messages_register_framework", "master/messages_register_slave",
-		"master/messages_reregister_framework", "master/messages_reregister_slave", "master/messages_resource_request",
-		"master/messages_revive_offers", "master/messages_status_update", "master/messages_status_update_acknowledgement",
-		"master/messages_unregister_framework", "master/messages_unregister_slave", "master/messages_update_slave",
-		"master/recovery_slave_removals", "master/slave_removals/reason_registered", "master/slave_removals/reason_unhealthy",
-		"master/slave_removals/reason_unregistered", "master/valid_framework_to_executor_messages", "master/valid_status_update_acknowledgements",
-		"master/valid_status_updates", "master/task_lost/source_master/reason_invalid_offers",
-		"master/task_lost/source_master/reason_slave_removed", "master/task_lost/source_slave/reason_executor_terminated",
-		"master/valid_executor_to_framework_messages", "master/event_queue_dispatches",
-		"master/event_queue_http_requests", "master/event_queue_messages", "registrar/state_fetch_ms",
-		"registrar/state_store_ms", "registrar/state_store_ms/max", "registrar/state_store_ms/min",
-		"registrar/state_store_ms/p50", "registrar/state_store_ms/p90", "registrar/state_store_ms/p95",
-		"registrar/state_store_ms/p99", "registrar/state_store_ms/p999", "registrar/state_store_ms/p9999"}
+	metricNames := []string{
+		// resources
+		"master/cpus_percent",
+		"master/cpus_used",
+		"master/cpus_total",
+		"master/cpus_revocable_percent",
+		"master/cpus_revocable_total",
+		"master/cpus_revocable_used",
+		"master/disk_percent",
+		"master/disk_used",
+		"master/disk_total",
+		"master/disk_revocable_percent",
+		"master/disk_revocable_total",
+		"master/disk_revocable_used",
+		"master/gpus_percent",
+		"master/gpus_used",
+		"master/gpus_total",
+		"master/gpus_revocable_percent",
+		"master/gpus_revocable_total",
+		"master/gpus_revocable_used",
+		"master/mem_percent",
+		"master/mem_used",
+		"master/mem_total",
+		"master/mem_revocable_percent",
+		"master/mem_revocable_total",
+		"master/mem_revocable_used",
+		// master
+		"master/elected",
+		"master/uptime_secs",
+		// system
+		"system/cpus_total",
+		"system/load_15min",
+		"system/load_5min",
+		"system/load_1min",
+		"system/mem_free_bytes",
+		"system/mem_total_bytes",
+		// agents
+		"master/slave_registrations",
+		"master/slave_removals",
+		"master/slave_reregistrations",
+		"master/slave_shutdowns_scheduled",
+		"master/slave_shutdowns_canceled",
+		"master/slave_shutdowns_completed",
+		"master/slaves_active",
+		"master/slaves_connected",
+		"master/slaves_disconnected",
+		"master/slaves_inactive",
+		// frameworks
+		"master/frameworks_active",
+		"master/frameworks_connected",
+		"master/frameworks_disconnected",
+		"master/frameworks_inactive",
+		"master/outstanding_offers",
+		// tasks
+		"master/tasks_error",
+		"master/tasks_failed",
+		"master/tasks_finished",
+		"master/tasks_killed",
+		"master/tasks_lost",
+		"master/tasks_running",
+		"master/tasks_staging",
+		"master/tasks_starting",
+		// messages
+		"master/invalid_executor_to_framework_messages",
+		"master/invalid_framework_to_executor_messages",
+		"master/invalid_status_update_acknowledgements",
+		"master/invalid_status_updates",
+		"master/dropped_messages",
+		"master/messages_authenticate",
+		"master/messages_deactivate_framework",
+		"master/messages_decline_offers",
+		"master/messages_executor_to_framework",
+		"master/messages_exited_executor",
+		"master/messages_framework_to_executor",
+		"master/messages_kill_task",
+		"master/messages_launch_tasks",
+		"master/messages_reconcile_tasks",
+		"master/messages_register_framework",
+		"master/messages_register_slave",
+		"master/messages_reregister_framework",
+		"master/messages_reregister_slave",
+		"master/messages_resource_request",
+		"master/messages_revive_offers",
+		"master/messages_status_update",
+		"master/messages_status_update_acknowledgement",
+		"master/messages_unregister_framework",
+		"master/messages_unregister_slave",
+		"master/messages_update_slave",
+		"master/recovery_slave_removals",
+		"master/slave_removals/reason_registered",
+		"master/slave_removals/reason_unhealthy",
+		"master/slave_removals/reason_unregistered",
+		"master/valid_framework_to_executor_messages",
+		"master/valid_status_update_acknowledgements",
+		"master/valid_status_updates",
+		"master/task_lost/source_master/reason_invalid_offers",
+		"master/task_lost/source_master/reason_slave_removed",
+		"master/task_lost/source_slave/reason_executor_terminated",
+		"master/valid_executor_to_framework_messages",
+		// evgqueue
+		"master/event_queue_dispatches",
+		"master/event_queue_http_requests",
+		"master/event_queue_messages",
+		// registrar
+		"registrar/state_fetch_ms",
+		"registrar/state_store_ms",
+		"registrar/state_store_ms/max",
+		"registrar/state_store_ms/min",
+		"registrar/state_store_ms/p50",
+		"registrar/state_store_ms/p90",
+		"registrar/state_store_ms/p95",
+		"registrar/state_store_ms/p99",
+		"registrar/state_store_ms/p999",
+		"registrar/state_store_ms/p9999",
+	}
 
 	for _, k := range metricNames {
-		mesosMetrics[k] = rand.Float64()
+		masterMetrics[k] = rand.Float64()
+	}
+
+	slaveMetrics = make(map[string]interface{})
+
+	metricNames = []string{
+		// resources
+		"slave/cpus_percent",
+		"slave/cpus_used",
+		"slave/cpus_total",
+		"slave/cpus_revocable_percent",
+		"slave/cpus_revocable_total",
+		"slave/cpus_revocable_used",
+		"slave/disk_percent",
+		"slave/disk_used",
+		"slave/disk_total",
+		"slave/disk_revocable_percent",
+		"slave/disk_revocable_total",
+		"slave/disk_revocable_used",
+		"slave/gpus_percent",
+		"slave/gpus_used",
+		"slave/gpus_total",
+		"slave/gpus_revocable_percent",
+		"slave/gpus_revocable_total",
+		"slave/gpus_revocable_used",
+		"slave/mem_percent",
+		"slave/mem_used",
+		"slave/mem_total",
+		"slave/mem_revocable_percent",
+		"slave/mem_revocable_total",
+		"slave/mem_revocable_used",
+		// agent
+		"slave/registered",
+		"slave/uptime_secs",
+		// system
+		"system/cpus_total",
+		"system/load_15min",
+		"system/load_5min",
+		"system/load_1min",
+		"system/mem_free_bytes",
+		"system/mem_total_bytes",
+		// executors
+		"containerizer/mesos/container_destroy_errors",
+		"slave/container_launch_errors",
+		"slave/executors_preempted",
+		"slave/frameworks_active",
+		"slave/executor_directory_max_allowed_age_secs",
+		"slave/executors_registering",
+		"slave/executors_running",
+		"slave/executors_terminated",
+		"slave/executors_terminating",
+		"slave/recovery_errors",
+		// tasks
+		"slave/tasks_failed",
+		"slave/tasks_finished",
+		"slave/tasks_killed",
+		"slave/tasks_lost",
+		"slave/tasks_running",
+		"slave/tasks_staging",
+		"slave/tasks_starting",
+		// messages
+		"slave/invalid_framework_messages",
+		"slave/invalid_status_updates",
+		"slave/valid_framework_messages",
+		"slave/valid_status_updates",
+	}
+
+	for _, k := range metricNames {
+		slaveMetrics[k] = rand.Float64()
+	}
+
+	slaveTaskMetrics = map[string]interface{}{
+		"executor_id":   fmt.Sprintf("task_%s", randUUID()),
+		"executor_name": "Some task description",
+		"framework_id":  randUUID(),
+		"source":        fmt.Sprintf("task_source_%s", randUUID()),
+		"statistics": map[string]interface{}{
+			"cpus_limit":                    rand.Float64(),
+			"cpus_system_time_secs":         rand.Float64(),
+			"cpus_user_time_secs":           rand.Float64(),
+			"mem_anon_bytes":                float64(rand.Int63()),
+			"mem_cache_bytes":               float64(rand.Int63()),
+			"mem_critical_pressure_counter": float64(rand.Int63()),
+			"mem_file_bytes":                float64(rand.Int63()),
+			"mem_limit_bytes":               float64(rand.Int63()),
+			"mem_low_pressure_counter":      float64(rand.Int63()),
+			"mem_mapped_file_bytes":         float64(rand.Int63()),
+			"mem_medium_pressure_counter":   float64(rand.Int63()),
+			"mem_rss_bytes":                 float64(rand.Int63()),
+			"mem_swap_bytes":                float64(rand.Int63()),
+			"mem_total_bytes":               float64(rand.Int63()),
+			"mem_total_memsw_bytes":         float64(rand.Int63()),
+			"mem_unevictable_bytes":         float64(rand.Int63()),
+			"timestamp":                     rand.Float64(),
+		},
 	}
 }
 
 func TestMain(m *testing.M) {
 	generateMetrics()
-	r := http.NewServeMux()
-	r.HandleFunc("/metrics/snapshot", func(w http.ResponseWriter, r *http.Request) {
+
+	masterRouter := http.NewServeMux()
+	masterRouter.HandleFunc("/metrics/snapshot", func(w http.ResponseWriter, r *http.Request) {
 		w.WriteHeader(http.StatusOK)
 		w.Header().Set("Content-Type", "application/json")
-		json.NewEncoder(w).Encode(mesosMetrics)
+		json.NewEncoder(w).Encode(masterMetrics)
 	})
-	ts = httptest.NewServer(r)
+	masterTestServer = httptest.NewServer(masterRouter)
+
+	slaveRouter := http.NewServeMux()
+	slaveRouter.HandleFunc("/metrics/snapshot", func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusOK)
+		w.Header().Set("Content-Type", "application/json")
+		json.NewEncoder(w).Encode(slaveMetrics)
+	})
+	slaveRouter.HandleFunc("/monitor/statistics", func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusOK)
+		w.Header().Set("Content-Type", "application/json")
+		json.NewEncoder(w).Encode([]map[string]interface{}{slaveTaskMetrics})
+	})
+	slaveTestServer = httptest.NewServer(slaveRouter)
+
 	rc := m.Run()
-	ts.Close()
+
+	masterTestServer.Close()
+	slaveTestServer.Close()
 	os.Exit(rc)
 }
 
@@ -73,7 +278,7 @@ func TestMesosMaster(t *testing.T) {
 	var acc testutil.Accumulator
 
 	m := Mesos{
-		Masters: []string{ts.Listener.Addr().String()},
+		Masters: []string{masterTestServer.Listener.Addr().String()},
 		Timeout: 10,
 	}
 
@@ -83,34 +288,88 @@ func TestMesosMaster(t *testing.T) {
 		t.Errorf(err.Error())
 	}
 
-	acc.AssertContainsFields(t, "mesos", mesosMetrics)
+	acc.AssertContainsFields(t, "mesos", masterMetrics)
 }
 
-func TestRemoveGroup(t *testing.T) {
-	generateMetrics()
-
+func TestMasterFilter(t *testing.T) {
 	m := Mesos{
 		MasterCols: []string{
 			"resources", "master", "registrar",
 		},
 	}
 	b := []string{
-		"system", "slaves", "frameworks",
-		"messages", "evqueue",
+		"system", "agents", "frameworks",
+		"messages", "evqueue", "tasks",
 	}
 
-	m.removeGroup(&mesosMetrics)
+	m.filterMetrics(MASTER, &masterMetrics)
 
 	for _, v := range b {
-		for _, x := range masterBlocks(v) {
-			if _, ok := mesosMetrics[x]; ok {
+		for _, x := range getMetrics(MASTER, v) {
+			if _, ok := masterMetrics[x]; ok {
 				t.Errorf("Found key %s, it should be gone.", x)
 			}
 		}
 	}
 	for _, v := range m.MasterCols {
-		for _, x := range masterBlocks(v) {
-			if _, ok := mesosMetrics[x]; !ok {
+		for _, x := range getMetrics(MASTER, v) {
+			if _, ok := masterMetrics[x]; !ok {
+				t.Errorf("Didn't find key %s, it should present.", x)
+			}
+		}
+	}
+}
+
+func TestMesosSlave(t *testing.T) {
+	var acc testutil.Accumulator
+
+	m := Mesos{
+		Masters:    []string{},
+		Slaves:     []string{slaveTestServer.Listener.Addr().String()},
+		SlaveTasks: true,
+		Timeout:    10,
+	}
+
+	err := m.Gather(&acc)
+
+	if err != nil {
+		t.Errorf(err.Error())
+	}
+
+	acc.AssertContainsFields(t, "mesos", slaveMetrics)
+
+	jf := jsonparser.JSONFlattener{}
+	err = jf.FlattenJSON("", slaveTaskMetrics)
+
+	if err != nil {
+		t.Errorf(err.Error())
+	}
+
+	acc.AssertContainsFields(t, "mesos-tasks", jf.Fields)
+}
+
+func TestSlaveFilter(t *testing.T) {
+	m := Mesos{
+		SlaveCols: []string{
+			"resources", "agent", "tasks",
+		},
+	}
+	b := []string{
+		"system", "executors", "messages",
+	}
+
+	m.filterMetrics(SLAVE, &slaveMetrics)
+
+	for _, v := range b {
+		for _, x := range getMetrics(SLAVE, v) {
+			if _, ok := slaveMetrics[x]; ok {
+				t.Errorf("Found key %s, it should be gone.", x)
+			}
+		}
+	}
+	for _, v := range m.MasterCols {
+		for _, x := range getMetrics(SLAVE, v) {
+			if _, ok := slaveMetrics[x]; !ok {
 				t.Errorf("Didn't find key %s, it should present.", x)
 			}
 		}

plugins/inputs/mock_Plugin.go

@@ -6,10 +6,22 @@ import (
 	"github.com/stretchr/testify/mock"
 )
 
+// MockPlugin struct should be named the same as the Plugin
 type MockPlugin struct {
 	mock.Mock
 }
 
+// Description will appear directly above the plugin definition in the config file
+func (m *MockPlugin) Description() string {
+	return `This is an example plugin`
+}
+
+// SampleConfig will populate the sample configuration portion of the plugin's configuration
+func (m *MockPlugin) SampleConfig() string {
+	return `  sampleVar = 'foo'`
+}
+
+// Gather defines what data the plugin will gather.
 func (m *MockPlugin) Gather(_a0 telegraf.Accumulator) error {
 	ret := m.Called(_a0)
 

plugins/inputs/mongodb/README.md

@@ -10,6 +10,7 @@
   ##   mongodb://10.10.3.33:18832,
   ##   10.0.0.1:10000, etc.
   servers = ["127.0.0.1:27017"]
+  gather_perdb_stats = false
 ```
 
 For authenticated mongodb istances use connection mongdb connection URI
@@ -52,3 +53,15 @@ and create a single measurement containing values e.g.
  * ttl_passes_per_sec
  * repl_lag
  * jumbo_chunks (only if mongos or mongo config)
+
+If gather_db_stats is set to true, it will also collect per database stats exposed by db.stats()
+creating another measurement called mongodb_db_stats and containing values:
+ * collections
+ * objects
+ * avg_obj_size
+ * data_size
+ * storage_size
+ * num_extents
+ * indexes
+ * index_size
+ * ok

plugins/inputs/mongodb/mongodb.go

@@ -10,14 +10,16 @@ import (
 	"time"
 
 	"github.com/influxdata/telegraf"
+	"github.com/influxdata/telegraf/internal/errchan"
 	"github.com/influxdata/telegraf/plugins/inputs"
 	"gopkg.in/mgo.v2"
 )
 
 type MongoDB struct {
-	Servers []string
-	Ssl     Ssl
-	mongos  map[string]*Server
+	Servers          []string
+	Ssl              Ssl
+	mongos           map[string]*Server
+	GatherPerdbStats bool
 }
 
 type Ssl struct {
@@ -32,6 +34,7 @@ var sampleConfig = `
   ##   mongodb://10.10.3.33:18832,
   ##   10.0.0.1:10000, etc.
   servers = ["127.0.0.1:27017"]
+  gather_perdb_stats = false
 `
 
 func (m *MongoDB) SampleConfig() string {
@@ -53,9 +56,7 @@ func (m *MongoDB) Gather(acc telegraf.Accumulator) error {
 	}
 
 	var wg sync.WaitGroup
-
-	var outerr error
-
+	errChan := errchan.New(len(m.Servers))
 	for _, serv := range m.Servers {
 		u, err := url.Parse(serv)
 		if err != nil {
@@ -71,13 +72,12 @@ func (m *MongoDB) Gather(acc telegraf.Accumulator) error {
 		wg.Add(1)
 		go func(srv *Server) {
 			defer wg.Done()
-			outerr = m.gatherServer(srv, acc)
+			errChan.C <- m.gatherServer(srv, acc)
 		}(m.getMongoServer(u))
 	}
 
 	wg.Wait()
-
-	return outerr
+	return errChan.Error()
 }
 
 func (m *MongoDB) getMongoServer(url *url.URL) *Server {
@@ -135,7 +135,7 @@ func (m *MongoDB) gatherServer(server *Server, acc telegraf.Accumulator) error {
 		}
 		server.Session = sess
 	}
-	return server.gatherData(acc)
+	return server.gatherData(acc, m.GatherPerdbStats)
 }
 
 func init() {

plugins/inputs/mongodb/mongodb_data.go

@@ -12,6 +12,12 @@ type MongodbData struct {
 	StatLine *StatLine
 	Fields   map[string]interface{}
 	Tags     map[string]string
+	DbData   []DbData
+}
+
+type DbData struct {
+	Name   string
+	Fields map[string]interface{}
 }
 
 func NewMongodbData(statLine *StatLine, tags map[string]string) *MongodbData {
@@ -22,6 +28,7 @@ func NewMongodbData(statLine *StatLine, tags map[string]string) *MongodbData {
 		StatLine: statLine,
 		Tags:     tags,
 		Fields:   make(map[string]interface{}),
+		DbData:   []DbData{},
 	}
 }
 
@@ -72,6 +79,34 @@ var WiredTigerStats = map[string]string{
 	"percent_cache_used":  "CacheUsedPercent",
 }
 
+var DbDataStats = map[string]string{
+	"collections":  "Collections",
+	"objects":      "Objects",
+	"avg_obj_size": "AvgObjSize",
+	"data_size":    "DataSize",
+	"storage_size": "StorageSize",
+	"num_extents":  "NumExtents",
+	"indexes":      "Indexes",
+	"index_size":   "IndexSize",
+	"ok":           "Ok",
+}
+
+func (d *MongodbData) AddDbStats() {
+	for _, dbstat := range d.StatLine.DbStatsLines {
+		dbStatLine := reflect.ValueOf(&dbstat).Elem()
+		newDbData := &DbData{
+			Name:   dbstat.Name,
+			Fields: make(map[string]interface{}),
+		}
+		newDbData.Fields["type"] = "db_stat"
+		for key, value := range DbDataStats {
+			val := dbStatLine.FieldByName(value).Interface()
+			newDbData.Fields[key] = val
+		}
+		d.DbData = append(d.DbData, *newDbData)
+	}
+}
+
 func (d *MongodbData) AddDefaultStats() {
 	statLine := reflect.ValueOf(d.StatLine).Elem()
 	d.addStat(statLine, DefaultStats)
@@ -113,4 +148,15 @@ func (d *MongodbData) flush(acc telegraf.Accumulator) {
 		d.StatLine.Time,
 	)
 	d.Fields = make(map[string]interface{})
+
+	for _, db := range d.DbData {
+		d.Tags["db_name"] = db.Name
+		acc.AddFields(
+			"mongodb_db_stats",
+			db.Fields,
+			d.Tags,
+			d.StatLine.Time,
+		)
+		db.Fields = make(map[string]interface{})
+	}
 }

plugins/inputs/mongodb/mongodb_server.go

@@ -22,7 +22,7 @@ func (s *Server) getDefaultTags() map[string]string {
 	return tags
 }
 
-func (s *Server) gatherData(acc telegraf.Accumulator) error {
+func (s *Server) gatherData(acc telegraf.Accumulator, gatherDbStats bool) error {
 	s.Session.SetMode(mgo.Eventual, true)
 	s.Session.SetSocketTimeout(0)
 	result_server := &ServerStatus{}
@@ -42,10 +42,34 @@ func (s *Server) gatherData(acc telegraf.Accumulator) error {
 		JumboChunksCount: int64(jumbo_chunks),
 	}
 
+	result_db_stats := &DbStats{}
+
+	if gatherDbStats == true {
+		names := []string{}
+		names, err = s.Session.DatabaseNames()
+		if err != nil {
+			log.Println("Error getting database names (" + err.Error() + ")")
+		}
+		for _, db_name := range names {
+			db_stat_line := &DbStatsData{}
+			err = s.Session.DB(db_name).Run(bson.D{{"dbStats", 1}}, db_stat_line)
+			if err != nil {
+				log.Println("Error getting db stats from " + db_name + "(" + err.Error() + ")")
+			}
+			db := &Db{
+				Name:        db_name,
+				DbStatsData: db_stat_line,
+			}
+
+			result_db_stats.Dbs = append(result_db_stats.Dbs, *db)
+		}
+	}
+
 	result := &MongoStatus{
 		ServerStatus:  result_server,
 		ReplSetStatus: result_repl,
 		ClusterStatus: result_cluster,
+		DbStats:       result_db_stats,
 	}
 
 	defer func() {
@@ -64,6 +88,7 @@ func (s *Server) gatherData(acc telegraf.Accumulator) error {
 			s.getDefaultTags(),
 		)
 		data.AddDefaultStats()
+		data.AddDbStats()
 		data.flush(acc)
 	}
 	return nil

plugins/inputs/mongodb/mongodb_server_test.go

@@ -29,12 +29,12 @@ func TestGetDefaultTags(t *testing.T) {
 func TestAddDefaultStats(t *testing.T) {
 	var acc testutil.Accumulator
 
-	err := server.gatherData(&acc)
+	err := server.gatherData(&acc, false)
 	require.NoError(t, err)
 
 	time.Sleep(time.Duration(1) * time.Second)
 	// need to call this twice so it can perform the diff
-	err = server.gatherData(&acc)
+	err = server.gatherData(&acc, false)
 	require.NoError(t, err)
 
 	for key, _ := range DefaultStats {

plugins/inputs/mongodb/mongostat.go

@@ -35,6 +35,7 @@ type MongoStatus struct {
 	ServerStatus  *ServerStatus
 	ReplSetStatus *ReplSetStatus
 	ClusterStatus *ClusterStatus
+	DbStats       *DbStats
 }
 
 type ServerStatus struct {
@@ -65,6 +66,32 @@ type ServerStatus struct {
 	Metrics            *MetricsStats          `bson:"metrics"`
 }
 
+// DbStats stores stats from all dbs
+type DbStats struct {
+	Dbs []Db
+}
+
+// Db represent a single DB
+type Db struct {
+	Name        string
+	DbStatsData *DbStatsData
+}
+
+// DbStatsData stores stats from a db
+type DbStatsData struct {
+	Db          string      `bson:"db"`
+	Collections int64       `bson:"collections"`
+	Objects     int64       `bson:"objects"`
+	AvgObjSize  float64     `bson:"avgObjSize"`
+	DataSize    int64       `bson:"dataSize"`
+	StorageSize int64       `bson:"storageSize"`
+	NumExtents  int64       `bson:"numExtents"`
+	Indexes     int64       `bson:"indexes"`
+	IndexSize   int64       `bson:"indexSize"`
+	Ok          int64       `bson:"ok"`
+	GleStats    interface{} `bson:"gleStats"`
+}
+
 // ClusterStatus stores information related to the whole cluster
 type ClusterStatus struct {
 	JumboChunksCount int64
@@ -396,6 +423,22 @@ type StatLine struct {
 
 	// Cluster fields
 	JumboChunksCount int64
+
+	// DB stats field
+	DbStatsLines []DbStatLine
+}
+
+type DbStatLine struct {
+	Name        string
+	Collections int64
+	Objects     int64
+	AvgObjSize  float64
+	DataSize    int64
+	StorageSize int64
+	NumExtents  int64
+	Indexes     int64
+	IndexSize   int64
+	Ok          int64
 }
 
 func parseLocks(stat ServerStatus) map[string]LockUsage {
@@ -677,5 +720,27 @@ func NewStatLine(oldMongo, newMongo MongoStatus, key string, all bool, sampleSec
 	newClusterStat := *newMongo.ClusterStatus
 	returnVal.JumboChunksCount = newClusterStat.JumboChunksCount
 
+	newDbStats := *newMongo.DbStats
+	for _, db := range newDbStats.Dbs {
+		dbStatsData := db.DbStatsData
+		// mongos doesn't have the db key, so setting the db name
+		if dbStatsData.Db == "" {
+			dbStatsData.Db = db.Name
+		}
+		dbStatLine := &DbStatLine{
+			Name:        dbStatsData.Db,
+			Collections: dbStatsData.Collections,
+			Objects:     dbStatsData.Objects,
+			AvgObjSize:  dbStatsData.AvgObjSize,
+			DataSize:    dbStatsData.DataSize,
+			StorageSize: dbStatsData.StorageSize,
+			NumExtents:  dbStatsData.NumExtents,
+			Indexes:     dbStatsData.Indexes,
+			IndexSize:   dbStatsData.IndexSize,
+			Ok:          dbStatsData.Ok,
+		}
+		returnVal.DbStatsLines = append(returnVal.DbStatsLines, *dbStatLine)
+	}
+
 	return returnVal
 }

plugins/inputs/mysql/mysql.go

@@ -7,10 +7,12 @@ import (
 	"net/url"
 	"strconv"
 	"strings"
+	"sync"
 	"time"
 
 	_ "github.com/go-sql-driver/mysql"
 	"github.com/influxdata/telegraf"
+	"github.com/influxdata/telegraf/internal/errchan"
 	"github.com/influxdata/telegraf/plugins/inputs"
 )
 
@@ -118,26 +120,27 @@ func (m *Mysql) InitMysql() {
 
 func (m *Mysql) Gather(acc telegraf.Accumulator) error {
 	if len(m.Servers) == 0 {
-		// if we can't get stats in this case, thats fine, don't report
-		// an error.
-		m.gatherServer(localhost, acc)
-		return nil
+		// default to localhost if nothing specified.
+		return m.gatherServer(localhost, acc)
 	}
-
 	// Initialise additional query intervals
 	if !initDone {
 		m.InitMysql()
 	}
+	var wg sync.WaitGroup
+	errChan := errchan.New(len(m.Servers))
 
 	// Loop through each server and collect metrics
-	for _, serv := range m.Servers {
-		err := m.gatherServer(serv, acc)
-		if err != nil {
-			return err
-		}
+	for _, server := range m.Servers {
+		wg.Add(1)
+		go func(s string) {
+			defer wg.Done()
+			errChan.C <- m.gatherServer(s, acc)
+		}(server)
 	}
 
-	return nil
+	wg.Wait()
+	return errChan.Error()
 }
 
 type mapping struct {
@@ -306,6 +309,10 @@ var mappings = []*mapping{
 		onServer: "Threadpool_",
 		inExport: "threadpool_",
 	},
+	{
+		onServer: "wsrep_",
+		inExport: "wsrep_",
+	},
 }
 
 var (
@@ -1369,6 +1376,7 @@ func (m *Mysql) gatherPerfEventsStatements(db *sql.DB, serv string, acc telegraf
 			&rowsAffected, &rowsSent, &rowsExamined,
 			&tmpTables, &tmpDiskTables,
 			&sortMergePasses, &sortRows,
+			&noIndexUsed,
 		)
 
 		if err != nil {