Release 0.13.1

* Adding Varnish HTTP Cache input plugin

* Applying PR feedback

- Linked to varnish in input README
- Updated README/CHANGELOG
- Cleaned up sampleConfig to remove formatting
- Shorted lines to 80 chars (except where test input requires long strings)
- Using internal.RunTimeout to wrap call to varnishtat
- Added dummy file for windows

.gitattributes

@@ -0,0 +1,2 @@
+CHANGELOG.md merge=union
+

.github/ISSUE_TEMPLATE.md

@@ -0,0 +1,42 @@
+## Directions
+
+GitHub Issues are reserved for actionable bug reports and feature requests.
+General questions should be sent to the [InfluxDB mailing list](https://groups.google.com/forum/#!forum/influxdb).
+
+Before opening an issue, search for similar bug reports or feature requests on GitHub Issues.
+If no similar issue can be found, fill out either the "Bug Report" or the "Feature Request" section below.
+Erase the other section and everything on and above this line.
+
+*Please note, the quickest way to fix a bug is to open a Pull Request.*
+
+## Bug report
+
+### System info:
+
+[Include Telegraf version, operating system name, and other relevant details]
+
+### Steps to reproduce:
+
+1. ...
+2. ...
+
+### Expected behavior:
+
+### Actual behavior:
+
+### Additional info:
+
+[Include gist of relevant config, logs, etc.]
+
+
+## Feature Request
+
+Opening a feature request kicks off a discussion.
+
+### Proposal:
+
+### Current behavior:
+
+### Desired behavior:
+
+### Use case: [Why is this important (helps with prioritizing requests)]

.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,5 @@
+### Required for all PRs:
+
+- [ ] CHANGELOG.md updated
+- [ ] Sign [CLA](https://influxdata.com/community/cla/) (if not already signed)
+- [ ] README.md updated (if adding a new plugin)

CHANGELOG.md

@@ -1,13 +1,282 @@
-## v0.10.4 [unreleased]
+## v1.0 [unreleased]
+
+## v0.13.1 [2016-05-24]
+
+### Release Notes
+
+- net_response and http_response plugins timeouts will now accept duration
+strings, ie, "2s" or "500ms".
+- Input plugin Gathers will no longer be logged by default, but a Gather for
+_each_ plugin will be logged in Debug mode.
+- Debug mode will no longer print every point added to the accumulator. This
+functionality can be duplicated using the `file` output plugin and printing
+to "stdout".
+
+### Features
+
+- [#1173](https://github.com/influxdata/telegraf/pull/1173): varnish input plugin. Thanks @sfox-xmatters!
+- [#1138](https://github.com/influxdata/telegraf/pull/1138): nstat input plugin. Thanks @Maksadbek!
+- [#1139](https://github.com/influxdata/telegraf/pull/1139): instrumental output plugin. Thanks @jasonroelofs!
+- [#1172](https://github.com/influxdata/telegraf/pull/1172): Ceph storage stats. Thanks @robinpercy!
+- [#1233](https://github.com/influxdata/telegraf/pull/1233): Updated golint gopsutil dependency.
+- [#1238](https://github.com/influxdata/telegraf/pull/1238): chrony input plugin. Thanks @zbindenren!
+- [#479](https://github.com/influxdata/telegraf/issues/479): per-plugin execution time added to debug output.
+- [#1249](https://github.com/influxdata/telegraf/issues/1249): influxdb output: added write_consistency argument.
+
+### Bugfixes
+
+- [#1195](https://github.com/influxdata/telegraf/pull/1195): Docker panic on timeout. Thanks @zstyblik!
+- [#1211](https://github.com/influxdata/telegraf/pull/1211): mongodb input. Fix possible panic. Thanks @kols!
+- [#1215](https://github.com/influxdata/telegraf/pull/1215): Fix for possible gopsutil-dependent plugin hangs.
+- [#1228](https://github.com/influxdata/telegraf/pull/1228): Fix service plugin host tag overwrite.
+- [#1198](https://github.com/influxdata/telegraf/pull/1198): http_response: override request Host header properly
+- [#1230](https://github.com/influxdata/telegraf/issues/1230): Fix Telegraf process hangup due to a single plugin hanging.
+- [#1214](https://github.com/influxdata/telegraf/issues/1214): Use TCP timeout argument in net_response plugin.
+- [#1243](https://github.com/influxdata/telegraf/pull/1243): Logfile not created on systemd.
+
+## v0.13 [2016-05-11]
+
+### Release Notes
+
+- **Breaking change** in jolokia plugin. See
+https://github.com/influxdata/telegraf/blob/master/plugins/inputs/jolokia/README.md
+for updated configuration. The plugin will now support proxy mode and will make
+POST requests.
+
+- New [agent] configuration option: `metric_batch_size`. This option tells
+telegraf the maximum batch size to allow to accumulate before sending a flush
+to the configured outputs. `metric_buffer_limit` now refers to the absolute
+maximum number of metrics that will accumulate before metrics are dropped.
+
+- There is no longer an option to
+`flush_buffer_when_full`, this is now the default and only behavior of telegraf.
+
+- **Breaking Change**: docker plugin tags. The cont_id tag no longer exists, it
+will now be a field, and be called container_id. Additionally, cont_image and
+cont_name are being renamed to container_image and container_name.
+
+- **Breaking Change**: docker plugin measurements. The `docker_cpu`, `docker_mem`,
+`docker_blkio` and `docker_net` measurements are being renamed to
+`docker_container_cpu`, `docker_container_mem`, `docker_container_blkio` and
+`docker_container_net`. Why? Because these metrics are
+specifically tracking per-container stats. The problem with per-container stats,
+in some use-cases, is that if containers are short-lived AND names are not
+kept consistent, then the series cardinality will balloon very quickly.
+So adding "container" to each metric will:
+(1) make it more clear that these metrics are per-container, and
+(2) allow users to easily drop per-container metrics if cardinality is an
+issue (`namedrop = ["docker_container_*"]`)
+
+- `tagexclude` and `taginclude` are now available, which can be used to remove
+tags from measurements on inputs and outputs. See
+[the configuration doc](https://github.com/influxdata/telegraf/blob/master/docs/CONFIGURATION.md)
+for more details.
+
+- **Measurement filtering:** All measurement filters now match based on glob
+only. Previously there was an undocumented behavior where filters would match
+based on _prefix_ in addition to globs. This means that a filter like
+`fielddrop = ["time_"]` will need to be changed to `fielddrop = ["time_*"]`
+
+- **datadog**: measurement and field names will no longer have `_` replaced by `.`
+
+- The following plugins have changed their tags to _not_ overwrite the host tag:
+  - cassandra: `host -> cassandra_host`
+  - disque: `host -> disque_host`
+  - rethinkdb: `host -> rethinkdb_host`
+
+- **Breaking Change**: The `win_perf_counters` input has been changed to
+sanitize field names, replacing `/Sec` and `/sec` with `_persec`, as well as
+spaces with underscores. This is needed because Graphite doesn't like slashes
+and spaces, and was failing to accept metrics that had them.
+The `/[sS]ec` -> `_persec` is just to make things clearer and uniform.
+
+- **Breaking Change**: snmp plugin. The `host` tag of the snmp plugin has been
+changed to the `snmp_host` tag.
+
+- The `disk` input plugin can now be configured with the `HOST_MOUNT_PREFIX` environment variable.
+This value is prepended to any mountpaths discovered before retrieving stats.
+It is not included on the report path. This is necessary for reporting host disk stats when running from within a container.
+
+### Features
+
+- [#1031](https://github.com/influxdata/telegraf/pull/1031): Jolokia plugin proxy mode. Thanks @saiello!
+- [#1017](https://github.com/influxdata/telegraf/pull/1017): taginclude and tagexclude arguments.
+- [#1015](https://github.com/influxdata/telegraf/pull/1015): Docker plugin schema refactor.
+- [#889](https://github.com/influxdata/telegraf/pull/889): Improved MySQL plugin. Thanks @maksadbek!
+- [#1060](https://github.com/influxdata/telegraf/pull/1060): TTL metrics added to MongoDB input plugin
+- [#1056](https://github.com/influxdata/telegraf/pull/1056): Don't allow inputs to overwrite host tags.
+- [#1035](https://github.com/influxdata/telegraf/issues/1035): Add `user`, `exe`, `pidfile` tags to procstat plugin.
+- [#1041](https://github.com/influxdata/telegraf/issues/1041): Add `n_cpus` field to the system plugin.
+- [#1072](https://github.com/influxdata/telegraf/pull/1072): New Input Plugin: filestat.
+- [#1066](https://github.com/influxdata/telegraf/pull/1066): Replication lag metrics for MongoDB input plugin
+- [#1086](https://github.com/influxdata/telegraf/pull/1086): Ability to specify AWS keys in config file. Thanks @johnrengleman!
+- [#1096](https://github.com/influxdata/telegraf/pull/1096): Performance refactor of running output buffers.
+- [#967](https://github.com/influxdata/telegraf/issues/967): Buffer logging improvements.
+- [#1107](https://github.com/influxdata/telegraf/issues/1107): Support lustre2 job stats. Thanks @hanleyja!
+- [#1122](https://github.com/influxdata/telegraf/pull/1122): Support setting config path through env variable and default paths.
+- [#1128](https://github.com/influxdata/telegraf/pull/1128): MongoDB jumbo chunks metric for MongoDB input plugin
+- [#1146](https://github.com/influxdata/telegraf/pull/1146): HAProxy socket support. Thanks weshmashian!
+
+### Bugfixes
+
+- [#1050](https://github.com/influxdata/telegraf/issues/1050): jolokia plugin - do not overwrite host tag. Thanks @saiello!
+- [#921](https://github.com/influxdata/telegraf/pull/921): mqtt_consumer stops gathering metrics. Thanks @chaton78!
+- [#1013](https://github.com/influxdata/telegraf/pull/1013): Close dead riemann output connections. Thanks @echupriyanov!
+- [#1012](https://github.com/influxdata/telegraf/pull/1012): Set default tags in test accumulator.
+- [#1024](https://github.com/influxdata/telegraf/issues/1024): Don't replace `.` with `_` in datadog output.
+- [#1058](https://github.com/influxdata/telegraf/issues/1058): Fix possible leaky TCP connections in influxdb output.
+- [#1044](https://github.com/influxdata/telegraf/pull/1044): Fix SNMP OID possible collisions. Thanks @relip
+- [#1022](https://github.com/influxdata/telegraf/issues/1022): Dont error deb/rpm install on systemd errors.
+- [#1078](https://github.com/influxdata/telegraf/issues/1078): Use default AWS credential chain.
+- [#1070](https://github.com/influxdata/telegraf/issues/1070): SQL Server input. Fix datatype conversion.
+- [#1089](https://github.com/influxdata/telegraf/issues/1089): Fix leaky TCP connections in phpfpm plugin.
+- [#914](https://github.com/influxdata/telegraf/issues/914): Telegraf can drop metrics on full buffers.
+- [#1098](https://github.com/influxdata/telegraf/issues/1098): Sanitize invalid OpenTSDB characters.
+- [#1110](https://github.com/influxdata/telegraf/pull/1110): Sanitize * to - in graphite serializer. Thanks @goodeggs!
+- [#1118](https://github.com/influxdata/telegraf/pull/1118): Sanitize Counter names for `win_perf_counters` input.
+- [#1125](https://github.com/influxdata/telegraf/pull/1125): Wrap all exec command runners with a timeout, so hung os processes don't halt Telegraf.
+- [#1113](https://github.com/influxdata/telegraf/pull/1113): Set MaxRetry and RequiredAcks defaults in Kafka output.
+- [#1090](https://github.com/influxdata/telegraf/issues/1090): [agent] and [global_tags] config sometimes not getting applied.
+- [#1133](https://github.com/influxdata/telegraf/issues/1133): Use a timeout for docker list & stat cmds.
+- [#1052](https://github.com/influxdata/telegraf/issues/1052): Docker panic fix when decode fails.
+- [#1136](https://github.com/influxdata/telegraf/pull/1136): "DELAYED" Inserts were deprecated in MySQL 5.6.6. Thanks @PierreF
+
+## v0.12.1 [2016-04-14]
+
+### Release Notes
+- Breaking change in the dovecot input plugin. See Features section below.
+- Graphite output templates are now supported. See
+https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_OUTPUT.md#graphite
+- Possible breaking change for the librato and graphite outputs. Telegraf will
+no longer insert field names when the field is simply named `value`. This is
+because the `value` field is redundant in the graphite/librato context.
+
+### Features
+- [#1009](https://github.com/influxdata/telegraf/pull/1009): Cassandra input plugin. Thanks @subhachandrachandra!
+- [#976](https://github.com/influxdata/telegraf/pull/976): Reduce allocations in the UDP and statsd inputs.
+- [#979](https://github.com/influxdata/telegraf/pull/979): Reduce allocations in the TCP listener.
+- [#992](https://github.com/influxdata/telegraf/pull/992): Refactor allocations in TCP/UDP listeners.
+- [#935](https://github.com/influxdata/telegraf/pull/935): AWS Cloudwatch input plugin. Thanks @joshhardy & @ljosa!
+- [#943](https://github.com/influxdata/telegraf/pull/943): http_response input plugin. Thanks @Lswith!
+- [#939](https://github.com/influxdata/telegraf/pull/939): sysstat input plugin. Thanks @zbindenren!
+- [#998](https://github.com/influxdata/telegraf/pull/998): **breaking change** enabled global, user and ip queries in dovecot plugin. Thanks @mikif70!
+- [#1001](https://github.com/influxdata/telegraf/pull/1001): Graphite serializer templates.
+- [#1008](https://github.com/influxdata/telegraf/pull/1008): Adding memstats metrics to the influxdb plugin.
+
+### Bugfixes
+- [#968](https://github.com/influxdata/telegraf/issues/968): Processes plugin gets unknown state when spaces are in (command name)
+- [#969](https://github.com/influxdata/telegraf/pull/969): ipmi_sensors: allow : in password. Thanks @awaw!
+- [#972](https://github.com/influxdata/telegraf/pull/972): dovecot: remove extra newline in dovecot command. Thanks @mrannanj!
+- [#645](https://github.com/influxdata/telegraf/issues/645): docker plugin i/o error on closed pipe. Thanks @tripledes!
+
+## v0.12.0 [2016-04-05]
+
+### Features
+- [#951](https://github.com/influxdata/telegraf/pull/951): Parse environment variables in the config file.
+- [#948](https://github.com/influxdata/telegraf/pull/948): Cleanup config file and make default package version include all plugins (but commented).
+- [#927](https://github.com/influxdata/telegraf/pull/927): Adds parsing of tags to the statsd input when using DataDog's dogstatsd extension
+- [#863](https://github.com/influxdata/telegraf/pull/863): AMQP output: allow external auth. Thanks @ekini!
+- [#707](https://github.com/influxdata/telegraf/pull/707): Improved prometheus plugin. Thanks @titilambert!
+- [#878](https://github.com/influxdata/telegraf/pull/878): Added json serializer. Thanks @ch3lo!
+- [#880](https://github.com/influxdata/telegraf/pull/880): Add the ability to specify the bearer token to the prometheus plugin. Thanks @jchauncey!
+- [#882](https://github.com/influxdata/telegraf/pull/882): Fixed SQL Server Plugin issues
+- [#849](https://github.com/influxdata/telegraf/issues/849): Adding ability to parse single values as an input data type.
+- [#844](https://github.com/influxdata/telegraf/pull/844): postgres_extensible plugin added. Thanks @menardorama!
+- [#866](https://github.com/influxdata/telegraf/pull/866): couchbase input plugin. Thanks @ljosa!
+- [#789](https://github.com/influxdata/telegraf/pull/789): Support multiple field specification and `field*` in graphite templates. Thanks @chrusty!
+- [#762](https://github.com/influxdata/telegraf/pull/762): Nagios parser for the exec plugin. Thanks @titilambert!
+- [#848](https://github.com/influxdata/telegraf/issues/848): Provide option to omit host tag from telegraf agent.
+- [#928](https://github.com/influxdata/telegraf/pull/928): Deprecating the statsd "convert_names" options, expose separator config.
+- [#919](https://github.com/influxdata/telegraf/pull/919): ipmi_sensor input plugin. Thanks @ebookbug!
+- [#945](https://github.com/influxdata/telegraf/pull/945): KAFKA output: codec, acks, and retry configuration. Thanks @framiere!
+
+### Bugfixes
+- [#890](https://github.com/influxdata/telegraf/issues/890): Create TLS config even if only ssl_ca is provided.
+- [#884](https://github.com/influxdata/telegraf/issues/884): Do not call write method if there are 0 metrics to write.
+- [#898](https://github.com/influxdata/telegraf/issues/898): Put database name in quotes, fixes special characters in the database name.
+- [#656](https://github.com/influxdata/telegraf/issues/656): No longer run `lsof` on linux to get netstat data, fixes permissions issue.
+- [#907](https://github.com/influxdata/telegraf/issues/907): Fix prometheus invalid label/measurement name key.
+- [#841](https://github.com/influxdata/telegraf/issues/841): Fix memcached unix socket panic.
+- [#873](https://github.com/influxdata/telegraf/issues/873): Fix SNMP plugin sometimes not returning metrics. Thanks @titiliambert!
+- [#934](https://github.com/influxdata/telegraf/pull/934): phpfpm: Fix fcgi uri path. Thanks @rudenkovk!
+- [#805](https://github.com/influxdata/telegraf/issues/805): Kafka consumer stops gathering after i/o timeout.
+- [#959](https://github.com/influxdata/telegraf/pull/959): reduce mongodb & prometheus collection timeouts. Thanks @PierreF!
+
+## v0.11.1 [2016-03-17]
+
+### Release Notes
+- Primarily this release was cut to fix [#859](https://github.com/influxdata/telegraf/issues/859)
+
+### Features
+- [#747](https://github.com/influxdata/telegraf/pull/747): Start telegraf on install & remove on uninstall. Thanks @pierref!
+- [#794](https://github.com/influxdata/telegraf/pull/794): Add service reload ability. Thanks @entertainyou!
+
+### Bugfixes
+- [#852](https://github.com/influxdata/telegraf/issues/852): Windows zip package fix
+- [#859](https://github.com/influxdata/telegraf/issues/859): httpjson plugin panic
+
+## v0.11.0 [2016-03-15]
+
+### Release Notes
+
+### Features
+- [#692](https://github.com/influxdata/telegraf/pull/770): Support InfluxDB retention policies
+- [#771](https://github.com/influxdata/telegraf/pull/771): Default timeouts for input plugns. Thanks @PierreF!
+- [#758](https://github.com/influxdata/telegraf/pull/758): UDP Listener input plugin, thanks @whatyouhide!
+- [#769](https://github.com/influxdata/telegraf/issues/769): httpjson plugin: allow specifying SSL configuration.
+- [#735](https://github.com/influxdata/telegraf/pull/735): SNMP Table feature. Thanks @titilambert!
+- [#754](https://github.com/influxdata/telegraf/pull/754): docker plugin: adding `docker info` metrics to output. Thanks @titilambert!
+- [#788](https://github.com/influxdata/telegraf/pull/788): -input-list and -output-list command-line options. Thanks @ebookbug!
+- [#778](https://github.com/influxdata/telegraf/pull/778): Adding a TCP input listener.
+- [#797](https://github.com/influxdata/telegraf/issues/797): Provide option for persistent MQTT consumer client sessions.
+- [#799](https://github.com/influxdata/telegraf/pull/799): Add number of threads for procstat input plugin. Thanks @titilambert!
+- [#776](https://github.com/influxdata/telegraf/pull/776): Add Zookeeper chroot option to kafka_consumer. Thanks @prune998!
+- [#811](https://github.com/influxdata/telegraf/pull/811): Add processes plugin for classifying total procs on system. Thanks @titilambert!
+- [#235](https://github.com/influxdata/telegraf/issues/235): Add number of users to the `system` input plugin.
+- [#826](https://github.com/influxdata/telegraf/pull/826): "kernel" linux plugin for /proc/stat metrics (context switches, interrupts, etc.)
+- [#847](https://github.com/influxdata/telegraf/pull/847): `ntpq`: Input plugin for running ntp query executable and gathering metrics.
+
+### Bugfixes
+- [#748](https://github.com/influxdata/telegraf/issues/748): Fix sensor plugin split on ":"
+- [#722](https://github.com/influxdata/telegraf/pull/722): Librato output plugin fixes. Thanks @chrusty!
+- [#745](https://github.com/influxdata/telegraf/issues/745): Fix Telegraf toml parse panic on large config files. Thanks @titilambert!
+- [#781](https://github.com/influxdata/telegraf/pull/781): Fix mqtt_consumer username not being set. Thanks @chaton78!
+- [#786](https://github.com/influxdata/telegraf/pull/786): Fix mqtt output username not being set. Thanks @msangoi!
+- [#773](https://github.com/influxdata/telegraf/issues/773): Fix duplicate measurements in snmp plugin. Thanks @titilambert!
+- [#708](https://github.com/influxdata/telegraf/issues/708): packaging: build ARM package
+- [#713](https://github.com/influxdata/telegraf/issues/713): packaging: insecure permissions error on log directory
+- [#816](https://github.com/influxdata/telegraf/issues/816): Fix phpfpm panic if fcgi endpoint unreachable.
+- [#828](https://github.com/influxdata/telegraf/issues/828): fix net_response plugin overwriting host tag.
+- [#821](https://github.com/influxdata/telegraf/issues/821): Remove postgres password from server tag. Thanks @menardorama!
+
+## v0.10.4.1
+
+### Release Notes
+- Bug in the build script broke deb and rpm packages.
+
+### Bugfixes
+- [#750](https://github.com/influxdata/telegraf/issues/750): deb package broken
+- [#752](https://github.com/influxdata/telegraf/issues/752): rpm package broken
+
+## v0.10.4 [2016-02-24]
+
+### Release Notes
+- The pass/drop parameters have been renamed to fielddrop/fieldpass parameters,
+to more accurately indicate their purpose.
+- There are also now namedrop/namepass parameters for passing/dropping based
+on the metric _name_.
+- Experimental windows builds now available.
 
 ### Features
 - [#727](https://github.com/influxdata/telegraf/pull/727): riak input, thanks @jcoene!
 - [#694](https://github.com/influxdata/telegraf/pull/694): DNS Query input, thanks @mjasion!
 - [#724](https://github.com/influxdata/telegraf/pull/724): username matching for procstat input, thanks @zorel!
 - [#736](https://github.com/influxdata/telegraf/pull/736): Ignore dummy filesystems from disk plugin. Thanks @PierreF!
+- [#737](https://github.com/influxdata/telegraf/pull/737): Support multiple fields for statsd input. Thanks @mattheath!
 
 ### Bugfixes
 - [#701](https://github.com/influxdata/telegraf/pull/701): output write count shouldnt print in quiet mode.
+- [#746](https://github.com/influxdata/telegraf/pull/746): httpjson plugin: Fix HTTP GET parameters.
 
 ## v0.10.3 [2016-02-18]
 

CONTRIBUTING.md

@@ -80,7 +80,7 @@ func (s *Simple) SampleConfig() string {
     return "ok = true # indicate if everything is fine"
 }
 
-func (s *Simple) Gather(acc inputs.Accumulator) error {
+func (s *Simple) Gather(acc telegraf.Accumulator) error {
     if s.Ok {
         acc.Add("state", "pretty good", nil)
     } else {
@@ -114,7 +114,7 @@ creating the `Parser` object.
 You should also add the following to your SampleConfig() return:
 
 ```toml
-  ## Data format to consume. This can be "json", "influx" or "graphite"
+  ## Data format to consume. 
   ## Each data format has it's own unique set of configuration options, read
   ## more about them here:
   ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md
@@ -244,7 +244,7 @@ instantiating and creating the `Serializer` object.
 You should also add the following to your SampleConfig() return:
 
 ```toml
-  ## Data format to output. This can be "influx" or "graphite"
+  ## Data format to output. 
   ## Each data format has it's own unique set of configuration options, read
   ## more about them here:
   ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_OUTPUT.md

Godeps

@@ -1,53 +1,58 @@
-git.eclipse.org/gitroot/paho/org.eclipse.paho.mqtt.golang.git 617c801af238c3af2d9e72c5d4a0f02edad03ce5
-github.com/Shopify/sarama d37c73f2b2bce85f7fa16b6a550d26c5372892ef
-github.com/Sirupsen/logrus f7f79f729e0fbe2fcc061db48a9ba0263f588252
-github.com/amir/raidman 6a8e089bbe32e6b907feae5ba688841974b3c339
-github.com/aws/aws-sdk-go 87b1e60a50b09e4812dee560b33a238f67305804
-github.com/beorn7/perks b965b613227fddccbfffe13eae360ed3fa822f8d
+github.com/Shopify/sarama 8aadb476e66ca998f2f6bb3c993e9a2daa3666b9
+github.com/Sirupsen/logrus 219c8cb75c258c552e999735be6df753ffc7afdc
+github.com/amir/raidman 53c1b967405155bfc8758557863bf2e14f814687
+github.com/aws/aws-sdk-go 13a12060f716145019378a10e2806c174356b857
+github.com/beorn7/perks 3ac7bf7a47d159a033b107610db8a1b6575507a4
 github.com/cenkalti/backoff 4dc77674aceaabba2c7e3da25d4c823edfb73f99
-github.com/dancannon/gorethink 6f088135ff288deb9d5546f4c71919207f891a70
+github.com/couchbase/go-couchbase cb664315a324d87d19c879d9cc67fda6be8c2ac1
+github.com/couchbase/gomemcached a5ea6356f648fec6ab89add00edd09151455b4b2
+github.com/couchbase/goutils 5823a0cbaaa9008406021dc5daf80125ea30bba6
+github.com/dancannon/gorethink e7cac92ea2bc52638791a021f212145acfedb1fc
 github.com/davecgh/go-spew 5215b55f46b2b919f50a1df0eaa5886afe4e3b3d
+github.com/docker/engine-api 8924d6900370b4c7e7984be5adc61f50a80d7537
+github.com/docker/go-connections f549a9393d05688dff0992ef3efd8bbe6c628aeb
+github.com/docker/go-units 5d2041e26a699eaca682e2ea41c8f891e1060444
 github.com/eapache/go-resiliency b86b1ec0dd4209a588dc1285cdd471e73525c0b3
 github.com/eapache/queue ded5959c0d4e360646dc9e9908cff48666781367
-github.com/fsouza/go-dockerclient 7b651349f9479f5114913eefbfd3c4eeddd79ab4
-github.com/go-ini/ini afbd495e5aaea13597b5e14fe514ddeaa4d76fc3
-github.com/go-sql-driver/mysql 7c7f556282622f94213bc028b4d0a7b6151ba239
-github.com/golang/protobuf 6aaa8d47701fa6cf07e914ec01fde3d4a1fe79c3
-github.com/golang/snappy 723cc1e459b8eea2dea4583200fd60757d40097a
+github.com/eclipse/paho.mqtt.golang 0f7a459f04f13a41b7ed752d47944528d4bf9a86
+github.com/go-sql-driver/mysql 1fca743146605a172a266e1654e01e5cd5669bee
+github.com/gobwas/glob d877f6352135181470c40c73ebb81aefa22115fa
+github.com/golang/protobuf 552c7b9542c194800fd493123b3798ef0a832032
+github.com/golang/snappy 427fb6fc07997f43afa32f35e850833760e489a7
 github.com/gonuts/go-shellquote e842a11b24c6abfb3dd27af69a17f482e4b483c2
-github.com/gorilla/context 1c83b3eabd45b6d76072b66b746c20815fb2872d
-github.com/gorilla/mux 26a6070f849969ba72b72256e9f14cf519751690
+github.com/gorilla/context 1ea25387ff6f684839d82767c1733ff4d4d15d0a
+github.com/gorilla/mux c9e326e2bdec29039a3761c07bece13133863e1e
 github.com/hailocab/go-hostpool e80d13ce29ede4452c43dea11e79b9bc8a15b478
-github.com/influxdata/config bae7cb98197d842374d3b8403905924094930f24
-github.com/influxdata/influxdb ef571fc104dc24b77cd3710c156cd95e5cfd7aa5
-github.com/jmespath/go-jmespath c01cf91b011868172fdcd9f41838e80c9d716264
-github.com/klauspost/crc32 999f3125931f6557b991b2f8472172bdfa578d38
-github.com/lib/pq 8ad2b298cadd691a77015666a5372eae5dbfac8f
+github.com/hpcloud/tail b2940955ab8b26e19d43a43c4da0475dd81bdb56
+github.com/influxdata/config b79f6829346b8d6e78ba73544b1e1038f1f1c9da
+github.com/influxdata/influxdb e094138084855d444195b252314dfee9eae34cab
+github.com/influxdata/toml af4df43894b16e3fd2b788d01bd27ad0776ef2d0
+github.com/klauspost/crc32 19b0b332c9e4516a6370a0456e6182c3b5036720
+github.com/lib/pq e182dc4027e2ded4b19396d638610f2653295f36
 github.com/matttproud/golang_protobuf_extensions d0c3fe89de86839aecf2e0579c40ba3bb336a453
+github.com/miekg/dns cce6c130cdb92c752850880fd285bea1d64439dd
 github.com/mreiferson/go-snappystream 028eae7ab5c4c9e2d1cb4c4ca1e53259bbe7e504
 github.com/naoina/go-stringutil 6b638e95a32d0c1131db0e7fe83775cbea4a0d0b
-github.com/naoina/toml 751171607256bb66e64c9f0220c00662420c38e9
-github.com/nats-io/nats 6a83f1a633cfbfd90aa648ac99fb38c06a8b40df
-github.com/nsqio/go-nsq 2118015c120962edc5d03325c680daf3163a8b5f
-github.com/pmezard/go-difflib 792786c7400a136282c1664665ae0a8db921c6c2
-github.com/prometheus/client_golang 67994f177195311c3ea3d4407ed0175e34a4256f
+github.com/nats-io/nats b13fc9d12b0b123ebc374e6b808c6228ae4234a3
+github.com/nats-io/nuid 4f84f5f3b2786224e336af2e13dba0a0a80b76fa
+github.com/nsqio/go-nsq 0b80d6f05e15ca1930e0c5e1d540ed627e299980
+github.com/opencontainers/runc 89ab7f2ccc1e45ddf6485eaa802c35dcf321dfc8
+github.com/prometheus/client_golang 18acf9993a863f4c4b40612e19cdd243e7c86831
 github.com/prometheus/client_model fa8ad6fec33561be4280a8f0514318c79d7f6cb6
-github.com/prometheus/common 14ca1097bbe21584194c15e391a9dab95ad42a59
+github.com/prometheus/common e8eabff8812b05acf522b45fdcd725a785188e37
 github.com/prometheus/procfs 406e5b7bfd8201a36e2bb5f7bdae0b03380c2ce8
 github.com/samuel/go-zookeeper 218e9c81c0dd8b3b18172b2bbfad92cc7d6db55f
-github.com/shirou/gopsutil e77438504d45b9985c99a75730fe65220ceea00e
+github.com/shirou/gopsutil 83c6e72cbdef6e8ada934549abf700ff0ba96776
 github.com/soniah/gosnmp b1b4f885b12c5dcbd021c5cee1c904110de6db7d
 github.com/streadway/amqp b4f3ceab0337f013208d31348b578d83c0064744
-github.com/stretchr/objx 1a9d0bb9f541897e62256577b352fdbc1fb4fd94
-github.com/stretchr/testify f390dcf405f7b83c997eac1b06768bb9f44dec18
-github.com/wvanbergen/kafka 1a8639a45164fcc245d5c7b4bd3ccfbd1a0ffbf3
+github.com/stretchr/testify 1f4a1643a57e798696635ea4c126e9127adb7d3c
+github.com/wvanbergen/kafka 46f9a1cf3f670edec492029fadded9c2d9e18866
 github.com/wvanbergen/kazoo-go 0f768712ae6f76454f987c3356177e138df258f8
 github.com/zensqlmonitor/go-mssqldb ffe5510c6fa5e15e6d983210ab501c815b56b363
-golang.org/x/crypto 1f22c0103821b9390939b6776727195525381532
-golang.org/x/net 04b9de9b512f58addf28c9853d50ebef61c3953e
-golang.org/x/text 6d3c22c4525a4da167968fa2479be5524d2e8bd0
-gopkg.in/dancannon/gorethink.v1 6f088135ff288deb9d5546f4c71919207f891a70
+golang.org/x/crypto 5dc8cb4b8a8eb076cbb5a06bc3b8682c15bdbbd3
+golang.org/x/net 6acef71eb69611914f7a30939ea9f6e194c78172
+golang.org/x/text a71fd10341b064c10f4a81ceac72bcf70f26ea34
+gopkg.in/dancannon/gorethink.v1 7d1af5be49cb5ecc7b177bf387d232050299d6ef
 gopkg.in/fatih/pool.v2 cba550ebf9bce999a02e963296d4bc7a486cb715
-gopkg.in/mgo.v2 03c9f3ee4c14c8e51ee521a6a7d0425658dd6f64
-gopkg.in/yaml.v2 f7716cbe52baa25d2e9b0d0da546fcf909fc16b4
-github.com/miekg/dns e0d84d97e59bcb6561eae269c4e94d25b66822cb
+gopkg.in/mgo.v2 d90005c5262a3463800497ea5a89aed5fe22c886
+gopkg.in/yaml.v2 a83829b6f1293c91addabc89d0571c246397bbf4

Godeps_windows

@@ -1,56 +1,59 @@
-git.eclipse.org/gitroot/paho/org.eclipse.paho.mqtt.golang.git 617c801af238c3af2d9e72c5d4a0f02edad03ce5
-github.com/Shopify/sarama d37c73f2b2bce85f7fa16b6a550d26c5372892ef
-github.com/Sirupsen/logrus f7f79f729e0fbe2fcc061db48a9ba0263f588252
+github.com/Microsoft/go-winio 9f57cbbcbcb41dea496528872a4f0e37a4f7ae98
+github.com/Shopify/sarama 8aadb476e66ca998f2f6bb3c993e9a2daa3666b9
+github.com/Sirupsen/logrus 219c8cb75c258c552e999735be6df753ffc7afdc
 github.com/StackExchange/wmi f3e2bae1e0cb5aef83e319133eabfee30013a4a5
-github.com/amir/raidman 6a8e089bbe32e6b907feae5ba688841974b3c339
-github.com/aws/aws-sdk-go 87b1e60a50b09e4812dee560b33a238f67305804
-github.com/beorn7/perks b965b613227fddccbfffe13eae360ed3fa822f8d
+github.com/amir/raidman 53c1b967405155bfc8758557863bf2e14f814687
+github.com/aws/aws-sdk-go 13a12060f716145019378a10e2806c174356b857
+github.com/beorn7/perks 3ac7bf7a47d159a033b107610db8a1b6575507a4
 github.com/cenkalti/backoff 4dc77674aceaabba2c7e3da25d4c823edfb73f99
-github.com/dancannon/gorethink 6f088135ff288deb9d5546f4c71919207f891a70
+github.com/couchbase/go-couchbase cb664315a324d87d19c879d9cc67fda6be8c2ac1
+github.com/couchbase/gomemcached a5ea6356f648fec6ab89add00edd09151455b4b2
+github.com/couchbase/goutils 5823a0cbaaa9008406021dc5daf80125ea30bba6
+github.com/dancannon/gorethink e7cac92ea2bc52638791a021f212145acfedb1fc
 github.com/davecgh/go-spew 5215b55f46b2b919f50a1df0eaa5886afe4e3b3d
+github.com/docker/engine-api 8924d6900370b4c7e7984be5adc61f50a80d7537
+github.com/docker/go-connections f549a9393d05688dff0992ef3efd8bbe6c628aeb
+github.com/docker/go-units 5d2041e26a699eaca682e2ea41c8f891e1060444
 github.com/eapache/go-resiliency b86b1ec0dd4209a588dc1285cdd471e73525c0b3
 github.com/eapache/queue ded5959c0d4e360646dc9e9908cff48666781367
-github.com/fsouza/go-dockerclient 7b651349f9479f5114913eefbfd3c4eeddd79ab4
-github.com/go-ini/ini afbd495e5aaea13597b5e14fe514ddeaa4d76fc3
+github.com/eclipse/paho.mqtt.golang 0f7a459f04f13a41b7ed752d47944528d4bf9a86
 github.com/go-ole/go-ole 50055884d646dd9434f16bbb5c9801749b9bafe4
-github.com/go-sql-driver/mysql 7c7f556282622f94213bc028b4d0a7b6151ba239
-github.com/golang/protobuf 6aaa8d47701fa6cf07e914ec01fde3d4a1fe79c3
-github.com/golang/snappy 723cc1e459b8eea2dea4583200fd60757d40097a
+github.com/go-sql-driver/mysql 1fca743146605a172a266e1654e01e5cd5669bee
+github.com/golang/protobuf 552c7b9542c194800fd493123b3798ef0a832032
+github.com/golang/snappy 427fb6fc07997f43afa32f35e850833760e489a7
 github.com/gonuts/go-shellquote e842a11b24c6abfb3dd27af69a17f482e4b483c2
-github.com/gorilla/context 1c83b3eabd45b6d76072b66b746c20815fb2872d
-github.com/gorilla/mux 26a6070f849969ba72b72256e9f14cf519751690
+github.com/gorilla/context 1ea25387ff6f684839d82767c1733ff4d4d15d0a
+github.com/gorilla/mux c9e326e2bdec29039a3761c07bece13133863e1e
 github.com/hailocab/go-hostpool e80d13ce29ede4452c43dea11e79b9bc8a15b478
-github.com/influxdata/config bae7cb98197d842374d3b8403905924094930f24
-github.com/influxdata/influxdb ef571fc104dc24b77cd3710c156cd95e5cfd7aa5
-github.com/jmespath/go-jmespath c01cf91b011868172fdcd9f41838e80c9d716264
-github.com/klauspost/crc32 999f3125931f6557b991b2f8472172bdfa578d38
-github.com/lib/pq 8ad2b298cadd691a77015666a5372eae5dbfac8f
+github.com/influxdata/config b79f6829346b8d6e78ba73544b1e1038f1f1c9da
+github.com/influxdata/influxdb e3fef5593c21644f2b43af55d6e17e70910b0e48
+github.com/influxdata/toml af4df43894b16e3fd2b788d01bd27ad0776ef2d0
+github.com/klauspost/crc32 19b0b332c9e4516a6370a0456e6182c3b5036720
+github.com/lib/pq e182dc4027e2ded4b19396d638610f2653295f36
 github.com/lxn/win 9a7734ea4db26bc593d52f6a8a957afdad39c5c1
 github.com/matttproud/golang_protobuf_extensions d0c3fe89de86839aecf2e0579c40ba3bb336a453
-github.com/miekg/dns e0d84d97e59bcb6561eae269c4e94d25b66822cb
+github.com/miekg/dns cce6c130cdb92c752850880fd285bea1d64439dd
 github.com/mreiferson/go-snappystream 028eae7ab5c4c9e2d1cb4c4ca1e53259bbe7e504
 github.com/naoina/go-stringutil 6b638e95a32d0c1131db0e7fe83775cbea4a0d0b
-github.com/naoina/toml 751171607256bb66e64c9f0220c00662420c38e9
-github.com/nats-io/nats 6a83f1a633cfbfd90aa648ac99fb38c06a8b40df
-github.com/nsqio/go-nsq 2118015c120962edc5d03325c680daf3163a8b5f
-github.com/pmezard/go-difflib 792786c7400a136282c1664665ae0a8db921c6c2
-github.com/prometheus/client_golang 67994f177195311c3ea3d4407ed0175e34a4256f
+github.com/nats-io/nats b13fc9d12b0b123ebc374e6b808c6228ae4234a3
+github.com/nats-io/nuid 4f84f5f3b2786224e336af2e13dba0a0a80b76fa
+github.com/nsqio/go-nsq 0b80d6f05e15ca1930e0c5e1d540ed627e299980
+github.com/prometheus/client_golang 18acf9993a863f4c4b40612e19cdd243e7c86831
 github.com/prometheus/client_model fa8ad6fec33561be4280a8f0514318c79d7f6cb6
-github.com/prometheus/common 14ca1097bbe21584194c15e391a9dab95ad42a59
+github.com/prometheus/common e8eabff8812b05acf522b45fdcd725a785188e37
 github.com/prometheus/procfs 406e5b7bfd8201a36e2bb5f7bdae0b03380c2ce8
 github.com/samuel/go-zookeeper 218e9c81c0dd8b3b18172b2bbfad92cc7d6db55f
-github.com/shirou/gopsutil e77438504d45b9985c99a75730fe65220ceea00e
+github.com/shirou/gopsutil 1f32ce1bb380845be7f5d174ac641a2c592c0c42
 github.com/shirou/w32 ada3ba68f000aa1b58580e45c9d308fe0b7fc5c5
 github.com/soniah/gosnmp b1b4f885b12c5dcbd021c5cee1c904110de6db7d
 github.com/streadway/amqp b4f3ceab0337f013208d31348b578d83c0064744
-github.com/stretchr/objx 1a9d0bb9f541897e62256577b352fdbc1fb4fd94
-github.com/stretchr/testify f390dcf405f7b83c997eac1b06768bb9f44dec18
-github.com/wvanbergen/kafka 1a8639a45164fcc245d5c7b4bd3ccfbd1a0ffbf3
+github.com/stretchr/testify 1f4a1643a57e798696635ea4c126e9127adb7d3c
+github.com/wvanbergen/kafka 46f9a1cf3f670edec492029fadded9c2d9e18866
 github.com/wvanbergen/kazoo-go 0f768712ae6f76454f987c3356177e138df258f8
 github.com/zensqlmonitor/go-mssqldb ffe5510c6fa5e15e6d983210ab501c815b56b363
-golang.org/x/net 04b9de9b512f58addf28c9853d50ebef61c3953e
-golang.org/x/text 6d3c22c4525a4da167968fa2479be5524d2e8bd0
-gopkg.in/dancannon/gorethink.v1 6f088135ff288deb9d5546f4c71919207f891a70
+golang.org/x/net 6acef71eb69611914f7a30939ea9f6e194c78172
+golang.org/x/text a71fd10341b064c10f4a81ceac72bcf70f26ea34
+gopkg.in/dancannon/gorethink.v1 7d1af5be49cb5ecc7b177bf387d232050299d6ef
 gopkg.in/fatih/pool.v2 cba550ebf9bce999a02e963296d4bc7a486cb715
-gopkg.in/mgo.v2 03c9f3ee4c14c8e51ee521a6a7d0425658dd6f64
-gopkg.in/yaml.v2 f7716cbe52baa25d2e9b0d0da546fcf909fc16b4
+gopkg.in/mgo.v2 d90005c5262a3463800497ea5a89aed5fe22c886
+gopkg.in/yaml.v2 a83829b6f1293c91addabc89d0571c246397bbf4

Makefile

@@ -14,21 +14,21 @@ windows: prepare-windows build-windows
 
 # Only run the build (no dependency grabbing)
 build:
-	go install -ldflags "-X main.Version=$(VERSION)" ./...
+	go install -ldflags "-X main.version=$(VERSION)" ./...
 
 build-windows:
 	go build -o telegraf.exe -ldflags \
-		"-X main.Version=$(VERSION)" \
+		"-X main.version=$(VERSION)" \
 		./cmd/telegraf/telegraf.go
 
 build-for-docker:
-	CGO_ENABLED=0 GOOS=linux go -o telegraf -ldflags \
-					"-X main.Version=$(VERSION)" \
+	CGO_ENABLED=0 GOOS=linux go build -installsuffix cgo -o telegraf -ldflags \
+					"-s -X main.version=$(VERSION)" \
 					./cmd/telegraf/telegraf.go
 
 # Build with race detector
 dev: prepare
-	go build -race -ldflags "-X main.Version=$(VERSION)" ./...
+	go build -race -ldflags "-X main.version=$(VERSION)" ./...
 
 # run package script
 package:

README.md

@@ -1,4 +1,4 @@
-# Telegraf [![Circle CI](https://circleci.com/gh/influxdata/telegraf.svg?style=svg)](https://circleci.com/gh/influxdata/telegraf)
+# Telegraf [![Circle CI](https://circleci.com/gh/influxdata/telegraf.svg?style=svg)](https://circleci.com/gh/influxdata/telegraf) [![Docker pulls](https://img.shields.io/docker/pulls/library/telegraf.svg)](https://hub.docker.com/_/telegraf/)
 
 Telegraf is an agent written in Go for collecting metrics from the system it's
 running on, or from other services, and writing them into InfluxDB or other
@@ -17,26 +17,15 @@ new plugins.
 
 ## Installation:
 
-NOTE: Telegraf 0.10.x is **not** backwards-compatible with previous versions
-of telegraf, both in the database layout and the configuration file. 0.2.x
-will continue to be supported, see below for download links.
-
-For more details on the differences between Telegraf 0.2.x and 0.10.x, see
-the [release blog post](https://influxdata.com/blog/announcing-telegraf-0-10-0/).
-
 ### Linux deb and rpm Packages:
 
 Latest:
-* http://get.influxdb.org/telegraf/telegraf_0.10.3-1_amd64.deb
-* http://get.influxdb.org/telegraf/telegraf-0.10.3-1.x86_64.rpm
+* https://dl.influxdata.com/telegraf/releases/telegraf_0.13.1_amd64.deb
+* https://dl.influxdata.com/telegraf/releases/telegraf-0.13.1.x86_64.rpm
 
 Latest (arm):
-* http://get.influxdb.org/telegraf/telegraf_0.10.3-1_arm.deb
-* http://get.influxdb.org/telegraf/telegraf-0.10.3-1.arm.rpm
-
-0.2.x:
-* http://get.influxdb.org/telegraf/telegraf_0.2.4_amd64.deb
-* http://get.influxdb.org/telegraf/telegraf-0.2.4-1.x86_64.rpm
+* https://dl.influxdata.com/telegraf/releases/telegraf_0.13.1_armhf.deb
+* https://dl.influxdata.com/telegraf/releases/telegraf-0.13.1.armhf.rpm
 
 ##### Package Instructions:
 
@@ -50,34 +39,21 @@ controlled via `systemctl [action] telegraf`
 ### yum/apt Repositories:
 
 There is a yum/apt repo available for the whole InfluxData stack, see
-[here](https://docs.influxdata.com/influxdb/v0.9/introduction/installation/#installation)
-for instructions, replacing the `influxdb` package name with `telegraf`.
+[here](https://docs.influxdata.com/influxdb/v0.10/introduction/installation/#installation)
+for instructions on setting up the repo. Once it is configured, you will be able
+to use this repo to install & update telegraf.
 
 ### Linux tarballs:
 
 Latest:
-* http://get.influxdb.org/telegraf/telegraf-0.10.3-1_linux_amd64.tar.gz
-* http://get.influxdb.org/telegraf/telegraf-0.10.3-1_linux_i386.tar.gz
-* http://get.influxdb.org/telegraf/telegraf-0.10.3-1_linux_arm.tar.gz
+* https://dl.influxdata.com/telegraf/releases/telegraf-0.13.1_linux_amd64.tar.gz
+* https://dl.influxdata.com/telegraf/releases/telegraf-0.13.1_linux_i386.tar.gz
+* https://dl.influxdata.com/telegraf/releases/telegraf-0.13.1_linux_armhf.tar.gz
 
-0.2.x:
-* http://get.influxdb.org/telegraf/telegraf_linux_amd64_0.2.4.tar.gz
-* http://get.influxdb.org/telegraf/telegraf_linux_386_0.2.4.tar.gz
-* http://get.influxdb.org/telegraf/telegraf_linux_arm_0.2.4.tar.gz
+### FreeBSD tarball:
 
-##### tarball Instructions:
-
-To install the full directory structure with config file, run:
-
-```
-sudo tar -C / -zxvf ./telegraf-0.10.3-1_linux_amd64.tar.gz
-```
-
-To extract only the binary, run:
-
-```
-tar -zxvf telegraf-0.10.3-1_linux_amd64.tar.gz --strip-components=3 ./usr/bin/telegraf
-```
+Latest:
+* https://dl.influxdata.com/telegraf/releases/telegraf-0.13.1_freebsd_amd64.tar.gz
 
 ### Ansible Role:
 
@@ -90,6 +66,12 @@ brew update
 brew install telegraf
 ```
 
+### Windows Binaries (EXPERIMENTAL)
+
+Latest:
+* https://dl.influxdata.com/telegraf/releases/telegraf-0.13.1_windows_amd64.zip
+* https://dl.influxdata.com/telegraf/releases/telegraf-0.13.1_windows_i386.zip
+
 ### From Source:
 
 Telegraf manages dependencies via [gdm](https://github.com/sparrc/gdm),
@@ -156,51 +138,64 @@ more information on each, please look at the directory of the same name in
 
 Currently implemented sources:
 
-* aerospike
-* apache
-* bcache
-* couchdb
-* disque
-* dns query time
-* docker
-* dovecot
-* elasticsearch
-* exec (generic executable plugin, support JSON, influx and graphite)
-* haproxy
-* httpjson (generic JSON-emitting http service plugin)
-* influxdb
-* jolokia
-* leofs
-* lustre2
-* mailchimp
-* memcached
-* mesos
-* mongodb
-* mysql
-* net_response
-* nginx
-* nsq
-* phpfpm
-* phusion passenger
-* ping
-* postgresql
-* powerdns
-* procstat
-* prometheus
-* puppetagent
-* rabbitmq
-* raindrops
-* redis
-* rethinkdb
-* riak
-* sensors (only available if built from source)
-* snmp
-* sql server (microsoft)
-* twemproxy
-* zfs
-* zookeeper
-* win_perf_counters (windows performance counters)
-* system
+* [aws cloudwatch](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/cloudwatch)
+* [aerospike](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/aerospike)
+* [apache](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/apache)
+* [bcache](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/bcache)
+* [cassandra](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/cassandra)
+* [ceph](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/ceph)
+* [chrony](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/chrony)
+* [couchbase](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/couchbase)
+* [couchdb](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/couchdb)
+* [disque](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/disque)
+* [dns query time](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/dns_query)
+* [docker](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/docker)
+* [dovecot](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/dovecot)
+* [elasticsearch](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/elasticsearch)
+* [exec](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/exec) (generic executable plugin, support JSON, influx, graphite and nagios)
+* [filestat](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/filestat)
+* [haproxy](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/haproxy)
+* [http_response](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/http_response)
+* [httpjson](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/httpjson) (generic JSON-emitting http service plugin)
+* [influxdb](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/influxdb)
+* [ipmi_sensor](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/ipmi_sensor)
+* [jolokia](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/jolokia)
+* [leofs](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/leofs)
+* [lustre2](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/lustre2)
+* [mailchimp](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/mailchimp)
+* [memcached](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/memcached)
+* [mesos](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/mesos)
+* [mongodb](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/mongodb)
+* [mysql](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/mysql)
+* [net_response](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/net_response)
+* [nginx](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/nginx)
+* [nsq](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/nsq)
+* [nstat](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/nstat)
+* [ntpq](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/ntpq)
+* [phpfpm](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/phpfpm)
+* [phusion passenger](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/passenger)
+* [ping](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/ping)
+* [postgresql](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/postgresql)
+* [postgresql_extensible](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/postgresql_extensible)
+* [powerdns](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/powerdns)
+* [procstat](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/procstat)
+* [prometheus](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/prometheus)
+* [puppetagent](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/puppetagent)
+* [rabbitmq](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/rabbitmq)
+* [raindrops](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/raindrops)
+* [redis](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/redis)
+* [rethinkdb](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/rethinkdb)
+* [riak](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/riak)
+* [sensors ](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/sensors) (only available if built from source)
+* [snmp](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/snmp)
+* [sql server](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/sqlserver) (microsoft)
+* [twemproxy](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/twemproxy)
+* [varnish](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/varnish)
+* [zfs](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/zfs)
+* [zookeeper](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/zookeeper)
+* [win_perf_counters ](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/win_perf_counters) (windows performance counters)
+* [sysstat](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/sysstat)
+* [system](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/system)
     * cpu
     * mem
     * net
@@ -208,34 +203,41 @@ Currently implemented sources:
     * disk
     * diskio
     * swap
+    * processes
+    * kernel (/proc/stat)
 
 Telegraf can also collect metrics via the following service plugins:
 
-* statsd
-* mqtt_consumer
-* kafka_consumer
-* nats_consumer
-* github_webhooks
+* [statsd](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/statsd)
+* [tail](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/tail)
+* [udp_listener](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/udp_listener)
+* [tcp_listener](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/tcp_listener)
+* [mqtt_consumer](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/mqtt_consumer)
+* [kafka_consumer](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/kafka_consumer)
+* [nats_consumer](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/nats_consumer)
+* [github_webhooks](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/github_webhooks)
 
 We'll be adding support for many more over the coming months. Read on if you
 want to add support for another service or third-party API.
 
 ## Supported Output Plugins
 
-* influxdb
-* amon
-* amqp
-* aws kinesis
-* aws cloudwatch
-* datadog
-* graphite
-* kafka
-* librato
-* mqtt
-* nsq
-* opentsdb
-* prometheus
-* riemann
+* [influxdb](https://github.com/influxdata/telegraf/tree/master/plugins/outputs/influxdb)
+* [amon](https://github.com/influxdata/telegraf/tree/master/plugins/outputs/amon)
+* [amqp](https://github.com/influxdata/telegraf/tree/master/plugins/outputs/amqp)
+* [aws kinesis](https://github.com/influxdata/telegraf/tree/master/plugins/outputs/kinesis)
+* [aws cloudwatch](https://github.com/influxdata/telegraf/tree/master/plugins/outputs/cloudwatch)
+* [datadog](https://github.com/influxdata/telegraf/tree/master/plugins/outputs/datadog)
+* [file](https://github.com/influxdata/telegraf/tree/master/plugins/outputs/file)
+* [graphite](https://github.com/influxdata/telegraf/tree/master/plugins/outputs/graphite)
+* [instrumental](https://github.com/influxdata/telegraf/tree/master/plugins/outputs/instrumental)
+* [kafka](https://github.com/influxdata/telegraf/tree/master/plugins/outputs/kafka)
+* [librato](https://github.com/influxdata/telegraf/tree/master/plugins/outputs/librato)
+* [mqtt](https://github.com/influxdata/telegraf/tree/master/plugins/outputs/mqtt)
+* [nsq](https://github.com/influxdata/telegraf/tree/master/plugins/outputs/nsq)
+* [opentsdb](https://github.com/influxdata/telegraf/tree/master/plugins/outputs/opentsdb)
+* [prometheus](https://github.com/influxdata/telegraf/tree/master/plugins/outputs/prometheus_client)
+* [riemann](https://github.com/influxdata/telegraf/tree/master/plugins/outputs/riemann)
 
 ## Contributing
 

agent/accumulator.go

@@ -4,7 +4,6 @@ import (
 	"fmt"
 	"log"
 	"math"
-	"sync"
 	"time"
 
 	"github.com/influxdata/telegraf"
@@ -22,13 +21,13 @@ func NewAccumulator(
 }
 
 type accumulator struct {
-	sync.Mutex
-
 	metrics chan telegraf.Metric
 
 	defaultTags map[string]string
 
 	debug bool
+	// print every point added to the accumulator
+	trace bool
 
 	inputConfig *internal_models.InputConfig
 
@@ -96,6 +95,7 @@ func (ac *accumulator) AddFields(
 			tags[k] = v
 		}
 	}
+	ac.inputConfig.Filter.FilterTags(tags)
 
 	result := make(map[string]interface{})
 	for k, v := range fields {
@@ -105,7 +105,6 @@ func (ac *accumulator) AddFields(
 				continue
 			}
 		}
-		result[k] = v
 
 		// Validate uint64 and float64 fields
 		switch val := v.(type) {
@@ -116,6 +115,7 @@ func (ac *accumulator) AddFields(
 			} else {
 				result[k] = int64(9223372036854775807)
 			}
+			continue
 		case float64:
 			// NaNs are invalid values in influxdb, skip measurement
 			if math.IsNaN(val) || math.IsInf(val, 0) {
@@ -127,6 +127,8 @@ func (ac *accumulator) AddFields(
 				continue
 			}
 		}
+
+		result[k] = v
 	}
 	fields = nil
 	if len(result) == 0 {
@@ -149,7 +151,7 @@ func (ac *accumulator) AddFields(
 		log.Printf("Error adding point [%s]: %s\n", measurement, err.Error())
 		return
 	}
-	if ac.debug {
+	if ac.trace {
 		fmt.Println("> " + m.String())
 	}
 	ac.metrics <- m
@@ -163,10 +165,21 @@ func (ac *accumulator) SetDebug(debug bool) {
 	ac.debug = debug
 }
 
+func (ac *accumulator) Trace() bool {
+	return ac.trace
+}
+
+func (ac *accumulator) SetTrace(trace bool) {
+	ac.trace = trace
+}
+
 func (ac *accumulator) setDefaultTags(tags map[string]string) {
 	ac.defaultTags = tags
 }
 
 func (ac *accumulator) addDefaultTag(key, value string) {
+	if ac.defaultTags == nil {
+		ac.defaultTags = make(map[string]string)
+	}
 	ac.defaultTags[key] = value
 }

agent/accumulator_test.go

@@ -0,0 +1,334 @@
+package agent
+
+import (
+	"fmt"
+	"math"
+	"testing"
+	"time"
+
+	"github.com/influxdata/telegraf"
+	"github.com/influxdata/telegraf/internal/models"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestAdd(t *testing.T) {
+	a := accumulator{}
+	now := time.Now()
+	a.metrics = make(chan telegraf.Metric, 10)
+	defer close(a.metrics)
+	a.inputConfig = &internal_models.InputConfig{}
+
+	a.Add("acctest", float64(101), map[string]string{})
+	a.Add("acctest", float64(101), map[string]string{"acc": "test"})
+	a.Add("acctest", float64(101), map[string]string{"acc": "test"}, now)
+
+	testm := <-a.metrics
+	actual := testm.String()
+	assert.Contains(t, actual, "acctest value=101")
+
+	testm = <-a.metrics
+	actual = testm.String()
+	assert.Contains(t, actual, "acctest,acc=test value=101")
+
+	testm = <-a.metrics
+	actual = testm.String()
+	assert.Equal(t,
+		fmt.Sprintf("acctest,acc=test value=101 %d", now.UnixNano()),
+		actual)
+}
+
+func TestAddDefaultTags(t *testing.T) {
+	a := accumulator{}
+	a.addDefaultTag("default", "tag")
+	now := time.Now()
+	a.metrics = make(chan telegraf.Metric, 10)
+	defer close(a.metrics)
+	a.inputConfig = &internal_models.InputConfig{}
+
+	a.Add("acctest", float64(101), map[string]string{})
+	a.Add("acctest", float64(101), map[string]string{"acc": "test"})
+	a.Add("acctest", float64(101), map[string]string{"acc": "test"}, now)
+
+	testm := <-a.metrics
+	actual := testm.String()
+	assert.Contains(t, actual, "acctest,default=tag value=101")
+
+	testm = <-a.metrics
+	actual = testm.String()
+	assert.Contains(t, actual, "acctest,acc=test,default=tag value=101")
+
+	testm = <-a.metrics
+	actual = testm.String()
+	assert.Equal(t,
+		fmt.Sprintf("acctest,acc=test,default=tag value=101 %d", now.UnixNano()),
+		actual)
+}
+
+func TestAddFields(t *testing.T) {
+	a := accumulator{}
+	now := time.Now()
+	a.metrics = make(chan telegraf.Metric, 10)
+	defer close(a.metrics)
+	a.inputConfig = &internal_models.InputConfig{}
+
+	fields := map[string]interface{}{
+		"usage": float64(99),
+	}
+	a.AddFields("acctest", fields, map[string]string{})
+	a.AddFields("acctest", fields, map[string]string{"acc": "test"})
+	a.AddFields("acctest", fields, map[string]string{"acc": "test"}, now)
+
+	testm := <-a.metrics
+	actual := testm.String()
+	assert.Contains(t, actual, "acctest usage=99")
+
+	testm = <-a.metrics
+	actual = testm.String()
+	assert.Contains(t, actual, "acctest,acc=test usage=99")
+
+	testm = <-a.metrics
+	actual = testm.String()
+	assert.Equal(t,
+		fmt.Sprintf("acctest,acc=test usage=99 %d", now.UnixNano()),
+		actual)
+}
+
+// Test that all Inf fields get dropped, and not added to metrics channel
+func TestAddInfFields(t *testing.T) {
+	inf := math.Inf(1)
+	ninf := math.Inf(-1)
+
+	a := accumulator{}
+	now := time.Now()
+	a.metrics = make(chan telegraf.Metric, 10)
+	defer close(a.metrics)
+	a.inputConfig = &internal_models.InputConfig{}
+
+	fields := map[string]interface{}{
+		"usage":  inf,
+		"nusage": ninf,
+	}
+	a.AddFields("acctest", fields, map[string]string{})
+	a.AddFields("acctest", fields, map[string]string{"acc": "test"})
+	a.AddFields("acctest", fields, map[string]string{"acc": "test"}, now)
+
+	assert.Len(t, a.metrics, 0)
+
+	// test that non-inf fields are kept and not dropped
+	fields["notinf"] = float64(100)
+	a.AddFields("acctest", fields, map[string]string{})
+	testm := <-a.metrics
+	actual := testm.String()
+	assert.Contains(t, actual, "acctest notinf=100")
+}
+
+// Test that nan fields are dropped and not added
+func TestAddNaNFields(t *testing.T) {
+	nan := math.NaN()
+
+	a := accumulator{}
+	now := time.Now()
+	a.metrics = make(chan telegraf.Metric, 10)
+	defer close(a.metrics)
+	a.inputConfig = &internal_models.InputConfig{}
+
+	fields := map[string]interface{}{
+		"usage": nan,
+	}
+	a.AddFields("acctest", fields, map[string]string{})
+	a.AddFields("acctest", fields, map[string]string{"acc": "test"})
+	a.AddFields("acctest", fields, map[string]string{"acc": "test"}, now)
+
+	assert.Len(t, a.metrics, 0)
+
+	// test that non-nan fields are kept and not dropped
+	fields["notnan"] = float64(100)
+	a.AddFields("acctest", fields, map[string]string{})
+	testm := <-a.metrics
+	actual := testm.String()
+	assert.Contains(t, actual, "acctest notnan=100")
+}
+
+func TestAddUint64Fields(t *testing.T) {
+	a := accumulator{}
+	now := time.Now()
+	a.metrics = make(chan telegraf.Metric, 10)
+	defer close(a.metrics)
+	a.inputConfig = &internal_models.InputConfig{}
+
+	fields := map[string]interface{}{
+		"usage": uint64(99),
+	}
+	a.AddFields("acctest", fields, map[string]string{})
+	a.AddFields("acctest", fields, map[string]string{"acc": "test"})
+	a.AddFields("acctest", fields, map[string]string{"acc": "test"}, now)
+
+	testm := <-a.metrics
+	actual := testm.String()
+	assert.Contains(t, actual, "acctest usage=99i")
+
+	testm = <-a.metrics
+	actual = testm.String()
+	assert.Contains(t, actual, "acctest,acc=test usage=99i")
+
+	testm = <-a.metrics
+	actual = testm.String()
+	assert.Equal(t,
+		fmt.Sprintf("acctest,acc=test usage=99i %d", now.UnixNano()),
+		actual)
+}
+
+func TestAddUint64Overflow(t *testing.T) {
+	a := accumulator{}
+	now := time.Now()
+	a.metrics = make(chan telegraf.Metric, 10)
+	defer close(a.metrics)
+	a.inputConfig = &internal_models.InputConfig{}
+
+	fields := map[string]interface{}{
+		"usage": uint64(9223372036854775808),
+	}
+	a.AddFields("acctest", fields, map[string]string{})
+	a.AddFields("acctest", fields, map[string]string{"acc": "test"})
+	a.AddFields("acctest", fields, map[string]string{"acc": "test"}, now)
+
+	testm := <-a.metrics
+	actual := testm.String()
+	assert.Contains(t, actual, "acctest usage=9223372036854775807i")
+
+	testm = <-a.metrics
+	actual = testm.String()
+	assert.Contains(t, actual, "acctest,acc=test usage=9223372036854775807i")
+
+	testm = <-a.metrics
+	actual = testm.String()
+	assert.Equal(t,
+		fmt.Sprintf("acctest,acc=test usage=9223372036854775807i %d", now.UnixNano()),
+		actual)
+}
+
+func TestAddInts(t *testing.T) {
+	a := accumulator{}
+	a.addDefaultTag("default", "tag")
+	now := time.Now()
+	a.metrics = make(chan telegraf.Metric, 10)
+	defer close(a.metrics)
+	a.inputConfig = &internal_models.InputConfig{}
+
+	a.Add("acctest", int(101), map[string]string{})
+	a.Add("acctest", int32(101), map[string]string{"acc": "test"})
+	a.Add("acctest", int64(101), map[string]string{"acc": "test"}, now)
+
+	testm := <-a.metrics
+	actual := testm.String()
+	assert.Contains(t, actual, "acctest,default=tag value=101i")
+
+	testm = <-a.metrics
+	actual = testm.String()
+	assert.Contains(t, actual, "acctest,acc=test,default=tag value=101i")
+
+	testm = <-a.metrics
+	actual = testm.String()
+	assert.Equal(t,
+		fmt.Sprintf("acctest,acc=test,default=tag value=101i %d", now.UnixNano()),
+		actual)
+}
+
+func TestAddFloats(t *testing.T) {
+	a := accumulator{}
+	a.addDefaultTag("default", "tag")
+	now := time.Now()
+	a.metrics = make(chan telegraf.Metric, 10)
+	defer close(a.metrics)
+	a.inputConfig = &internal_models.InputConfig{}
+
+	a.Add("acctest", float32(101), map[string]string{"acc": "test"})
+	a.Add("acctest", float64(101), map[string]string{"acc": "test"}, now)
+
+	testm := <-a.metrics
+	actual := testm.String()
+	assert.Contains(t, actual, "acctest,acc=test,default=tag value=101")
+
+	testm = <-a.metrics
+	actual = testm.String()
+	assert.Equal(t,
+		fmt.Sprintf("acctest,acc=test,default=tag value=101 %d", now.UnixNano()),
+		actual)
+}
+
+func TestAddStrings(t *testing.T) {
+	a := accumulator{}
+	a.addDefaultTag("default", "tag")
+	now := time.Now()
+	a.metrics = make(chan telegraf.Metric, 10)
+	defer close(a.metrics)
+	a.inputConfig = &internal_models.InputConfig{}
+
+	a.Add("acctest", "test", map[string]string{"acc": "test"})
+	a.Add("acctest", "foo", map[string]string{"acc": "test"}, now)
+
+	testm := <-a.metrics
+	actual := testm.String()
+	assert.Contains(t, actual, "acctest,acc=test,default=tag value=\"test\"")
+
+	testm = <-a.metrics
+	actual = testm.String()
+	assert.Equal(t,
+		fmt.Sprintf("acctest,acc=test,default=tag value=\"foo\" %d", now.UnixNano()),
+		actual)
+}
+
+func TestAddBools(t *testing.T) {
+	a := accumulator{}
+	a.addDefaultTag("default", "tag")
+	now := time.Now()
+	a.metrics = make(chan telegraf.Metric, 10)
+	defer close(a.metrics)
+	a.inputConfig = &internal_models.InputConfig{}
+
+	a.Add("acctest", true, map[string]string{"acc": "test"})
+	a.Add("acctest", false, map[string]string{"acc": "test"}, now)
+
+	testm := <-a.metrics
+	actual := testm.String()
+	assert.Contains(t, actual, "acctest,acc=test,default=tag value=true")
+
+	testm = <-a.metrics
+	actual = testm.String()
+	assert.Equal(t,
+		fmt.Sprintf("acctest,acc=test,default=tag value=false %d", now.UnixNano()),
+		actual)
+}
+
+// Test that tag filters get applied to metrics.
+func TestAccFilterTags(t *testing.T) {
+	a := accumulator{}
+	now := time.Now()
+	a.metrics = make(chan telegraf.Metric, 10)
+	defer close(a.metrics)
+	filter := internal_models.Filter{
+		TagExclude: []string{"acc"},
+	}
+	assert.NoError(t, filter.CompileFilter())
+	a.inputConfig = &internal_models.InputConfig{}
+	a.inputConfig.Filter = filter
+
+	a.Add("acctest", float64(101), map[string]string{})
+	a.Add("acctest", float64(101), map[string]string{"acc": "test"})
+	a.Add("acctest", float64(101), map[string]string{"acc": "test"}, now)
+
+	testm := <-a.metrics
+	actual := testm.String()
+	assert.Contains(t, actual, "acctest value=101")
+
+	testm = <-a.metrics
+	actual = testm.String()
+	assert.Contains(t, actual, "acctest value=101")
+
+	testm = <-a.metrics
+	actual = testm.String()
+	assert.Equal(t,
+		fmt.Sprintf("acctest value=101 %d", now.UnixNano()),
+		actual)
+}

agent/agent.go

@@ -27,17 +27,19 @@ func NewAgent(config *config.Config) (*Agent, error) {
 		Config: config,
 	}
 
-	if a.Config.Agent.Hostname == "" {
-		hostname, err := os.Hostname()
-		if err != nil {
-			return nil, err
+	if !a.Config.Agent.OmitHostname {
+		if a.Config.Agent.Hostname == "" {
+			hostname, err := os.Hostname()
+			if err != nil {
+				return nil, err
+			}
+
+			a.Config.Agent.Hostname = hostname
 		}
 
-		a.Config.Agent.Hostname = hostname
+		config.Tags["host"] = a.Config.Agent.Hostname
 	}
 
-	config.Tags["host"] = a.Config.Agent.Hostname
-
 	return a, nil
 }
 
@@ -100,71 +102,20 @@ func panicRecover(input *internal_models.RunningInput) {
 	}
 }
 
-// gatherParallel runs the inputs that are using the same reporting interval
-// as the telegraf agent.
-func (a *Agent) gatherParallel(metricC chan telegraf.Metric) error {
-	var wg sync.WaitGroup
-
-	start := time.Now()
-	counter := 0
-	jitter := a.Config.Agent.CollectionJitter.Duration.Nanoseconds()
-	for _, input := range a.Config.Inputs {
-		if input.Config.Interval != 0 {
-			continue
-		}
-
-		wg.Add(1)
-		counter++
-		go func(input *internal_models.RunningInput) {
-			defer panicRecover(input)
-			defer wg.Done()
-
-			acc := NewAccumulator(input.Config, metricC)
-			acc.SetDebug(a.Config.Agent.Debug)
-			acc.setDefaultTags(a.Config.Tags)
-
-			if jitter != 0 {
-				nanoSleep := rand.Int63n(jitter)
-				d, err := time.ParseDuration(fmt.Sprintf("%dns", nanoSleep))
-				if err != nil {
-					log.Printf("Jittering collection interval failed for plugin %s",
-						input.Name)
-				} else {
-					time.Sleep(d)
-				}
-			}
-
-			if err := input.Input.Gather(acc); err != nil {
-				log.Printf("Error in input [%s]: %s", input.Name, err)
-			}
-
-		}(input)
-	}
-
-	if counter == 0 {
-		return nil
-	}
-
-	wg.Wait()
-
-	elapsed := time.Since(start)
-	if !a.Config.Agent.Quiet {
-		log.Printf("Gathered metrics, (%s interval), from %d inputs in %s\n",
-			a.Config.Agent.Interval.Duration, counter, elapsed)
-	}
-	return nil
-}
-
-// gatherSeparate runs the inputs that have been configured with their own
+// gatherer runs the inputs that have been configured with their own
 // reporting interval.
-func (a *Agent) gatherSeparate(
+func (a *Agent) gatherer(
 	shutdown chan struct{},
 	input *internal_models.RunningInput,
+	interval time.Duration,
 	metricC chan telegraf.Metric,
 ) error {
 	defer panicRecover(input)
 
-	ticker := time.NewTicker(input.Config.Interval)
+	ticker := time.NewTicker(interval)
+	defer ticker.Stop()
+
+	jitter := a.Config.Agent.CollectionJitter.Duration.Nanoseconds()
 
 	for {
 		var outerr error
@@ -174,19 +125,27 @@ func (a *Agent) gatherSeparate(
 		acc.SetDebug(a.Config.Agent.Debug)
 		acc.setDefaultTags(a.Config.Tags)
 
-		if err := input.Input.Gather(acc); err != nil {
-			log.Printf("Error in input [%s]: %s", input.Name, err)
+		if jitter != 0 {
+			nanoSleep := rand.Int63n(jitter)
+			d, err := time.ParseDuration(fmt.Sprintf("%dns", nanoSleep))
+			if err != nil {
+				log.Printf("Jittering collection interval failed for plugin %s",
+					input.Name)
+			} else {
+				time.Sleep(d)
+			}
 		}
 
+		gatherWithTimeout(shutdown, input, acc, interval)
 		elapsed := time.Since(start)
-		if !a.Config.Agent.Quiet {
-			log.Printf("Gathered metrics, (separate %s interval), from %s in %s\n",
-				input.Config.Interval, input.Name, elapsed)
-		}
 
 		if outerr != nil {
 			return outerr
 		}
+		if a.Config.Agent.Debug {
+			log.Printf("Input [%s] gathered metrics, (%s interval) in %s\n",
+				input.Name, interval, elapsed)
+		}
 
 		select {
 		case <-shutdown:
@@ -197,6 +156,42 @@ func (a *Agent) gatherSeparate(
 	}
 }
 
+// gatherWithTimeout gathers from the given input, with the given timeout.
+//   when the given timeout is reached, gatherWithTimeout logs an error message
+//   but continues waiting for it to return. This is to avoid leaving behind
+//   hung processes, and to prevent re-calling the same hung process over and
+//   over.
+func gatherWithTimeout(
+	shutdown chan struct{},
+	input *internal_models.RunningInput,
+	acc *accumulator,
+	timeout time.Duration,
+) {
+	ticker := time.NewTicker(timeout)
+	defer ticker.Stop()
+	done := make(chan error)
+	go func() {
+		done <- input.Input.Gather(acc)
+	}()
+
+	for {
+		select {
+		case err := <-done:
+			if err != nil {
+				log.Printf("ERROR in input [%s]: %s", input.Name, err)
+			}
+			return
+		case <-ticker.C:
+			log.Printf("ERROR: input [%s] took longer to collect than "+
+				"collection interval (%s)",
+				input.Name, timeout)
+			continue
+		case <-shutdown:
+			return
+		}
+	}
+}
+
 // Test verifies that we can 'Gather' from all inputs with their configured
 // Config struct
 func (a *Agent) Test() error {
@@ -218,7 +213,8 @@ func (a *Agent) Test() error {
 
 	for _, input := range a.Config.Inputs {
 		acc := NewAccumulator(input.Config, metricC)
-		acc.SetDebug(true)
+		acc.SetTrace(true)
+		acc.setDefaultTags(a.Config.Tags)
 
 		fmt.Printf("* Plugin: %s, Collection 1\n", input.Name)
 		if input.Config.Interval != 0 {
@@ -345,7 +341,6 @@ func (a *Agent) Run(shutdown chan struct{}) error {
 		i := int64(a.Config.Agent.Interval.Duration)
 		time.Sleep(time.Duration(i - (time.Now().UnixNano() % i)))
 	}
-	ticker := time.NewTicker(a.Config.Agent.Interval.Duration)
 
 	wg.Add(1)
 	go func() {
@@ -356,32 +351,21 @@ func (a *Agent) Run(shutdown chan struct{}) error {
 		}
 	}()
 
+	wg.Add(len(a.Config.Inputs))
 	for _, input := range a.Config.Inputs {
-		// Special handling for inputs that have their own collection interval
-		// configured. Default intervals are handled below with gatherParallel
+		interval := a.Config.Agent.Interval.Duration
+		// overwrite global interval if this plugin has it's own.
 		if input.Config.Interval != 0 {
-			wg.Add(1)
-			go func(input *internal_models.RunningInput) {
-				defer wg.Done()
-				if err := a.gatherSeparate(shutdown, input, metricC); err != nil {
-					log.Printf(err.Error())
-				}
-			}(input)
+			interval = input.Config.Interval
 		}
+		go func(in *internal_models.RunningInput, interv time.Duration) {
+			defer wg.Done()
+			if err := a.gatherer(shutdown, in, interv, metricC); err != nil {
+				log.Printf(err.Error())
+			}
+		}(input, interval)
 	}
 
-	defer wg.Wait()
-
-	for {
-		if err := a.gatherParallel(metricC); err != nil {
-			log.Printf(err.Error())
-		}
-
-		select {
-		case <-shutdown:
-			return nil
-		case <-ticker.C:
-			continue
-		}
-	}
+	wg.Wait()
+	return nil
 }

agent/agent_test.go

@@ -1,7 +1,6 @@
 package agent
 
 import (
-	"github.com/stretchr/testify/assert"
 	"testing"
 	"time"
 
@@ -11,8 +10,18 @@ import (
 	_ "github.com/influxdata/telegraf/plugins/inputs/all"
 	// needing to load the outputs
 	_ "github.com/influxdata/telegraf/plugins/outputs/all"
+
+	"github.com/stretchr/testify/assert"
 )
 
+func TestAgent_OmitHostname(t *testing.T) {
+	c := config.NewConfig()
+	c.Agent.OmitHostname = true
+	_, err := NewAgent(c)
+	assert.NoError(t, err)
+	assert.NotContains(t, c.Tags, "host")
+}
+
 func TestAgent_LoadPlugin(t *testing.T) {
 	c := config.NewConfig()
 	c.InputFilters = []string{"mysql"}

circle.yml

@@ -4,9 +4,9 @@ machine:
   post:
     - sudo service zookeeper stop
     - go version
-    - go version | grep 1.5.3 || sudo rm -rf /usr/local/go
-    - wget https://storage.googleapis.com/golang/go1.5.3.linux-amd64.tar.gz
-    - sudo tar -C /usr/local -xzf go1.5.3.linux-amd64.tar.gz
+    - go version | grep 1.6.2 || sudo rm -rf /usr/local/go
+    - wget https://storage.googleapis.com/golang/go1.6.2.linux-amd64.tar.gz
+    - sudo tar -C /usr/local -xzf go1.6.2.linux-amd64.tar.gz
     - go version
 
 dependencies:

cmd/telegraf/telegraf.go

@@ -11,8 +11,9 @@ import (
 
 	"github.com/influxdata/telegraf/agent"
 	"github.com/influxdata/telegraf/internal/config"
-
+	"github.com/influxdata/telegraf/plugins/inputs"
 	_ "github.com/influxdata/telegraf/plugins/inputs/all"
+	"github.com/influxdata/telegraf/plugins/outputs"
 	_ "github.com/influxdata/telegraf/plugins/outputs/all"
 )
 
@@ -30,11 +31,14 @@ var fSampleConfig = flag.Bool("sample-config", false,
 var fPidfile = flag.String("pidfile", "", "file to write our pid to")
 var fInputFilters = flag.String("input-filter", "",
 	"filter the inputs to enable, separator is :")
+var fInputList = flag.Bool("input-list", false,
+	"print available input plugins.")
 var fOutputFilters = flag.String("output-filter", "",
 	"filter the outputs to enable, separator is :")
+var fOutputList = flag.Bool("output-list", false,
+	"print available output plugins.")
 var fUsage = flag.String("usage", "",
 	"print usage for a plugin, ie, 'telegraf -usage mysql'")
-
 var fInputFiltersLegacy = flag.String("filter", "",
 	"filter the inputs to enable, separator is :")
 var fOutputFiltersLegacy = flag.String("outputfilter", "",
@@ -42,9 +46,13 @@ var fOutputFiltersLegacy = flag.String("outputfilter", "",
 var fConfigDirectoryLegacy = flag.String("configdirectory", "",
 	"directory containing additional *.conf files")
 
-// Telegraf version
-//	-ldflags "-X main.Version=`git describe --always --tags`"
-var Version string
+// Telegraf version, populated linker.
+//   ie, -ldflags "-X main.version=`git describe --always --tags`"
+var (
+	version string
+	commit  string
+	branch  string
+)
 
 const usage = `Telegraf, The plugin-driven server agent for collecting and reporting metrics.
 
@@ -59,12 +67,21 @@ The flags are:
   -sample-config     print out full sample configuration to stdout
   -config-directory  directory containing additional *.conf files
   -input-filter      filter the input plugins to enable, separator is :
+  -input-list        print all the plugins inputs
   -output-filter     filter the output plugins to enable, separator is :
+  -output-list       print all the available outputs
   -usage             print usage for a plugin, ie, 'telegraf -usage mysql'
   -debug             print metrics as they're generated to stdout
   -quiet             run in quiet mode
   -version           print the version to stdout
 
+In addition to the -config flag, telegraf will also load the config file from
+an environment variable or default location. Precedence is:
+  1. -config flag
+  2. $TELEGRAF_CONFIG_PATH environment variable
+  3. $HOME/.telegraf/telegraf.conf
+  4. /etc/telegraf/telegraf.conf
+
 Examples:
 
   # generate a telegraf config file:
@@ -90,13 +107,12 @@ func main() {
 		reload <- false
 		flag.Usage = func() { usageExit(0) }
 		flag.Parse()
-
-		if flag.NFlag() == 0 {
-			usageExit(0)
-		}
+		args := flag.Args()
 
 		var inputFilters []string
 		if *fInputFiltersLegacy != "" {
+			fmt.Printf("WARNING '--filter' flag is deprecated, please use" +
+				" '--input-filter'")
 			inputFilter := strings.TrimSpace(*fInputFiltersLegacy)
 			inputFilters = strings.Split(":"+inputFilter+":", ":")
 		}
@@ -107,6 +123,8 @@ func main() {
 
 		var outputFilters []string
 		if *fOutputFiltersLegacy != "" {
+			fmt.Printf("WARNING '--outputfilter' flag is deprecated, please use" +
+				" '--output-filter'")
 			outputFilter := strings.TrimSpace(*fOutputFiltersLegacy)
 			outputFilters = strings.Split(":"+outputFilter+":", ":")
 		}
@@ -115,8 +133,36 @@ func main() {
 			outputFilters = strings.Split(":"+outputFilter+":", ":")
 		}
 
+		if len(args) > 0 {
+			switch args[0] {
+			case "version":
+				v := fmt.Sprintf("Telegraf - version %s", version)
+				fmt.Println(v)
+				return
+			case "config":
+				config.PrintSampleConfig(inputFilters, outputFilters)
+				return
+			}
+		}
+
+		if *fOutputList {
+			fmt.Println("Available Output Plugins:")
+			for k, _ := range outputs.Outputs {
+				fmt.Printf("  %s\n", k)
+			}
+			return
+		}
+
+		if *fInputList {
+			fmt.Println("Available Input Plugins:")
+			for k, _ := range inputs.Inputs {
+				fmt.Printf("  %s\n", k)
+			}
+			return
+		}
+
 		if *fVersion {
-			v := fmt.Sprintf("Telegraf - Version %s", Version)
+			v := fmt.Sprintf("Telegraf - version %s", version)
 			fmt.Println(v)
 			return
 		}
@@ -135,25 +181,19 @@ func main() {
 			return
 		}
 
-		var (
-			c   *config.Config
-			err error
-		)
-
-		if *fConfig != "" {
-			c = config.NewConfig()
-			c.OutputFilters = outputFilters
-			c.InputFilters = inputFilters
-			err = c.LoadConfig(*fConfig)
-			if err != nil {
-				log.Fatal(err)
-			}
-		} else {
-			fmt.Println("You must specify a config file. See telegraf --help")
+		// If no other options are specified, load the config file and run.
+		c := config.NewConfig()
+		c.OutputFilters = outputFilters
+		c.InputFilters = inputFilters
+		err := c.LoadConfig(*fConfig)
+		if err != nil {
+			fmt.Println(err)
 			os.Exit(1)
 		}
 
 		if *fConfigDirectoryLegacy != "" {
+			fmt.Printf("WARNING '--configdirectory' flag is deprecated, please use" +
+				" '--config-directory'")
 			err = c.LoadDirectory(*fConfigDirectoryLegacy)
 			if err != nil {
 				log.Fatal(err)
@@ -215,7 +255,7 @@ func main() {
 			}
 		}()
 
-		log.Printf("Starting Telegraf (version %s)\n", Version)
+		log.Printf("Starting Telegraf (version %s)\n", version)
 		log.Printf("Loaded outputs: %s", strings.Join(c.OutputNames(), " "))
 		log.Printf("Loaded inputs: %s", strings.Join(c.InputNames(), " "))
 		log.Printf("Tags enabled: %s", c.ListTags())

docs/CONFIGURATION.md

@@ -3,16 +3,31 @@
 ## Generating a Configuration File
 
 A default Telegraf config file can be generated using the -sample-config flag:
-`telegraf -sample-config > telegraf.conf`
+
+```
+telegraf -sample-config > telegraf.conf
+```
 
 To generate a file with specific inputs and outputs, you can use the
 -input-filter and -output-filter flags:
-`telegraf -sample-config -input-filter cpu:mem:net:swap -output-filter influxdb:kafka`
+
+```
+telegraf -sample-config -input-filter cpu:mem:net:swap -output-filter influxdb:kafka
+```
+
+You can see the latest config file with all available plugins here:
+[telegraf.conf](https://github.com/influxdata/telegraf/blob/master/etc/telegraf.conf)
+
+## Environment Variables
+
+Environment variables can be used anywhere in the config file, simply prepend
+them with $. For strings the variable must be within quotes (ie, "$STR_VAR"),
+for numbers and booleans they should be plain (ie, $INT_VAR, $BOOL_VAR)
 
 ## `[global_tags]` Configuration
 
-Global tags can be specific in the `[global_tags]` section of the config file in
-key="value" format. All metrics being gathered on this host will be tagged
+Global tags can be specified in the `[global_tags]` section of the config file
+in key="value" format. All metrics being gathered on this host will be tagged
 with the tags specified here.
 
 ## `[agent]` Configuration
@@ -23,8 +38,12 @@ config.
 * **interval**: Default data collection interval for all inputs
 * **round_interval**: Rounds collection interval to 'interval'
 ie, if interval="10s" then always collect on :00, :10, :20, etc.
+* **metric_batch_size**: Telegraf will send metrics to output in batch of at
+most metric_batch_size metrics.
 * **metric_buffer_limit**: Telegraf will cache metric_buffer_limit metrics
 for each output, and will flush this buffer on a successful write.
+This should be a multiple of metric_batch_size and could not be less
+than 2 times metric_batch_size.
 * **collection_jitter**: Collection jitter is used to jitter
 the collection by a random amount.
 Each plugin will sleep for a random time within jitter before collecting.
@@ -41,9 +60,35 @@ ie, a jitter of 5s and flush_interval 10s means flushes will happen every 10-15s
 * **quiet**: Run telegraf in quiet mode.
 * **hostname**: Override default hostname, if empty use os.Hostname().
 
-## `[inputs.xxx]` Configuration
+#### Measurement Filtering
 
-There are some configuration options that are configurable per input:
+Filters can be configured per input or output, see below for examples.
+
+* **namepass**: An array of strings that is used to filter metrics generated by the
+current input. Each string in the array is tested as a glob match against
+measurement names and if it matches, the field is emitted.
+* **namedrop**: The inverse of pass, if a measurement name matches, it is not emitted.
+* **fieldpass**: An array of strings that is used to filter metrics generated by the
+current input. Each string in the array is tested as a glob match against field names
+and if it matches, the field is emitted. fieldpass is not available for outputs.
+* **fielddrop**: The inverse of pass, if a field name matches, it is not emitted.
+fielddrop is not available for outputs.
+* **tagpass**: tag names and arrays of strings that are used to filter
+measurements by the current input. Each string in the array is tested as a glob
+match against the tag name, and if it matches the measurement is emitted.
+* **tagdrop**: The inverse of tagpass. If a tag matches, the measurement is not
+emitted. This is tested on measurements that have passed the tagpass test.
+* **tagexclude**: tagexclude can be used to exclude a tag from measurement(s).
+As opposed to tagdrop, which will drop an entire measurement based on it's
+tags, tagexclude simply strips the given tag keys from the measurement. This
+can be used on inputs & outputs, but it is _recommended_ to be used on inputs,
+as it is more efficient to filter out tags at the ingestion point.
+* **taginclude**: taginclude is the inverse of tagexclude. It will only include
+the tag keys in the final measurement.
+
+## Input Configuration
+
+Some configuration options are configurable per input:
 
 * **name_override**: Override the base name of the measurement.
 (Default is the name of the input).
@@ -54,24 +99,6 @@ There are some configuration options that are configurable per input:
 global interval, but if one particular input should be run less or more often,
 you can configure that here.
 
-#### Input Filters
-
-There are also filters that can be configured per input:
-
-* **namepass**: An array of strings that is used to filter metrics generated by the
-current input. Each string in the array is tested as a glob match against
-measurement names and if it matches, the field is emitted.
-* **namedrop**: The inverse of pass, if a measurement name matches, it is not emitted.
-* **fieldpass**: An array of strings that is used to filter metrics generated by the
-current input. Each string in the array is tested as a glob match against field names
-and if it matches, the field is emitted.
-* **fielddrop**: The inverse of pass, if a field name matches, it is not emitted.
-* **tagpass**: tag names and arrays of strings that are used to filter
-measurements by the current input. Each string in the array is tested as a glob
-match against the tag name, and if it matches the measurement is emitted.
-* **tagdrop**: The inverse of tagpass. If a tag matches, the measurement is not
-emitted. This is tested on measurements that have passed the tagpass test.
-
 #### Input Configuration Examples
 
 This is a full working config that will output CPU data to an InfluxDB instance
@@ -97,7 +124,7 @@ fields which begin with `time_`.
   percpu = true
   totalcpu = false
   # filter all fields beginning with 'time_'
-  drop = ["time_*"]
+  fielddrop = ["time_*"]
 ```
 
 #### Input Config: tagpass and tagdrop
@@ -106,7 +133,7 @@ fields which begin with `time_`.
 [[inputs.cpu]]
   percpu = true
   totalcpu = false
-  drop = ["cpu_time"]
+  fielddrop = ["cpu_time"]
   # Don't collect CPU data for cpu6 & cpu7
   [inputs.cpu.tagdrop]
     cpu = [ "cpu6", "cpu7" ]
@@ -141,12 +168,26 @@ fields which begin with `time_`.
 # Drop all metrics about containers for kubelet
 [[inputs.prometheus]]
   urls = ["http://kube-node-1:4194/metrics"]
-  namedrop = ["container_"]
+  namedrop = ["container_*"]
 
 # Only store rest client related metrics for kubelet
 [[inputs.prometheus]]
   urls = ["http://kube-node-1:4194/metrics"]
-  namepass = ["rest_client_"]
+  namepass = ["rest_client_*"]
+```
+
+#### Input Config: taginclude and tagexclude
+
+```toml
+# Only include the "cpu" tag in the measurements for the cpu plugin.
+[[inputs.cpu]]
+  percpu = true
+  totalcpu = true
+  taginclude = ["cpu"]
+
+# Exclude the "fstype" tag from the measurements for the disk plugin.
+[[inputs.disk]]
+  tagexclude = ["fstype"]
 ```
 
 #### Input config: prefix, suffix, and override
@@ -174,6 +215,9 @@ This will emit measurements with the name `foobar`
 This plugin will emit measurements with two additional tags: `tag1=foo` and
 `tag2=bar`
 
+NOTE: Order matters, the `[inputs.cpu.tags]` table must be at the _end_ of the
+plugin definition.
+
 ```toml
 [[inputs.cpu]]
   percpu = false
@@ -199,18 +243,15 @@ to avoid measurement collisions:
   percpu = true
   totalcpu = false
   name_override = "percpu_usage"
-  drop = ["cpu_time*"]
+  fielddrop = ["cpu_time*"]
 ```
 
-## `[outputs.xxx]` Configuration
+## Output Configuration
 
 Telegraf also supports specifying multiple output sinks to send data to,
 configuring each output sink is different, but examples can be
 found by running `telegraf -sample-config`.
 
-Outputs also support the same configurable options as inputs
-(namepass, namedrop, tagpass, tagdrop)
-
 ```toml
 [[outputs.influxdb]]
   urls = [ "http://localhost:8086" ]

docs/DATA_FORMATS_INPUT.md

@@ -1,5 +1,13 @@
 # Telegraf Input Data Formats
 
+Telegraf is able to parse the following input data formats into metrics:
+
+1. [InfluxDB Line Protocol](https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md#influx)
+1. [JSON](https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md#json)
+1. [Graphite](https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md#graphite)
+1. [Value](https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md#value), ie: 45 or "booyah"
+1. [Nagios](https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md#nagios) (exec input only)
+
 Telegraf metrics, like InfluxDB
 [points](https://docs.influxdata.com/influxdb/v0.10/write_protocols/line/),
 are a combination of four basic parts:
@@ -31,7 +39,7 @@ example, in the exec plugin:
   ## measurement name suffix (for separating different commands)
   name_suffix = "_mycollector"
 
-  ## Data format to consume. This can be "json", "influx" or "graphite"
+  ## Data format to consume.
   ## Each data format has it's own unique set of configuration options, read
   ## more about them here:
   ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md
@@ -43,7 +51,7 @@ example, in the exec plugin:
 Each data_format has an additional set of configuration options available, which
 I'll go over below.
 
-## Influx:
+# Influx:
 
 There are no additional configuration options for InfluxDB line-protocol. The
 metrics are parsed directly into Telegraf metrics.
@@ -58,23 +66,28 @@ metrics are parsed directly into Telegraf metrics.
   ## measurement name suffix (for separating different commands)
   name_suffix = "_mycollector"
 
-  ## Data format to consume. This can be "json", "influx" or "graphite"
+  ## Data format to consume.
   ## Each data format has it's own unique set of configuration options, read
   ## more about them here:
   ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md
   data_format = "influx"
 ```
 
-## JSON:
+# JSON:
 
-The JSON data format flattens JSON into metric _fields_. For example, this JSON:
+The JSON data format flattens JSON into metric _fields_.
+NOTE: Only numerical values are converted to fields, and they are converted
+into a float. strings are ignored unless specified as a tag_key (see below).
+
+So for example, this JSON:
 
 ```json
 {
     "a": 5,
     "b": {
         "c": 6
-    }
+    },
+    "ignored": "I'm a string"
 }
 ```
 
@@ -103,7 +116,7 @@ For example, if you had this configuration:
   ## measurement name suffix (for separating different commands)
   name_suffix = "_mycollector"
 
-  ## Data format to consume. This can be "json", "influx" or "graphite"
+  ## Data format to consume.
   ## Each data format has it's own unique set of configuration options, read
   ## more about them here:
   ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md
@@ -134,7 +147,43 @@ Your Telegraf metrics would get tagged with "my_tag_1"
 exec_mycollector,my_tag_1=foo a=5,b_c=6
 ```
 
-## Graphite:
+# Value:
+
+The "value" data format translates single values into Telegraf metrics. This
+is done by assigning a measurement name and setting a single field ("value")
+as the parsed metric.
+
+#### Value Configuration:
+
+You **must** tell Telegraf what type of metric to collect by using the
+`data_type` configuration option. Available options are:
+
+1. integer
+2. float or long
+3. string
+4. boolean
+
+**Note:** It is also recommended that you set `name_override` to a measurement
+name that makes sense for your metric, otherwise it will just be set to the
+name of the plugin.
+
+```toml
+[[inputs.exec]]
+  ## Commands array
+  commands = ["cat /proc/sys/kernel/random/entropy_avail"]
+
+  ## override the default metric name of "exec"
+  name_override = "entropy_available"
+
+  ## Data format to consume.
+  ## Each data format has it's own unique set of configuration options, read
+  ## more about them here:
+  ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md
+  data_format = "value"
+  data_type = "integer" # required
+```
+
+# Graphite:
 
 The Graphite data format translates graphite _dot_ buckets directly into
 telegraf measurement names, with a single value field, and without any tags. For
@@ -181,17 +230,32 @@ So the following template:
 
 ```toml
 templates = [
-    "measurement.measurement.field.region"
+    "measurement.measurement.field.field.region"
 ]
 ```
 
 would result in the following Graphite -> Telegraf transformation.
 
 ```
-cpu.usage.idle.us-west 100
-=> cpu_usage,region=us-west idle=100
+cpu.usage.idle.percent.us-west 100
+=> cpu_usage,region=us-west idle_percent=100
 ```
 
+The field key can also be derived from the second "half" of the input metric-name by specifying ```field*```:
+```toml
+templates = [
+    "measurement.measurement.region.field*"
+]
+```
+
+would result in the following Graphite -> Telegraf transformation.
+
+```
+cpu.usage.us-west.idle.percentage 100
+=> cpu_usage,region=us-west idle_percentage=100
+```
+(This cannot be used in conjunction with "measurement*"!)
+
 #### Filter Templates:
 
 Users can also filter the template(s) to use based on the name of the bucket,
@@ -247,7 +311,7 @@ There are many more options available,
   ## measurement name suffix (for separating different commands)
   name_suffix = "_mycollector"
 
-  ## Data format to consume. This can be "json", "influx" or "graphite" (line-protocol)
+  ## Data format to consume.
   ## Each data format has it's own unique set of configuration options, read
   ## more about them here:
   ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md
@@ -272,3 +336,27 @@ There are many more options available,
     "measurement*"
   ]
 ```
+
+# Nagios:
+
+There are no additional configuration options for Nagios line-protocol. The
+metrics are parsed directly into Telegraf metrics.
+
+Note: Nagios Input Data Formats is only supported in `exec` input plugin.
+
+#### Nagios Configuration:
+
+```toml
+[[inputs.exec]]
+  ## Commands array
+  commands = ["/usr/lib/nagios/plugins/check_load", "-w 5,6,7 -c 7,8,9"]
+
+  ## measurement name suffix (for separating different commands)
+  name_suffix = "_mycollector"
+
+  ## Data format to consume.
+  ## Each data format has it's own unique set of configuration options, read
+  ## more about them here:
+  ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md
+  data_format = "nagios"
+```

docs/DATA_FORMATS_OUTPUT.md

@@ -1,5 +1,11 @@
 # Telegraf Output Data Formats
 
+Telegraf is able to serialize metrics into the following output data formats:
+
+1. [InfluxDB Line Protocol](https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_OUTPUT.md#influx)
+1. [JSON](https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_OUTPUT.md#json)
+1. [Graphite](https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_OUTPUT.md#graphite)
+
 Telegraf metrics, like InfluxDB
 [points](https://docs.influxdata.com/influxdb/v0.10/write_protocols/line/),
 are a combination of four basic parts:
@@ -29,7 +35,7 @@ config option, for example, in the `file` output plugin:
   ## Files to write to, "stdout" is a specially handled file.
   files = ["stdout"]
 
-  ## Data format to output. This can be "influx" or "graphite"
+  ## Data format to output.
   ## Each data format has it's own unique set of configuration options, read
   ## more about them here:
   ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_OUTPUT.md
@@ -41,34 +47,46 @@ config option, for example, in the `file` output plugin:
 Each data_format has an additional set of configuration options available, which
 I'll go over below.
 
-## Influx:
+# Influx:
 
 There are no additional configuration options for InfluxDB line-protocol. The
 metrics are serialized directly into InfluxDB line-protocol.
 
-#### Influx Configuration:
+### Influx Configuration:
 
 ```toml
 [[outputs.file]]
   ## Files to write to, "stdout" is a specially handled file.
   files = ["stdout", "/tmp/metrics.out"]
 
-  ## Data format to output. This can be "influx" or "graphite"
+  ## Data format to output.
   ## Each data format has it's own unique set of configuration options, read
   ## more about them here:
   ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_OUTPUT.md
   data_format = "influx"
 ```
 
-## Graphite:
+# Graphite:
 
-The Graphite data format translates Telegraf metrics into _dot_ buckets.
-The format is:
+The Graphite data format translates Telegraf metrics into _dot_ buckets. A
+template can be specified for the output of Telegraf metrics into Graphite
+buckets. The default template is:
 
 ```
-[prefix].[host tag].[all tags (alphabetical)].[measurement name].[field name] value timestamp
+template = "host.tags.measurement.field"
 ```
 
+In the above template, we have four parts:
+
+1. _host_ is a tag key. This can be any tag key that is in the Telegraf
+metric(s). If the key doesn't exist, it will be ignored. If it does exist, the
+tag value will be filled in.
+1. _tags_ is a special keyword that outputs all remaining tag values, separated
+by dots and in alphabetical order (by tag key). These will be filled after all
+tag keys are filled.
+1. _measurement_ is a special keyword that outputs the measurement name.
+1. _field_ is a special keyword that outputs the field name.
+
 Which means the following influx metric -> graphite conversion would happen:
 
 ```
@@ -78,20 +96,55 @@ tars.cpu-total.us-east-1.cpu.usage_user 0.89 1455320690
 tars.cpu-total.us-east-1.cpu.usage_idle 98.09 1455320690
 ```
 
-`prefix` is a configuration option when using the graphite output data format.
-
-#### Graphite Configuration:
+### Graphite Configuration:
 
 ```toml
 [[outputs.file]]
   ## Files to write to, "stdout" is a specially handled file.
   files = ["stdout", "/tmp/metrics.out"]
 
-  ## Data format to output. This can be "influx" or "graphite"
+  ## Data format to output.
   ## Each data format has it's own unique set of configuration options, read
   ## more about them here:
   ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_OUTPUT.md
-  data_format = "influx"
+  data_format = "graphite"
 
+  # prefix each graphite bucket
   prefix = "telegraf"
+  # graphite template
+  template = "host.tags.measurement.field"
+```
+
+# JSON:
+
+The JSON data format serialized Telegraf metrics in json format. The format is:
+
+```json
+{
+   "fields":{
+      "field_1":30,
+      "field_2":4,
+      "field_N":59,
+      "n_images":660
+   },
+   "name":"docker",
+   "tags":{
+      "host":"raynor"
+   },
+   "timestamp":1458229140
+}
+```
+
+### JSON Configuration:
+
+```toml
+[[outputs.file]]
+  ## Files to write to, "stdout" is a specially handled file.
+  files = ["stdout", "/tmp/metrics.out"]
+
+  ## Data format to output.
+  ## Each data format has it's own unique set of configuration options, read
+  ## more about them here:
+  ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_OUTPUT.md
+  data_format = "json"
 ```

docs/LICENSE_OF_DEPENDENCIES.md

@@ -28,6 +28,5 @@
 - github.com/wvanbergen/kazoo-go [MIT LICENSE](https://github.com/wvanbergen/kazoo-go/blob/master/MIT-LICENSE)
 - gopkg.in/dancannon/gorethink.v1 [APACHE LICENSE](https://github.com/dancannon/gorethink/blob/v1.1.2/LICENSE)
 - gopkg.in/mgo.v2 [BSD LICENSE](https://github.com/go-mgo/mgo/blob/v2/LICENSE)
-- golang.org/x/crypto/* [BSD LICENSE](https://github.com/golang/crypto/blob/master/LICENSE)
-- internal Glob function [MIT LICENSE](https://github.com/ryanuber/go-glob/blob/master/LICENSE)
+- golang.org/x/crypto/ [BSD LICENSE](https://github.com/golang/crypto/blob/master/LICENSE)
 

docs/WINDOWS_SERVICE.md

@@ -0,0 +1,36 @@
+# Running Telegraf as a Windows Service
+
+If you have tried to install Go binaries as Windows Services with the **sc.exe**
+tool you may have seen that the service errors and stops running after a while.
+
+**NSSM** (the Non-Sucking Service Manager) is a tool that helps you in a 
+[number of scenarios](http://nssm.cc/scenarios) including running Go binaries
+that were not specifically designed to run only in Windows platforms.
+
+## NSSM Installation via Chocolatey
+
+You can install [Chocolatey](https://chocolatey.org/) and [NSSM](http://nssm.cc/) 
+with these commands
+
+```powershell
+iex ((new-object net.webclient).DownloadString('https://chocolatey.org/install.ps1'))
+choco install -y nssm
+```
+
+## Installing Telegraf as a Windows Service with NSSM
+
+You can download the latest Telegraf Windows binaries (still Experimental at 
+the moment) from [the Telegraf Github repo](https://github.com/influxdata/telegraf).
+
+Then you can create a C:\telegraf folder, unzip the binary there and modify the 
+**telegraf.conf** sample to allocate the metrics you want to send to **InfluxDB**.
+
+Once you have NSSM installed in your system, the process is quite straightforward.
+You only need to type this command in your Windows shell
+
+```powershell
+nssm install Telegraf c:\telegraf\telegraf.exe -config c:\telegraf\telegraf.config
+```
+
+And now your service will be installed in Windows and you will be able to start and
+stop it gracefully

etc/telegraf_windows.conf

@@ -0,0 +1,164 @@
+# Telegraf configuration
+
+# Telegraf is entirely plugin driven. All metrics are gathered from the
+# declared inputs, and sent to the declared outputs.
+
+# Plugins must be declared in here to be active.
+# To deactivate a plugin, comment out the name and any variables.
+
+# Use 'telegraf -config telegraf.conf -test' to see what metrics a config
+# file would generate.
+
+# Global tags can be specified here in key="value" format.
+[global_tags]
+  # dc = "us-east-1" # will tag all metrics with dc=us-east-1
+  # rack = "1a"
+
+# Configuration for telegraf agent
+[agent]
+  ## Default data collection interval for all inputs
+  interval = "10s"
+  ## Rounds collection interval to 'interval'
+  ## ie, if interval="10s" then always collect on :00, :10, :20, etc.
+  round_interval = true
+
+  ## Telegraf will cache metric_buffer_limit metrics for each output, and will
+  ## flush this buffer on a successful write.
+  metric_buffer_limit = 1000
+  ## Flush the buffer whenever full, regardless of flush_interval.
+  flush_buffer_when_full = true
+
+  ## Collection jitter is used to jitter the collection by a random amount.
+  ## Each plugin will sleep for a random time within jitter before collecting.
+  ## This can be used to avoid many plugins querying things like sysfs at the
+  ## same time, which can have a measurable effect on the system.
+  collection_jitter = "0s"
+
+  ## Default flushing interval for all outputs. You shouldn't set this below
+  ## interval. Maximum flush_interval will be flush_interval + flush_jitter
+  flush_interval = "10s"
+  ## Jitter the flush interval by a random amount. This is primarily to avoid
+  ## large write spikes for users running a large number of telegraf instances.
+  ## ie, a jitter of 5s and interval 10s means flushes will happen every 10-15s
+  flush_jitter = "0s"
+
+  ## Run telegraf in debug mode
+  debug = false
+  ## Run telegraf in quiet mode
+  quiet = false
+  ## Override default hostname, if empty use os.Hostname()
+  hostname = ""
+
+
+###############################################################################
+#                                  OUTPUTS                                    #
+###############################################################################
+
+# Configuration for influxdb server to send metrics to
+[[outputs.influxdb]]
+  # The full HTTP or UDP endpoint URL for your InfluxDB instance.
+  # Multiple urls can be specified but it is assumed that they are part of the same
+  # cluster, this means that only ONE of the urls will be written to each interval.
+  # urls = ["udp://localhost:8089"] # UDP endpoint example
+  urls = ["http://localhost:8086"] # required
+  # The target database for metrics (telegraf will create it if not exists)
+  database = "telegraf" # required
+  # Precision of writes, valid values are "ns", "us" (or "µs"), "ms", "s", "m", "h".
+  # note: using second precision greatly helps InfluxDB compression
+  precision = "s"
+
+  ## Write timeout (for the InfluxDB client), formatted as a string.
+  ## If not provided, will default to 5s. 0s means no timeout (not recommended).
+  timeout = "5s"
+  # username = "telegraf"
+  # password = "metricsmetricsmetricsmetrics"
+  # Set the user agent for HTTP POSTs (can be useful for log differentiation)
+  # user_agent = "telegraf"
+  # Set UDP payload size, defaults to InfluxDB UDP Client default (512 bytes)
+  # udp_payload = 512
+
+
+###############################################################################
+#                                  INPUTS                                     #
+###############################################################################
+
+# Windows Performance Counters plugin.
+# These are the recommended method of monitoring system metrics on windows,
+# as the regular system plugins (inputs.cpu, inputs.mem, etc.) rely on WMI,
+# which utilizes a lot of system resources.
+#
+# See more configuration examples at:
+#   https://github.com/influxdata/telegraf/tree/master/plugins/inputs/win_perf_counters
+
+[[inputs.win_perf_counters]]
+  [[inputs.win_perf_counters.object]]
+    # Processor usage, alternative to native, reports on a per core.
+    ObjectName = "Processor"
+    Instances = ["*"]
+    Counters = ["% Idle Time", "% Interrupt Time", "% Privileged Time", "% User Time", "% Processor Time"]
+    Measurement = "win_cpu"
+    #IncludeTotal=false #Set to true to include _Total instance when querying for all (*).
+
+  [[inputs.win_perf_counters.object]]
+    # Disk times and queues
+    ObjectName = "LogicalDisk"
+    Instances = ["*"]
+    Counters = ["% Idle Time", "% Disk Time","% Disk Read Time", "% Disk Write Time", "% User Time", "Current Disk Queue Length"]
+    Measurement = "win_disk"
+    #IncludeTotal=false #Set to true to include _Total instance when querying for all (*).
+
+  [[inputs.win_perf_counters.object]]
+    ObjectName = "System"
+    Counters = ["Context Switches/sec","System Calls/sec"]
+    Instances = ["------"]
+    Measurement = "win_system"
+    #IncludeTotal=false #Set to true to include _Total instance when querying for all (*).
+
+  [[inputs.win_perf_counters.object]]
+    # Example query where the Instance portion must be removed to get data back, such as from the Memory object.
+    ObjectName = "Memory"
+    Counters = ["Available Bytes","Cache Faults/sec","Demand Zero Faults/sec","Page Faults/sec","Pages/sec","Transition Faults/sec","Pool Nonpaged Bytes","Pool Paged Bytes"]
+    Instances = ["------"] # Use 6 x - to remove the Instance bit from the query.
+    Measurement = "win_mem"
+    #IncludeTotal=false #Set to true to include _Total instance when querying for all (*).
+
+
+# Windows system plugins using WMI (disabled by default, using
+# win_perf_counters over WMI is recommended)
+
+# Read metrics about cpu usage
+#[[inputs.cpu]]
+  ## Whether to report per-cpu stats or not
+  #percpu = true
+  ## Whether to report total system cpu stats or not
+  #totalcpu = true
+  ## Comment this line if you want the raw CPU time metrics
+  #fielddrop = ["time_*"]
+
+# Read metrics about disk usage by mount point
+#[[inputs.disk]]
+  ## By default, telegraf gather stats for all mountpoints.
+  ## Setting mountpoints will restrict the stats to the specified mountpoints.
+  ## mount_points=["/"]
+
+  ## Ignore some mountpoints by filesystem type. For example (dev)tmpfs (usually
+  ## present on /run, /var/run, /dev/shm or /dev).
+  #ignore_fs = ["tmpfs", "devtmpfs"]
+
+# Read metrics about disk IO by device
+#[[inputs.diskio]]
+  ## By default, telegraf will gather stats for all devices including
+  ## disk partitions.
+  ## Setting devices will restrict the stats to the specified devices.
+  ## devices = ["sda", "sdb"]
+  ## Uncomment the following line if you do not need disk serial numbers.
+  ## skip_serial_number = true
+
+# Read metrics about memory usage
+#[[inputs.mem]]
+  # no configuration
+
+# Read metrics about swap memory usage
+#[[inputs.swap]]
+  # no configuration
+

internal/buffer/buffer.go

@@ -0,0 +1,77 @@
+package buffer
+
+import (
+	"github.com/influxdata/telegraf"
+)
+
+// Buffer is an object for storing metrics in a circular buffer.
+type Buffer struct {
+	buf chan telegraf.Metric
+	// total dropped metrics
+	drops int
+	// total metrics added
+	total int
+}
+
+// NewBuffer returns a Buffer
+//   size is the maximum number of metrics that Buffer will cache. If Add is
+//   called when the buffer is full, then the oldest metric(s) will be dropped.
+func NewBuffer(size int) *Buffer {
+	return &Buffer{
+		buf: make(chan telegraf.Metric, size),
+	}
+}
+
+// IsEmpty returns true if Buffer is empty.
+func (b *Buffer) IsEmpty() bool {
+	return len(b.buf) == 0
+}
+
+// Len returns the current length of the buffer.
+func (b *Buffer) Len() int {
+	return len(b.buf)
+}
+
+// Drops returns the total number of dropped metrics that have occured in this
+// buffer since instantiation.
+func (b *Buffer) Drops() int {
+	return b.drops
+}
+
+// Total returns the total number of metrics that have been added to this buffer.
+func (b *Buffer) Total() int {
+	return b.total
+}
+
+// Add adds metrics to the buffer.
+func (b *Buffer) Add(metrics ...telegraf.Metric) {
+	for i, _ := range metrics {
+		b.total++
+		select {
+		case b.buf <- metrics[i]:
+		default:
+			b.drops++
+			<-b.buf
+			b.buf <- metrics[i]
+		}
+	}
+}
+
+// Batch returns a batch of metrics of size batchSize.
+// the batch will be of maximum length batchSize. It can be less than batchSize,
+// if the length of Buffer is less than batchSize.
+func (b *Buffer) Batch(batchSize int) []telegraf.Metric {
+	n := min(len(b.buf), batchSize)
+	out := make([]telegraf.Metric, n)
+	for i := 0; i < n; i++ {
+		out[i] = <-b.buf
+	}
+	return out
+}
+
+func min(a, b int) int {
+	if b < a {
+		return b
+	}
+	return a
+}

internal/buffer/buffer_test.go

@@ -0,0 +1,94 @@
+package buffer
+
+import (
+	"testing"
+
+	"github.com/influxdata/telegraf"
+	"github.com/influxdata/telegraf/testutil"
+
+	"github.com/stretchr/testify/assert"
+)
+
+var metricList = []telegraf.Metric{
+	testutil.TestMetric(2, "mymetric1"),
+	testutil.TestMetric(1, "mymetric2"),
+	testutil.TestMetric(11, "mymetric3"),
+	testutil.TestMetric(15, "mymetric4"),
+	testutil.TestMetric(8, "mymetric5"),
+}
+
+func BenchmarkAddMetrics(b *testing.B) {
+	buf := NewBuffer(10000)
+	m := testutil.TestMetric(1, "mymetric")
+	for n := 0; n < b.N; n++ {
+		buf.Add(m)
+	}
+}
+
+func TestNewBufferBasicFuncs(t *testing.T) {
+	b := NewBuffer(10)
+
+	assert.True(t, b.IsEmpty())
+	assert.Zero(t, b.Len())
+	assert.Zero(t, b.Drops())
+	assert.Zero(t, b.Total())
+
+	m := testutil.TestMetric(1, "mymetric")
+	b.Add(m)
+	assert.False(t, b.IsEmpty())
+	assert.Equal(t, b.Len(), 1)
+	assert.Equal(t, b.Drops(), 0)
+	assert.Equal(t, b.Total(), 1)
+
+	b.Add(metricList...)
+	assert.False(t, b.IsEmpty())
+	assert.Equal(t, b.Len(), 6)
+	assert.Equal(t, b.Drops(), 0)
+	assert.Equal(t, b.Total(), 6)
+}
+
+func TestDroppingMetrics(t *testing.T) {
+	b := NewBuffer(10)
+
+	// Add up to the size of the buffer
+	b.Add(metricList...)
+	b.Add(metricList...)
+	assert.False(t, b.IsEmpty())
+	assert.Equal(t, b.Len(), 10)
+	assert.Equal(t, b.Drops(), 0)
+	assert.Equal(t, b.Total(), 10)
+
+	// Add 5 more and verify they were dropped
+	b.Add(metricList...)
+	assert.False(t, b.IsEmpty())
+	assert.Equal(t, b.Len(), 10)
+	assert.Equal(t, b.Drops(), 5)
+	assert.Equal(t, b.Total(), 15)
+}
+
+func TestGettingBatches(t *testing.T) {
+	b := NewBuffer(20)
+
+	// Verify that the buffer returned is smaller than requested when there are
+	// not as many items as requested.
+	b.Add(metricList...)
+	batch := b.Batch(10)
+	assert.Len(t, batch, 5)
+
+	// Verify that the buffer is now empty
+	assert.True(t, b.IsEmpty())
+	assert.Zero(t, b.Len())
+	assert.Zero(t, b.Drops())
+	assert.Equal(t, b.Total(), 5)
+
+	// Verify that the buffer returned is not more than the size requested
+	b.Add(metricList...)
+	batch = b.Batch(3)
+	assert.Len(t, batch, 3)
+
+	// Verify that buffer is not empty
+	assert.False(t, b.IsEmpty())
+	assert.Equal(t, b.Len(), 2)
+	assert.Equal(t, b.Drops(), 0)
+	assert.Equal(t, b.Total(), 10)
+}

internal/config/config.go

@@ -1,11 +1,14 @@
 package config
 
 import (
+	"bytes"
 	"errors"
 	"fmt"
 	"io/ioutil"
 	"log"
+	"os"
 	"path/filepath"
+	"regexp"
 	"sort"
 	"strings"
 	"time"
@@ -19,7 +22,20 @@ import (
 	"github.com/influxdata/telegraf/plugins/serializers"
 
 	"github.com/influxdata/config"
-	"github.com/naoina/toml/ast"
+	"github.com/influxdata/toml"
+	"github.com/influxdata/toml/ast"
+)
+
+var (
+	// Default input plugins
+	inputDefaults = []string{"cpu", "mem", "swap", "system", "kernel",
+		"processes", "disk", "diskio"}
+
+	// Default output plugins
+	outputDefaults = []string{"influxdb"}
+
+	// envVarRe is a regex to find environment variables in the config file
+	envVarRe = regexp.MustCompile(`\$\w+`)
 )
 
 // Config specifies the URL/user/password for the database that telegraf
@@ -77,9 +93,15 @@ type AgentConfig struct {
 	// ie, a jitter of 5s and interval 10s means flushes will happen every 10-15s
 	FlushJitter internal.Duration
 
+	// MetricBatchSize is the maximum number of metrics that is wrote to an
+	// output plugin in one call.
+	MetricBatchSize int
+
 	// MetricBufferLimit is the max number of metrics that each output plugin
 	// will cache. The buffer is cleared when a successful write occurs. When
-	// full, the oldest metrics will be overwritten.
+	// full, the oldest metrics will be overwritten. This number should be a
+	// multiple of MetricBatchSize. Due to current implementation, this could
+	// not be less than 2 times MetricBatchSize.
 	MetricBufferLimit int
 
 	// FlushBufferWhenFull tells Telegraf to flush the metric buffer whenever
@@ -97,8 +119,9 @@ type AgentConfig struct {
 	Debug bool
 
 	// Quiet is the option for running in quiet mode
-	Quiet    bool
-	Hostname string
+	Quiet        bool
+	Hostname     string
+	OmitHostname bool
 }
 
 // Inputs returns a list of strings of the configured inputs.
@@ -134,20 +157,28 @@ func (c *Config) ListTags() string {
 }
 
 var header = `# Telegraf Configuration
-
+#
 # Telegraf is entirely plugin driven. All metrics are gathered from the
 # declared inputs, and sent to the declared outputs.
-
+#
 # Plugins must be declared in here to be active.
 # To deactivate a plugin, comment out the name and any variables.
-
+#
 # Use 'telegraf -config telegraf.conf -test' to see what metrics a config
 # file would generate.
+#
+# Environment variables can be used anywhere in this config file, simply prepend
+# them with $. For strings the variable must be within quotes (ie, "$STR_VAR"),
+# for numbers and booleans they should be plain (ie, $INT_VAR, $BOOL_VAR)
+
 
 # Global tags can be specified here in key="value" format.
 [global_tags]
   # dc = "us-east-1" # will tag all metrics with dc=us-east-1
   # rack = "1a"
+  ## Environment variables can be used as tags, and throughout the config file
+  # user = "$USER"
+
 
 # Configuration for telegraf agent
 [agent]
@@ -157,11 +188,13 @@ var header = `# Telegraf Configuration
   ## ie, if interval="10s" then always collect on :00, :10, :20, etc.
   round_interval = true
 
-  ## Telegraf will cache metric_buffer_limit metrics for each output, and will
-  ## flush this buffer on a successful write.
+  ## Telegraf will send metrics to outputs in batches of at
+  ## most metric_batch_size metrics.
+  metric_batch_size = 1000
+  ## For failed writes, telegraf will cache metric_buffer_limit metrics for each
+  ## output, and will flush this buffer on a successful write. Oldest metrics
+  ## are dropped first when this buffer fills.
   metric_buffer_limit = 10000
-  ## Flush the buffer whenever full, regardless of flush_interval.
-  flush_buffer_when_full = true
 
   ## Collection jitter is used to jitter the collection by a random amount.
   ## Each plugin will sleep for a random time within jitter before collecting.
@@ -183,34 +216,111 @@ var header = `# Telegraf Configuration
   quiet = false
   ## Override default hostname, if empty use os.Hostname()
   hostname = ""
+  ## If set to true, do no set the "host" tag in the telegraf agent.
+  omit_hostname = false
 
 
-#
-# OUTPUTS:
-#
-
+###############################################################################
+#                            OUTPUT PLUGINS                                   #
+###############################################################################
 `
 
-var pluginHeader = `
-#
-# INPUTS:
-#
+var inputHeader = `
+
+###############################################################################
+#                            INPUT PLUGINS                                    #
+###############################################################################
 `
 
 var serviceInputHeader = `
-#
-# SERVICE INPUTS:
-#
+
+###############################################################################
+#                            SERVICE INPUT PLUGINS                            #
+###############################################################################
 `
 
 // PrintSampleConfig prints the sample config
-func PrintSampleConfig(pluginFilters []string, outputFilters []string) {
+func PrintSampleConfig(inputFilters []string, outputFilters []string) {
 	fmt.Printf(header)
 
+	if len(outputFilters) != 0 {
+		printFilteredOutputs(outputFilters, false)
+	} else {
+		printFilteredOutputs(outputDefaults, false)
+		// Print non-default outputs, commented
+		var pnames []string
+		for pname := range outputs.Outputs {
+			if !sliceContains(pname, outputDefaults) {
+				pnames = append(pnames, pname)
+			}
+		}
+		sort.Strings(pnames)
+		printFilteredOutputs(pnames, true)
+	}
+
+	fmt.Printf(inputHeader)
+	if len(inputFilters) != 0 {
+		printFilteredInputs(inputFilters, false)
+	} else {
+		printFilteredInputs(inputDefaults, false)
+		// Print non-default inputs, commented
+		var pnames []string
+		for pname := range inputs.Inputs {
+			if !sliceContains(pname, inputDefaults) {
+				pnames = append(pnames, pname)
+			}
+		}
+		sort.Strings(pnames)
+		printFilteredInputs(pnames, true)
+	}
+}
+
+func printFilteredInputs(inputFilters []string, commented bool) {
+	// Filter inputs
+	var pnames []string
+	for pname := range inputs.Inputs {
+		if sliceContains(pname, inputFilters) {
+			pnames = append(pnames, pname)
+		}
+	}
+	sort.Strings(pnames)
+
+	// cache service inputs to print them at the end
+	servInputs := make(map[string]telegraf.ServiceInput)
+	// for alphabetical looping:
+	servInputNames := []string{}
+
+	// Print Inputs
+	for _, pname := range pnames {
+		creator := inputs.Inputs[pname]
+		input := creator()
+
+		switch p := input.(type) {
+		case telegraf.ServiceInput:
+			servInputs[pname] = p
+			servInputNames = append(servInputNames, pname)
+			continue
+		}
+
+		printConfig(pname, input, "inputs", commented)
+	}
+
+	// Print Service Inputs
+	if len(servInputs) == 0 {
+		return
+	}
+	sort.Strings(servInputNames)
+	fmt.Printf(serviceInputHeader)
+	for _, name := range servInputNames {
+		printConfig(name, servInputs[name], "inputs", commented)
+	}
+}
+
+func printFilteredOutputs(outputFilters []string, commented bool) {
 	// Filter outputs
 	var onames []string
 	for oname := range outputs.Outputs {
-		if len(outputFilters) == 0 || sliceContains(oname, outputFilters) {
+		if sliceContains(oname, outputFilters) {
 			onames = append(onames, oname)
 		}
 	}
@@ -220,38 +330,7 @@ func PrintSampleConfig(pluginFilters []string, outputFilters []string) {
 	for _, oname := range onames {
 		creator := outputs.Outputs[oname]
 		output := creator()
-		printConfig(oname, output, "outputs")
-	}
-
-	// Filter inputs
-	var pnames []string
-	for pname := range inputs.Inputs {
-		if len(pluginFilters) == 0 || sliceContains(pname, pluginFilters) {
-			pnames = append(pnames, pname)
-		}
-	}
-	sort.Strings(pnames)
-
-	// Print Inputs
-	fmt.Printf(pluginHeader)
-	servInputs := make(map[string]telegraf.ServiceInput)
-	for _, pname := range pnames {
-		creator := inputs.Inputs[pname]
-		input := creator()
-
-		switch p := input.(type) {
-		case telegraf.ServiceInput:
-			servInputs[pname] = p
-			continue
-		}
-
-		printConfig(pname, input, "inputs")
-	}
-
-	// Print Service Inputs
-	fmt.Printf(serviceInputHeader)
-	for name, input := range servInputs {
-		printConfig(name, input, "inputs")
+		printConfig(oname, output, "outputs", commented)
 	}
 }
 
@@ -260,13 +339,26 @@ type printer interface {
 	SampleConfig() string
 }
 
-func printConfig(name string, p printer, op string) {
-	fmt.Printf("\n# %s\n[[%s.%s]]", p.Description(), op, name)
+func printConfig(name string, p printer, op string, commented bool) {
+	comment := ""
+	if commented {
+		comment = "# "
+	}
+	fmt.Printf("\n%s# %s\n%s[[%s.%s]]", comment, p.Description(), comment,
+		op, name)
+
 	config := p.SampleConfig()
 	if config == "" {
-		fmt.Printf("\n  # no configuration\n")
+		fmt.Printf("\n%s  # no configuration\n\n", comment)
 	} else {
-		fmt.Printf(config)
+		lines := strings.Split(config, "\n")
+		for i, line := range lines {
+			if i == 0 || i == len(lines)-1 {
+				fmt.Print("\n")
+				continue
+			}
+			fmt.Print(comment + line + "\n")
+		}
 	}
 }
 
@@ -282,7 +374,7 @@ func sliceContains(name string, list []string) bool {
 // PrintInputConfig prints the config usage of a single input.
 func PrintInputConfig(name string) error {
 	if creator, ok := inputs.Inputs[name]; ok {
-		printConfig(name, creator(), "inputs")
+		printConfig(name, creator(), "inputs", false)
 	} else {
 		return errors.New(fmt.Sprintf("Input %s not found", name))
 	}
@@ -292,7 +384,7 @@ func PrintInputConfig(name string) error {
 // PrintOutputConfig prints the config usage of a single output.
 func PrintOutputConfig(name string) error {
 	if creator, ok := outputs.Outputs[name]; ok {
-		printConfig(name, creator(), "outputs")
+		printConfig(name, creator(), "outputs", false)
 	} else {
 		return errors.New(fmt.Sprintf("Output %s not found", name))
 	}
@@ -320,46 +412,91 @@ func (c *Config) LoadDirectory(path string) error {
 	return nil
 }
 
-// LoadConfig loads the given config file and applies it to c
-func (c *Config) LoadConfig(path string) error {
-	tbl, err := config.ParseFile(path)
-	if err != nil {
-		return err
+// Try to find a default config file at these locations (in order):
+//   1. $TELEGRAF_CONFIG_PATH
+//   2. $HOME/.telegraf/telegraf.conf
+//   3. /etc/telegraf/telegraf.conf
+//
+func getDefaultConfigPath() (string, error) {
+	envfile := os.Getenv("TELEGRAF_CONFIG_PATH")
+	homefile := os.ExpandEnv("${HOME}/.telegraf/telegraf.conf")
+	etcfile := "/etc/telegraf/telegraf.conf"
+	for _, path := range []string{envfile, homefile, etcfile} {
+		if _, err := os.Stat(path); err == nil {
+			log.Printf("Using config file: %s", path)
+			return path, nil
+		}
 	}
 
+	// if we got here, we didn't find a file in a default location
+	return "", fmt.Errorf("No config file specified, and could not find one"+
+		" in $TELEGRAF_CONFIG_PATH, %s, or %s", homefile, etcfile)
+}
+
+// LoadConfig loads the given config file and applies it to c
+func (c *Config) LoadConfig(path string) error {
+	var err error
+	if path == "" {
+		if path, err = getDefaultConfigPath(); err != nil {
+			return err
+		}
+	}
+	tbl, err := parseFile(path)
+	if err != nil {
+		return fmt.Errorf("Error parsing %s, %s", path, err)
+	}
+
+	// Parse tags tables first:
+	for _, tableName := range []string{"tags", "global_tags"} {
+		if val, ok := tbl.Fields[tableName]; ok {
+			subTable, ok := val.(*ast.Table)
+			if !ok {
+				return fmt.Errorf("%s: invalid configuration", path)
+			}
+			if err = config.UnmarshalTable(subTable, c.Tags); err != nil {
+				log.Printf("Could not parse [global_tags] config\n")
+				return fmt.Errorf("Error parsing %s, %s", path, err)
+			}
+		}
+	}
+
+	// Parse agent table:
+	if val, ok := tbl.Fields["agent"]; ok {
+		subTable, ok := val.(*ast.Table)
+		if !ok {
+			return fmt.Errorf("%s: invalid configuration", path)
+		}
+		if err = config.UnmarshalTable(subTable, c.Agent); err != nil {
+			log.Printf("Could not parse [agent] config\n")
+			return fmt.Errorf("Error parsing %s, %s", path, err)
+		}
+	}
+
+	// Parse all the rest of the plugins:
 	for name, val := range tbl.Fields {
 		subTable, ok := val.(*ast.Table)
 		if !ok {
-			return errors.New("invalid configuration")
+			return fmt.Errorf("%s: invalid configuration", path)
 		}
 
 		switch name {
-		case "agent":
-			if err = config.UnmarshalTable(subTable, c.Agent); err != nil {
-				log.Printf("Could not parse [agent] config\n")
-				return err
-			}
-		case "global_tags", "tags":
-			if err = config.UnmarshalTable(subTable, c.Tags); err != nil {
-				log.Printf("Could not parse [global_tags] config\n")
-				return err
-			}
+		case "agent", "global_tags", "tags":
 		case "outputs":
 			for pluginName, pluginVal := range subTable.Fields {
 				switch pluginSubTable := pluginVal.(type) {
 				case *ast.Table:
 					if err = c.addOutput(pluginName, pluginSubTable); err != nil {
-						return err
+						return fmt.Errorf("Error parsing %s, %s", path, err)
 					}
 				case []*ast.Table:
 					for _, t := range pluginSubTable {
 						if err = c.addOutput(pluginName, t); err != nil {
-							return err
+							return fmt.Errorf("Error parsing %s, %s", path, err)
 						}
 					}
 				default:
-					return fmt.Errorf("Unsupported config format: %s",
-						pluginName)
+					return fmt.Errorf("Unsupported config format: %s, file %s",
+						pluginName, path)
 				}
 			}
 		case "inputs", "plugins":
@@ -367,30 +504,50 @@ func (c *Config) LoadConfig(path string) error {
 				switch pluginSubTable := pluginVal.(type) {
 				case *ast.Table:
 					if err = c.addInput(pluginName, pluginSubTable); err != nil {
-						return err
+						return fmt.Errorf("Error parsing %s, %s", path, err)
 					}
 				case []*ast.Table:
 					for _, t := range pluginSubTable {
 						if err = c.addInput(pluginName, t); err != nil {
-							return err
+							return fmt.Errorf("Error parsing %s, %s", path, err)
 						}
 					}
 				default:
-					return fmt.Errorf("Unsupported config format: %s",
-						pluginName)
+					return fmt.Errorf("Unsupported config format: %s, file %s",
+						pluginName, path)
 				}
 			}
 		// Assume it's an input input for legacy config file support if no other
 		// identifiers are present
 		default:
 			if err = c.addInput(name, subTable); err != nil {
-				return err
+				return fmt.Errorf("Error parsing %s, %s", path, err)
 			}
 		}
 	}
 	return nil
 }
 
+// parseFile loads a TOML configuration from a provided path and
+// returns the AST produced from the TOML parser. When loading the file, it
+// will find environment variables and replace them.
+func parseFile(fpath string) (*ast.Table, error) {
+	contents, err := ioutil.ReadFile(fpath)
+	if err != nil {
+		return nil, err
+	}
+
+	env_vars := envVarRe.FindAll(contents, -1)
+	for _, env_var := range env_vars {
+		env_val := os.Getenv(strings.TrimPrefix(string(env_var), "$"))
+		if env_val != "" {
+			contents = bytes.Replace(contents, env_var, []byte(env_val), 1)
+		}
+	}
+
+	return toml.Parse(contents)
+}
+
 func (c *Config) addOutput(name string, table *ast.Table) error {
 	if len(c.OutputFilters) > 0 && !sliceContains(name, c.OutputFilters) {
 		return nil
@@ -421,11 +578,8 @@ func (c *Config) addOutput(name string, table *ast.Table) error {
 		return err
 	}
 
-	ro := internal_models.NewRunningOutput(name, output, outputConfig)
-	if c.Agent.MetricBufferLimit > 0 {
-		ro.MetricBufferLimit = c.Agent.MetricBufferLimit
-	}
-	ro.FlushBufferWhenFull = c.Agent.FlushBufferWhenFull
+	ro := internal_models.NewRunningOutput(name, output, outputConfig,
+		c.Agent.MetricBatchSize, c.Agent.MetricBufferLimit)
 	c.Outputs = append(c.Outputs, ro)
 	return nil
 }
@@ -476,9 +630,9 @@ func (c *Config) addInput(name string, table *ast.Table) error {
 
 // buildFilter builds a Filter
 // (tagpass/tagdrop/namepass/namedrop/fieldpass/fielddrop) to
-// be inserted into the internal_models.OutputConfig/internal_models.InputConfig to be used for prefix
-// filtering on tags and measurements
-func buildFilter(tbl *ast.Table) internal_models.Filter {
+// be inserted into the internal_models.OutputConfig/internal_models.InputConfig
+// to be used for glob filtering on tags and measurements
+func buildFilter(tbl *ast.Table) (internal_models.Filter, error) {
 	f := internal_models.Filter{}
 
 	if node, ok := tbl.Fields["namepass"]; ok {
@@ -577,6 +731,33 @@ func buildFilter(tbl *ast.Table) internal_models.Filter {
 		}
 	}
 
+	if node, ok := tbl.Fields["tagexclude"]; ok {
+		if kv, ok := node.(*ast.KeyValue); ok {
+			if ary, ok := kv.Value.(*ast.Array); ok {
+				for _, elem := range ary.Value {
+					if str, ok := elem.(*ast.String); ok {
+						f.TagExclude = append(f.TagExclude, str.Value)
+					}
+				}
+			}
+		}
+	}
+
+	if node, ok := tbl.Fields["taginclude"]; ok {
+		if kv, ok := node.(*ast.KeyValue); ok {
+			if ary, ok := kv.Value.(*ast.Array); ok {
+				for _, elem := range ary.Value {
+					if str, ok := elem.(*ast.String); ok {
+						f.TagInclude = append(f.TagInclude, str.Value)
+					}
+				}
+			}
+		}
+	}
+	if err := f.CompileFilter(); err != nil {
+		return f, err
+	}
+
 	delete(tbl.Fields, "namedrop")
 	delete(tbl.Fields, "namepass")
 	delete(tbl.Fields, "fielddrop")
@@ -585,7 +766,9 @@ func buildFilter(tbl *ast.Table) internal_models.Filter {
 	delete(tbl.Fields, "pass")
 	delete(tbl.Fields, "tagdrop")
 	delete(tbl.Fields, "tagpass")
-	return f
+	delete(tbl.Fields, "tagexclude")
+	delete(tbl.Fields, "taginclude")
+	return f, nil
 }
 
 // buildInput parses input specific items from the ast.Table,
@@ -644,7 +827,11 @@ func buildInput(name string, tbl *ast.Table) (*internal_models.InputConfig, erro
 	delete(tbl.Fields, "name_override")
 	delete(tbl.Fields, "interval")
 	delete(tbl.Fields, "tags")
-	cp.Filter = buildFilter(tbl)
+	var err error
+	cp.Filter, err = buildFilter(tbl)
+	if err != nil {
+		return cp, err
+	}
 	return cp, nil
 }
 
@@ -701,12 +888,21 @@ func buildParser(name string, tbl *ast.Table) (parsers.Parser, error) {
 		}
 	}
 
+	if node, ok := tbl.Fields["data_type"]; ok {
+		if kv, ok := node.(*ast.KeyValue); ok {
+			if str, ok := kv.Value.(*ast.String); ok {
+				c.DataType = str.Value
+			}
+		}
+	}
+
 	c.MetricName = name
 
 	delete(tbl.Fields, "data_format")
 	delete(tbl.Fields, "separator")
 	delete(tbl.Fields, "templates")
 	delete(tbl.Fields, "tag_keys")
+	delete(tbl.Fields, "data_type")
 
 	return parsers.NewParser(c)
 }
@@ -737,18 +933,32 @@ func buildSerializer(name string, tbl *ast.Table) (serializers.Serializer, error
 		}
 	}
 
+	if node, ok := tbl.Fields["template"]; ok {
+		if kv, ok := node.(*ast.KeyValue); ok {
+			if str, ok := kv.Value.(*ast.String); ok {
+				c.Template = str.Value
+			}
+		}
+	}
+
 	delete(tbl.Fields, "data_format")
 	delete(tbl.Fields, "prefix")
+	delete(tbl.Fields, "template")
 	return serializers.NewSerializer(c)
 }
 
-// buildOutput parses output specific items from the ast.Table, builds the filter and returns an
+// buildOutput parses output specific items from the ast.Table,
+// builds the filter and returns an
 // internal_models.OutputConfig to be inserted into internal_models.RunningInput
 // Note: error exists in the return for future calls that might require error
 func buildOutput(name string, tbl *ast.Table) (*internal_models.OutputConfig, error) {
+	filter, err := buildFilter(tbl)
+	if err != nil {
+		return nil, err
+	}
 	oc := &internal_models.OutputConfig{
 		Name:   name,
-		Filter: buildFilter(tbl),
+		Filter: filter,
 	}
 	// Outputs don't support FieldDrop/FieldPass, so set to NameDrop/NamePass
 	if len(oc.Filter.FieldDrop) > 0 {

internal/config/config_test.go

@@ -1,6 +1,7 @@
 package config
 
 import (
+	"os"
 	"testing"
 	"time"
 
@@ -10,9 +11,54 @@ import (
 	"github.com/influxdata/telegraf/plugins/inputs/memcached"
 	"github.com/influxdata/telegraf/plugins/inputs/procstat"
 	"github.com/influxdata/telegraf/plugins/parsers"
+
 	"github.com/stretchr/testify/assert"
 )
 
+func TestConfig_LoadSingleInputWithEnvVars(t *testing.T) {
+	c := NewConfig()
+	err := os.Setenv("MY_TEST_SERVER", "192.168.1.1")
+	assert.NoError(t, err)
+	err = os.Setenv("TEST_INTERVAL", "10s")
+	assert.NoError(t, err)
+	c.LoadConfig("./testdata/single_plugin_env_vars.toml")
+
+	memcached := inputs.Inputs["memcached"]().(*memcached.Memcached)
+	memcached.Servers = []string{"192.168.1.1"}
+
+	filter := internal_models.Filter{
+		NameDrop:  []string{"metricname2"},
+		NamePass:  []string{"metricname1"},
+		FieldDrop: []string{"other", "stuff"},
+		FieldPass: []string{"some", "strings"},
+		TagDrop: []internal_models.TagFilter{
+			internal_models.TagFilter{
+				Name:   "badtag",
+				Filter: []string{"othertag"},
+			},
+		},
+		TagPass: []internal_models.TagFilter{
+			internal_models.TagFilter{
+				Name:   "goodtag",
+				Filter: []string{"mytag"},
+			},
+		},
+		IsActive: true,
+	}
+	assert.NoError(t, filter.CompileFilter())
+	mConfig := &internal_models.InputConfig{
+		Name:     "memcached",
+		Filter:   filter,
+		Interval: 10 * time.Second,
+	}
+	mConfig.Tags = make(map[string]string)
+
+	assert.Equal(t, memcached, c.Inputs[0].Input,
+		"Testdata did not produce a correct memcached struct.")
+	assert.Equal(t, mConfig, c.Inputs[0].Config,
+		"Testdata did not produce correct memcached metadata.")
+}
+
 func TestConfig_LoadSingleInput(t *testing.T) {
 	c := NewConfig()
 	c.LoadConfig("./testdata/single_plugin.toml")
@@ -20,27 +66,29 @@ func TestConfig_LoadSingleInput(t *testing.T) {
 	memcached := inputs.Inputs["memcached"]().(*memcached.Memcached)
 	memcached.Servers = []string{"localhost"}
 
-	mConfig := &internal_models.InputConfig{
-		Name: "memcached",
-		Filter: internal_models.Filter{
-			NameDrop:  []string{"metricname2"},
-			NamePass:  []string{"metricname1"},
-			FieldDrop: []string{"other", "stuff"},
-			FieldPass: []string{"some", "strings"},
-			TagDrop: []internal_models.TagFilter{
-				internal_models.TagFilter{
-					Name:   "badtag",
-					Filter: []string{"othertag"},
-				},
+	filter := internal_models.Filter{
+		NameDrop:  []string{"metricname2"},
+		NamePass:  []string{"metricname1"},
+		FieldDrop: []string{"other", "stuff"},
+		FieldPass: []string{"some", "strings"},
+		TagDrop: []internal_models.TagFilter{
+			internal_models.TagFilter{
+				Name:   "badtag",
+				Filter: []string{"othertag"},
 			},
-			TagPass: []internal_models.TagFilter{
-				internal_models.TagFilter{
-					Name:   "goodtag",
-					Filter: []string{"mytag"},
-				},
-			},
-			IsActive: true,
 		},
+		TagPass: []internal_models.TagFilter{
+			internal_models.TagFilter{
+				Name:   "goodtag",
+				Filter: []string{"mytag"},
+			},
+		},
+		IsActive: true,
+	}
+	assert.NoError(t, filter.CompileFilter())
+	mConfig := &internal_models.InputConfig{
+		Name:     "memcached",
+		Filter:   filter,
 		Interval: 5 * time.Second,
 	}
 	mConfig.Tags = make(map[string]string)
@@ -65,27 +113,29 @@ func TestConfig_LoadDirectory(t *testing.T) {
 	memcached := inputs.Inputs["memcached"]().(*memcached.Memcached)
 	memcached.Servers = []string{"localhost"}
 
-	mConfig := &internal_models.InputConfig{
-		Name: "memcached",
-		Filter: internal_models.Filter{
-			NameDrop:  []string{"metricname2"},
-			NamePass:  []string{"metricname1"},
-			FieldDrop: []string{"other", "stuff"},
-			FieldPass: []string{"some", "strings"},
-			TagDrop: []internal_models.TagFilter{
-				internal_models.TagFilter{
-					Name:   "badtag",
-					Filter: []string{"othertag"},
-				},
+	filter := internal_models.Filter{
+		NameDrop:  []string{"metricname2"},
+		NamePass:  []string{"metricname1"},
+		FieldDrop: []string{"other", "stuff"},
+		FieldPass: []string{"some", "strings"},
+		TagDrop: []internal_models.TagFilter{
+			internal_models.TagFilter{
+				Name:   "badtag",
+				Filter: []string{"othertag"},
 			},
-			TagPass: []internal_models.TagFilter{
-				internal_models.TagFilter{
-					Name:   "goodtag",
-					Filter: []string{"mytag"},
-				},
-			},
-			IsActive: true,
 		},
+		TagPass: []internal_models.TagFilter{
+			internal_models.TagFilter{
+				Name:   "goodtag",
+				Filter: []string{"mytag"},
+			},
+		},
+		IsActive: true,
+	}
+	assert.NoError(t, filter.CompileFilter())
+	mConfig := &internal_models.InputConfig{
+		Name:     "memcached",
+		Filter:   filter,
 		Interval: 5 * time.Second,
 	}
 	mConfig.Tags = make(map[string]string)

internal/config/testdata/single_plugin_env_vars.toml

@@ -0,0 +1,11 @@
+[[inputs.memcached]]
+  servers = ["$MY_TEST_SERVER"]
+  namepass = ["metricname1"]
+  namedrop = ["metricname2"]
+  fieldpass = ["some", "strings"]
+  fielddrop = ["other", "stuff"]
+  interval = "$TEST_INTERVAL"
+  [inputs.memcached.tagpass]
+    goodtag = ["mytag"]
+  [inputs.memcached.tagdrop]
+    badtag = ["othertag"]

internal/globpath/globpath.go

@@ -0,0 +1,98 @@
+package globpath
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/gobwas/glob"
+)
+
+var sepStr = fmt.Sprintf("%v", string(os.PathSeparator))
+
+type GlobPath struct {
+	path    string
+	hasMeta bool
+	g       glob.Glob
+	root    string
+}
+
+func Compile(path string) (*GlobPath, error) {
+	out := GlobPath{
+		hasMeta: hasMeta(path),
+		path:    path,
+	}
+
+	// if there are no glob meta characters in the path, don't bother compiling
+	// a glob object or finding the root directory. (see short-circuit in Match)
+	if !out.hasMeta {
+		return &out, nil
+	}
+
+	var err error
+	if out.g, err = glob.Compile(path, os.PathSeparator); err != nil {
+		return nil, err
+	}
+	// Get the root directory for this filepath
+	out.root = findRootDir(path)
+	return &out, nil
+}
+
+func (g *GlobPath) Match() map[string]os.FileInfo {
+	if !g.hasMeta {
+		out := make(map[string]os.FileInfo)
+		info, err := os.Stat(g.path)
+		if !os.IsNotExist(err) {
+			out[g.path] = info
+		}
+		return out
+	}
+	return walkFilePath(g.root, g.g)
+}
+
+// walk the filepath from the given root and return a list of files that match
+// the given glob.
+func walkFilePath(root string, g glob.Glob) map[string]os.FileInfo {
+	matchedFiles := make(map[string]os.FileInfo)
+	walkfn := func(path string, info os.FileInfo, _ error) error {
+		if g.Match(path) {
+			matchedFiles[path] = info
+		}
+		return nil
+	}
+	filepath.Walk(root, walkfn)
+	return matchedFiles
+}
+
+// find the root dir of the given path (could include globs).
+// ie:
+//   /var/log/telegraf.conf -> /var/log
+//   /home/** ->               /home
+//   /home/*/** ->             /home
+//   /lib/share/*/*/**.txt ->  /lib/share
+func findRootDir(path string) string {
+	pathItems := strings.Split(path, sepStr)
+	out := sepStr
+	for i, item := range pathItems {
+		if i == len(pathItems)-1 {
+			break
+		}
+		if item == "" {
+			continue
+		}
+		if hasMeta(item) {
+			break
+		}
+		out += item + sepStr
+	}
+	if out != "/" {
+		out = strings.TrimSuffix(out, "/")
+	}
+	return out
+}
+
+// hasMeta reports whether path contains any magic glob characters.
+func hasMeta(path string) bool {
+	return strings.IndexAny(path, "*?[") >= 0
+}

internal/globpath/globpath_test.go

@@ -0,0 +1,62 @@
+package globpath
+
+import (
+	"runtime"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestCompileAndMatch(t *testing.T) {
+	dir := getTestdataDir()
+	// test super asterisk
+	g1, err := Compile(dir + "/**")
+	require.NoError(t, err)
+	// test single asterisk
+	g2, err := Compile(dir + "/*.log")
+	require.NoError(t, err)
+	// test no meta characters (file exists)
+	g3, err := Compile(dir + "/log1.log")
+	require.NoError(t, err)
+	// test file that doesn't exist
+	g4, err := Compile(dir + "/i_dont_exist.log")
+	require.NoError(t, err)
+	// test super asterisk that doesn't exist
+	g5, err := Compile(dir + "/dir_doesnt_exist/**")
+	require.NoError(t, err)
+
+	matches := g1.Match()
+	assert.Len(t, matches, 3)
+	matches = g2.Match()
+	assert.Len(t, matches, 2)
+	matches = g3.Match()
+	assert.Len(t, matches, 1)
+	matches = g4.Match()
+	assert.Len(t, matches, 0)
+	matches = g5.Match()
+	assert.Len(t, matches, 0)
+}
+
+func TestFindRootDir(t *testing.T) {
+	tests := []struct {
+		input  string
+		output string
+	}{
+		{"/var/log/telegraf.conf", "/var/log"},
+		{"/home/**", "/home"},
+		{"/home/*/**", "/home"},
+		{"/lib/share/*/*/**.txt", "/lib/share"},
+	}
+
+	for _, test := range tests {
+		actual := findRootDir(test.input)
+		assert.Equal(t, test.output, actual)
+	}
+}
+
+func getTestdataDir() string {
+	_, filename, _, _ := runtime.Caller(1)
+	return strings.Replace(filename, "globpath_test.go", "testdata", 1)
+}

internal/globpath/testdata/test.conf

@@ -0,0 +1,5 @@
+# this is a fake testing config file
+# for testing the filestat plugin
+
+option1 = "foo"
+option2 = "bar"

internal/internal.go

@@ -2,19 +2,30 @@ package internal
 
 import (
 	"bufio"
+	"bytes"
 	"crypto/rand"
 	"crypto/tls"
 	"crypto/x509"
 	"errors"
 	"fmt"
 	"io/ioutil"
+	"log"
 	"os"
+	"os/exec"
+	"strconv"
 	"strings"
 	"time"
+	"unicode"
 )
 
 const alphanum string = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
 
+var (
+	TimeoutErr = errors.New("Command timed out.")
+
+	NotImplementedError = errors.New("not implemented yet")
+)
+
 // Duration just wraps time.Duration
 type Duration struct {
 	Duration time.Duration
@@ -22,18 +33,29 @@ type Duration struct {
 
 // UnmarshalTOML parses the duration from the TOML config file
 func (d *Duration) UnmarshalTOML(b []byte) error {
-	dur, err := time.ParseDuration(string(b[1 : len(b)-1]))
-	if err != nil {
-		return err
+	var err error
+	// Parse string duration, ie, "1s"
+	d.Duration, err = time.ParseDuration(string(b[1 : len(b)-1]))
+	if err == nil {
+		return nil
 	}
 
-	d.Duration = dur
+	// First try parsing as integer seconds
+	sI, err := strconv.ParseInt(string(b), 10, 64)
+	if err == nil {
+		d.Duration = time.Second * time.Duration(sI)
+		return nil
+	}
+	// Second try parsing as float seconds
+	sF, err := strconv.ParseFloat(string(b), 64)
+	if err == nil {
+		d.Duration = time.Second * time.Duration(sF)
+		return nil
+	}
 
 	return nil
 }
 
-var NotImplementedError = errors.New("not implemented yet")
-
 // ReadLines reads contents from a file and splits them by new lines.
 // A convenience wrapper to ReadLinesOffsetN(filename, 0, -1).
 func ReadLines(filename string) ([]string, error) {
@@ -86,15 +108,15 @@ func GetTLSConfig(
 	SSLCert, SSLKey, SSLCA string,
 	InsecureSkipVerify bool,
 ) (*tls.Config, error) {
-	t := &tls.Config{}
-	if SSLCert != "" && SSLKey != "" && SSLCA != "" {
-		cert, err := tls.LoadX509KeyPair(SSLCert, SSLKey)
-		if err != nil {
-			return nil, errors.New(fmt.Sprintf(
-				"Could not load TLS client key/certificate: %s",
-				err))
-		}
+	if SSLCert == "" && SSLKey == "" && SSLCA == "" && !InsecureSkipVerify {
+		return nil, nil
+	}
 
+	t := &tls.Config{
+		InsecureSkipVerify: InsecureSkipVerify,
+	}
+
+	if SSLCA != "" {
 		caCert, err := ioutil.ReadFile(SSLCA)
 		if err != nil {
 			return nil, errors.New(fmt.Sprintf("Could not load TLS CA: %s",
@@ -103,75 +125,83 @@ func GetTLSConfig(
 
 		caCertPool := x509.NewCertPool()
 		caCertPool.AppendCertsFromPEM(caCert)
-
-		t = &tls.Config{
-			Certificates:       []tls.Certificate{cert},
-			RootCAs:            caCertPool,
-			InsecureSkipVerify: InsecureSkipVerify,
-		}
-	} else {
-		if InsecureSkipVerify {
-			t.InsecureSkipVerify = true
-		} else {
-			return nil, nil
-		}
+		t.RootCAs = caCertPool
 	}
+
+	if SSLCert != "" && SSLKey != "" {
+		cert, err := tls.LoadX509KeyPair(SSLCert, SSLKey)
+		if err != nil {
+			return nil, errors.New(fmt.Sprintf(
+				"Could not load TLS client key/certificate: %s",
+				err))
+		}
+
+		t.Certificates = []tls.Certificate{cert}
+		t.BuildNameToCertificate()
+	}
+
 	// will be nil by default if nothing is provided
 	return t, nil
 }
 
-// Glob will test a string pattern, potentially containing globs, against a
-// subject string. The result is a simple true/false, determining whether or
-// not the glob pattern matched the subject text.
-//
-// Adapted from https://github.com/ryanuber/go-glob/blob/master/glob.go
-// thanks Ryan Uber!
-func Glob(pattern, measurement string) bool {
-	// Empty pattern can only match empty subject
-	if pattern == "" {
-		return measurement == pattern
-	}
+// SnakeCase converts the given string to snake case following the Golang format:
+// acronyms are converted to lower-case and preceded by an underscore.
+func SnakeCase(in string) string {
+	runes := []rune(in)
+	length := len(runes)
 
-	// If the pattern _is_ a glob, it matches everything
-	if pattern == "*" {
-		return true
-	}
-
-	parts := strings.Split(pattern, "*")
-
-	if len(parts) == 1 {
-		// No globs in pattern, so test for match
-		return pattern == measurement
-	}
-
-	leadingGlob := strings.HasPrefix(pattern, "*")
-	trailingGlob := strings.HasSuffix(pattern, "*")
-	end := len(parts) - 1
-
-	for i, part := range parts {
-		switch i {
-		case 0:
-			if leadingGlob {
-				continue
-			}
-			if !strings.HasPrefix(measurement, part) {
-				return false
-			}
-		case end:
-			if len(measurement) > 0 {
-				return trailingGlob || strings.HasSuffix(measurement, part)
-			}
-		default:
-			if !strings.Contains(measurement, part) {
-				return false
-			}
+	var out []rune
+	for i := 0; i < length; i++ {
+		if i > 0 && unicode.IsUpper(runes[i]) && ((i+1 < length && unicode.IsLower(runes[i+1])) || unicode.IsLower(runes[i-1])) {
+			out = append(out, '_')
 		}
-
-		// Trim evaluated text from measurement as we loop over the pattern.
-		idx := strings.Index(measurement, part) + len(part)
-		measurement = measurement[idx:]
+		out = append(out, unicode.ToLower(runes[i]))
 	}
 
-	// All parts of the pattern matched
-	return true
+	return string(out)
+}
+
+// CombinedOutputTimeout runs the given command with the given timeout and
+// returns the combined output of stdout and stderr.
+// If the command times out, it attempts to kill the process.
+func CombinedOutputTimeout(c *exec.Cmd, timeout time.Duration) ([]byte, error) {
+	var b bytes.Buffer
+	c.Stdout = &b
+	c.Stderr = &b
+	if err := c.Start(); err != nil {
+		return nil, err
+	}
+	err := WaitTimeout(c, timeout)
+	return b.Bytes(), err
+}
+
+// RunTimeout runs the given command with the given timeout.
+// If the command times out, it attempts to kill the process.
+func RunTimeout(c *exec.Cmd, timeout time.Duration) error {
+	if err := c.Start(); err != nil {
+		return err
+	}
+	return WaitTimeout(c, timeout)
+}
+
+// WaitTimeout waits for the given command to finish with a timeout.
+// It assumes the command has already been started.
+// If the command times out, it attempts to kill the process.
+func WaitTimeout(c *exec.Cmd, timeout time.Duration) error {
+	timer := time.NewTimer(timeout)
+	done := make(chan error)
+	go func() { done <- c.Wait() }()
+	select {
+	case err := <-done:
+		timer.Stop()
+		return err
+	case <-timer.C:
+		if err := c.Process.Kill(); err != nil {
+			log.Printf("FATAL error killing process: %s", err)
+			return err
+		}
+		// wait for the command to return after killing it
+		<-done
+		return TimeoutErr
+	}
 }

internal/internal_test.go

@@ -1,44 +1,108 @@
 package internal
 
-import "testing"
+import (
+	"os/exec"
+	"testing"
+	"time"
 
-func testGlobMatch(t *testing.T, pattern, subj string) {
-	if !Glob(pattern, subj) {
-		t.Errorf("%s should match %s", pattern, subj)
+	"github.com/stretchr/testify/assert"
+)
+
+type SnakeTest struct {
+	input  string
+	output string
+}
+
+var tests = []SnakeTest{
+	{"a", "a"},
+	{"snake", "snake"},
+	{"A", "a"},
+	{"ID", "id"},
+	{"MOTD", "motd"},
+	{"Snake", "snake"},
+	{"SnakeTest", "snake_test"},
+	{"APIResponse", "api_response"},
+	{"SnakeID", "snake_id"},
+	{"SnakeIDGoogle", "snake_id_google"},
+	{"LinuxMOTD", "linux_motd"},
+	{"OMGWTFBBQ", "omgwtfbbq"},
+	{"omg_wtf_bbq", "omg_wtf_bbq"},
+}
+
+func TestSnakeCase(t *testing.T) {
+	for _, test := range tests {
+		if SnakeCase(test.input) != test.output {
+			t.Errorf(`SnakeCase("%s"), wanted "%s", got \%s"`, test.input, test.output, SnakeCase(test.input))
+		}
 	}
 }
 
-func testGlobNoMatch(t *testing.T, pattern, subj string) {
-	if Glob(pattern, subj) {
-		t.Errorf("%s should not match %s", pattern, subj)
+var (
+	sleepbin, _ = exec.LookPath("sleep")
+	echobin, _  = exec.LookPath("echo")
+)
+
+func TestRunTimeout(t *testing.T) {
+	if sleepbin == "" {
+		t.Skip("'sleep' binary not available on OS, skipping.")
 	}
+	cmd := exec.Command(sleepbin, "10")
+	start := time.Now()
+	err := RunTimeout(cmd, time.Millisecond*20)
+	elapsed := time.Since(start)
+
+	assert.Equal(t, TimeoutErr, err)
+	// Verify that command gets killed in 20ms, with some breathing room
+	assert.True(t, elapsed < time.Millisecond*75)
 }
 
-func TestEmptyPattern(t *testing.T) {
-	testGlobMatch(t, "", "")
-	testGlobNoMatch(t, "", "test")
-}
-
-func TestPatternWithoutGlobs(t *testing.T) {
-	testGlobMatch(t, "test", "test")
-}
-
-func TestGlob(t *testing.T) {
-	for _, pattern := range []string{
-		"*test",           // Leading glob
-		"this*",           // Trailing glob
-		"*is*a*",          // Lots of globs
-		"**test**",        // Double glob characters
-		"**is**a***test*", // Varying number of globs
-	} {
-		testGlobMatch(t, pattern, "this_is_a_test")
+func TestCombinedOutputTimeout(t *testing.T) {
+	if sleepbin == "" {
+		t.Skip("'sleep' binary not available on OS, skipping.")
 	}
+	cmd := exec.Command(sleepbin, "10")
+	start := time.Now()
+	_, err := CombinedOutputTimeout(cmd, time.Millisecond*20)
+	elapsed := time.Since(start)
 
-	for _, pattern := range []string{
-		"test*", // Implicit substring match should fail
-		"*is",   // Partial match should fail
-		"*no*",  // Globs without a match between them should fail
-	} {
-		testGlobNoMatch(t, pattern, "this_is_a_test")
-	}
+	assert.Equal(t, TimeoutErr, err)
+	// Verify that command gets killed in 20ms, with some breathing room
+	assert.True(t, elapsed < time.Millisecond*75)
+}
+
+func TestCombinedOutput(t *testing.T) {
+	if echobin == "" {
+		t.Skip("'echo' binary not available on OS, skipping.")
+	}
+	cmd := exec.Command(echobin, "foo")
+	out, err := CombinedOutputTimeout(cmd, time.Second)
+
+	assert.NoError(t, err)
+	assert.Equal(t, "foo\n", string(out))
+}
+
+// test that CombinedOutputTimeout and exec.Cmd.CombinedOutput return
+// the same output from a failed command.
+func TestCombinedOutputError(t *testing.T) {
+	if sleepbin == "" {
+		t.Skip("'sleep' binary not available on OS, skipping.")
+	}
+	cmd := exec.Command(sleepbin, "foo")
+	expected, err := cmd.CombinedOutput()
+
+	cmd2 := exec.Command(sleepbin, "foo")
+	actual, err := CombinedOutputTimeout(cmd2, time.Second)
+
+	assert.Error(t, err)
+	assert.Equal(t, expected, actual)
+}
+
+func TestRunError(t *testing.T) {
+	if sleepbin == "" {
+		t.Skip("'sleep' binary not available on OS, skipping.")
+	}
+	cmd := exec.Command(sleepbin, "foo")
+	err := RunTimeout(cmd, time.Second)
+
+	assert.Error(t, err)
 }

internal/models/filter.go

@@ -1,33 +1,104 @@
 package internal_models
 
 import (
+	"fmt"
 	"strings"
 
+	"github.com/gobwas/glob"
+
 	"github.com/influxdata/telegraf"
-	"github.com/influxdata/telegraf/internal"
 )
 
 // TagFilter is the name of a tag, and the values on which to filter
 type TagFilter struct {
 	Name   string
 	Filter []string
+	filter glob.Glob
 }
 
 // Filter containing drop/pass and tagdrop/tagpass rules
 type Filter struct {
 	NameDrop []string
+	nameDrop glob.Glob
 	NamePass []string
+	namePass glob.Glob
 
 	FieldDrop []string
+	fieldDrop glob.Glob
 	FieldPass []string
+	fieldPass glob.Glob
 
 	TagDrop []TagFilter
 	TagPass []TagFilter
 
+	TagExclude []string
+	tagExclude glob.Glob
+	TagInclude []string
+	tagInclude glob.Glob
+
 	IsActive bool
 }
 
-func (f Filter) ShouldMetricPass(metric telegraf.Metric) bool {
+// Compile all Filter lists into glob.Glob objects.
+func (f *Filter) CompileFilter() error {
+	var err error
+	f.nameDrop, err = compileFilter(f.NameDrop)
+	if err != nil {
+		return fmt.Errorf("Error compiling 'namedrop', %s", err)
+	}
+	f.namePass, err = compileFilter(f.NamePass)
+	if err != nil {
+		return fmt.Errorf("Error compiling 'namepass', %s", err)
+	}
+
+	f.fieldDrop, err = compileFilter(f.FieldDrop)
+	if err != nil {
+		return fmt.Errorf("Error compiling 'fielddrop', %s", err)
+	}
+	f.fieldPass, err = compileFilter(f.FieldPass)
+	if err != nil {
+		return fmt.Errorf("Error compiling 'fieldpass', %s", err)
+	}
+
+	f.tagExclude, err = compileFilter(f.TagExclude)
+	if err != nil {
+		return fmt.Errorf("Error compiling 'tagexclude', %s", err)
+	}
+	f.tagInclude, err = compileFilter(f.TagInclude)
+	if err != nil {
+		return fmt.Errorf("Error compiling 'taginclude', %s", err)
+	}
+
+	for i, _ := range f.TagDrop {
+		f.TagDrop[i].filter, err = compileFilter(f.TagDrop[i].Filter)
+		if err != nil {
+			return fmt.Errorf("Error compiling 'tagdrop', %s", err)
+		}
+	}
+	for i, _ := range f.TagPass {
+		f.TagPass[i].filter, err = compileFilter(f.TagPass[i].Filter)
+		if err != nil {
+			return fmt.Errorf("Error compiling 'tagpass', %s", err)
+		}
+	}
+	return nil
+}
+
+func compileFilter(filter []string) (glob.Glob, error) {
+	if len(filter) == 0 {
+		return nil, nil
+	}
+	var g glob.Glob
+	var err error
+	if len(filter) == 1 {
+		g, err = glob.Compile(filter[0])
+	} else {
+		g, err = glob.Compile("{" + strings.Join(filter, ",") + "}")
+	}
+	return g, err
+}
+
+func (f *Filter) ShouldMetricPass(metric telegraf.Metric) bool {
 	if f.ShouldNamePass(metric.Name()) && f.ShouldTagsPass(metric.Tags()) {
 		return true
 	}
@@ -36,70 +107,51 @@ func (f Filter) ShouldMetricPass(metric telegraf.Metric) bool {
 
 // ShouldFieldsPass returns true if the metric should pass, false if should drop
 // based on the drop/pass filter parameters
-func (f Filter) ShouldNamePass(key string) bool {
-	if f.NamePass != nil {
-		for _, pat := range f.NamePass {
-			// TODO remove HasPrefix check, leaving it for now for legacy support.
-			// Cam, 2015-12-07
-			if strings.HasPrefix(key, pat) || internal.Glob(pat, key) {
-				return true
-			}
+func (f *Filter) ShouldNamePass(key string) bool {
+	if f.namePass != nil {
+		if f.namePass.Match(key) {
+			return true
 		}
 		return false
 	}
 
-	if f.NameDrop != nil {
-		for _, pat := range f.NameDrop {
-			// TODO remove HasPrefix check, leaving it for now for legacy support.
-			// Cam, 2015-12-07
-			if strings.HasPrefix(key, pat) || internal.Glob(pat, key) {
-				return false
-			}
+	if f.nameDrop != nil {
+		if f.nameDrop.Match(key) {
+			return false
 		}
-
-		return true
 	}
 	return true
 }
 
 // ShouldFieldsPass returns true if the metric should pass, false if should drop
 // based on the drop/pass filter parameters
-func (f Filter) ShouldFieldsPass(key string) bool {
-	if f.FieldPass != nil {
-		for _, pat := range f.FieldPass {
-			// TODO remove HasPrefix check, leaving it for now for legacy support.
-			// Cam, 2015-12-07
-			if strings.HasPrefix(key, pat) || internal.Glob(pat, key) {
-				return true
-			}
+func (f *Filter) ShouldFieldsPass(key string) bool {
+	if f.fieldPass != nil {
+		if f.fieldPass.Match(key) {
+			return true
 		}
 		return false
 	}
 
-	if f.FieldDrop != nil {
-		for _, pat := range f.FieldDrop {
-			// TODO remove HasPrefix check, leaving it for now for legacy support.
-			// Cam, 2015-12-07
-			if strings.HasPrefix(key, pat) || internal.Glob(pat, key) {
-				return false
-			}
+	if f.fieldDrop != nil {
+		if f.fieldDrop.Match(key) {
+			return false
 		}
-
-		return true
 	}
 	return true
 }
 
 // ShouldTagsPass returns true if the metric should pass, false if should drop
 // based on the tagdrop/tagpass filter parameters
-func (f Filter) ShouldTagsPass(tags map[string]string) bool {
+func (f *Filter) ShouldTagsPass(tags map[string]string) bool {
 	if f.TagPass != nil {
 		for _, pat := range f.TagPass {
+			if pat.filter == nil {
+				continue
+			}
 			if tagval, ok := tags[pat.Name]; ok {
-				for _, filter := range pat.Filter {
-					if internal.Glob(filter, tagval) {
-						return true
-					}
+				if pat.filter.Match(tagval) {
+					return true
 				}
 			}
 		}
@@ -108,11 +160,12 @@ func (f Filter) ShouldTagsPass(tags map[string]string) bool {
 
 	if f.TagDrop != nil {
 		for _, pat := range f.TagDrop {
+			if pat.filter == nil {
+				continue
+			}
 			if tagval, ok := tags[pat.Name]; ok {
-				for _, filter := range pat.Filter {
-					if internal.Glob(filter, tagval) {
-						return false
-					}
+				if pat.filter.Match(tagval) {
+					return false
 				}
 			}
 		}
@@ -121,3 +174,23 @@ func (f Filter) ShouldTagsPass(tags map[string]string) bool {
 
 	return true
 }
+
+// Apply TagInclude and TagExclude filters.
+// modifies the tags map in-place.
+func (f *Filter) FilterTags(tags map[string]string) {
+	if f.tagInclude != nil {
+		for k, _ := range tags {
+			if !f.tagInclude.Match(k) {
+				delete(tags, k)
+			}
+		}
+	}
+
+	if f.tagExclude != nil {
+		for k, _ := range tags {
+			if f.tagExclude.Match(k) {
+				delete(tags, k)
+			}
+		}
+	}
+}

internal/models/filter_test.go

@@ -2,6 +2,11 @@ package internal_models
 
 import (
 	"testing"
+
+	"github.com/influxdata/telegraf/testutil"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
 func TestFilter_Empty(t *testing.T) {
@@ -28,6 +33,7 @@ func TestFilter_NamePass(t *testing.T) {
 	f := Filter{
 		NamePass: []string{"foo*", "cpu_usage_idle"},
 	}
+	require.NoError(t, f.CompileFilter())
 
 	passes := []string{
 		"foo",
@@ -61,6 +67,7 @@ func TestFilter_NameDrop(t *testing.T) {
 	f := Filter{
 		NameDrop: []string{"foo*", "cpu_usage_idle"},
 	}
+	require.NoError(t, f.CompileFilter())
 
 	drops := []string{
 		"foo",
@@ -94,6 +101,7 @@ func TestFilter_FieldPass(t *testing.T) {
 	f := Filter{
 		FieldPass: []string{"foo*", "cpu_usage_idle"},
 	}
+	require.NoError(t, f.CompileFilter())
 
 	passes := []string{
 		"foo",
@@ -127,6 +135,7 @@ func TestFilter_FieldDrop(t *testing.T) {
 	f := Filter{
 		FieldDrop: []string{"foo*", "cpu_usage_idle"},
 	}
+	require.NoError(t, f.CompileFilter())
 
 	drops := []string{
 		"foo",
@@ -169,6 +178,7 @@ func TestFilter_TagPass(t *testing.T) {
 	f := Filter{
 		TagPass: filters,
 	}
+	require.NoError(t, f.CompileFilter())
 
 	passes := []map[string]string{
 		{"cpu": "cpu-total"},
@@ -212,6 +222,7 @@ func TestFilter_TagDrop(t *testing.T) {
 	f := Filter{
 		TagDrop: filters,
 	}
+	require.NoError(t, f.CompileFilter())
 
 	drops := []map[string]string{
 		{"cpu": "cpu-total"},
@@ -241,3 +252,115 @@ func TestFilter_TagDrop(t *testing.T) {
 		}
 	}
 }
+
+func TestFilter_CompileFilterError(t *testing.T) {
+	f := Filter{
+		NameDrop: []string{"", ""},
+	}
+	assert.Error(t, f.CompileFilter())
+	f = Filter{
+		NamePass: []string{"", ""},
+	}
+	assert.Error(t, f.CompileFilter())
+	f = Filter{
+		FieldDrop: []string{"", ""},
+	}
+	assert.Error(t, f.CompileFilter())
+	f = Filter{
+		FieldPass: []string{"", ""},
+	}
+	assert.Error(t, f.CompileFilter())
+	f = Filter{
+		TagExclude: []string{"", ""},
+	}
+	assert.Error(t, f.CompileFilter())
+	f = Filter{
+		TagInclude: []string{"", ""},
+	}
+	assert.Error(t, f.CompileFilter())
+	filters := []TagFilter{
+		TagFilter{
+			Name:   "cpu",
+			Filter: []string{"{foobar}"},
+		}}
+	f = Filter{
+		TagDrop: filters,
+	}
+	require.Error(t, f.CompileFilter())
+	filters = []TagFilter{
+		TagFilter{
+			Name:   "cpu",
+			Filter: []string{"{foobar}"},
+		}}
+	f = Filter{
+		TagPass: filters,
+	}
+	require.Error(t, f.CompileFilter())
+}
+
+func TestFilter_ShouldMetricsPass(t *testing.T) {
+	m := testutil.TestMetric(1, "testmetric")
+	f := Filter{
+		NameDrop: []string{"foobar"},
+	}
+	require.NoError(t, f.CompileFilter())
+	require.True(t, f.ShouldMetricPass(m))
+
+	m = testutil.TestMetric(1, "foobar")
+	require.False(t, f.ShouldMetricPass(m))
+}
+
+func TestFilter_FilterTagsNoMatches(t *testing.T) {
+	pretags := map[string]string{
+		"host":  "localhost",
+		"mytag": "foobar",
+	}
+	f := Filter{
+		TagExclude: []string{"nomatch"},
+	}
+	require.NoError(t, f.CompileFilter())
+
+	f.FilterTags(pretags)
+	assert.Equal(t, map[string]string{
+		"host":  "localhost",
+		"mytag": "foobar",
+	}, pretags)
+
+	f = Filter{
+		TagInclude: []string{"nomatch"},
+	}
+	require.NoError(t, f.CompileFilter())
+
+	f.FilterTags(pretags)
+	assert.Equal(t, map[string]string{}, pretags)
+}
+
+func TestFilter_FilterTagsMatches(t *testing.T) {
+	pretags := map[string]string{
+		"host":  "localhost",
+		"mytag": "foobar",
+	}
+	f := Filter{
+		TagExclude: []string{"ho*"},
+	}
+	require.NoError(t, f.CompileFilter())
+
+	f.FilterTags(pretags)
+	assert.Equal(t, map[string]string{
+		"mytag": "foobar",
+	}, pretags)
+
+	pretags = map[string]string{
+		"host":  "localhost",
+		"mytag": "foobar",
+	}
+	f = Filter{
+		TagInclude: []string{"my*"},
+	}
+	require.NoError(t, f.CompileFilter())
+
+	f.FilterTags(pretags)
+	assert.Equal(t, map[string]string{
+		"mytag": "foobar",
+	}, pretags)
+}

internal/models/running_output.go

@@ -2,48 +2,54 @@ package internal_models
 
 import (
 	"log"
-	"sync"
 	"time"
 
 	"github.com/influxdata/telegraf"
+	"github.com/influxdata/telegraf/internal/buffer"
 )
 
 const (
-	// Default number of metrics kept between flushes.
-	DEFAULT_METRIC_BUFFER_LIMIT = 10000
+	// Default size of metrics batch size.
+	DEFAULT_METRIC_BATCH_SIZE = 1000
 
-	// Limit how many full metric buffers are kept due to failed writes.
-	FULL_METRIC_BUFFERS_LIMIT = 100
+	// Default number of metrics kept. It should be a multiple of batch size.
+	DEFAULT_METRIC_BUFFER_LIMIT = 10000
 )
 
+// RunningOutput contains the output configuration
 type RunningOutput struct {
-	Name                string
-	Output              telegraf.Output
-	Config              *OutputConfig
-	Quiet               bool
-	MetricBufferLimit   int
-	FlushBufferWhenFull bool
+	Name              string
+	Output            telegraf.Output
+	Config            *OutputConfig
+	Quiet             bool
+	MetricBufferLimit int
+	MetricBatchSize   int
 
-	metrics    []telegraf.Metric
-	tmpmetrics map[int][]telegraf.Metric
-	overwriteI int
-	mapI       int
-
-	sync.Mutex
+	metrics     *buffer.Buffer
+	failMetrics *buffer.Buffer
 }
 
 func NewRunningOutput(
 	name string,
 	output telegraf.Output,
 	conf *OutputConfig,
+	batchSize int,
+	bufferLimit int,
 ) *RunningOutput {
+	if bufferLimit == 0 {
+		bufferLimit = DEFAULT_METRIC_BUFFER_LIMIT
+	}
+	if batchSize == 0 {
+		batchSize = DEFAULT_METRIC_BATCH_SIZE
+	}
 	ro := &RunningOutput{
 		Name:              name,
-		metrics:           make([]telegraf.Metric, 0),
-		tmpmetrics:        make(map[int][]telegraf.Metric),
+		metrics:           buffer.NewBuffer(batchSize),
+		failMetrics:       buffer.NewBuffer(bufferLimit),
 		Output:            output,
 		Config:            conf,
-		MetricBufferLimit: DEFAULT_METRIC_BUFFER_LIMIT,
+		MetricBufferLimit: bufferLimit,
+		MetricBatchSize:   batchSize,
 	}
 	return ro
 }
@@ -56,76 +62,92 @@ func (ro *RunningOutput) AddMetric(metric telegraf.Metric) {
 			return
 		}
 	}
-	ro.Lock()
-	defer ro.Unlock()
 
-	if len(ro.metrics) < ro.MetricBufferLimit {
-		ro.metrics = append(ro.metrics, metric)
-	} else {
-		if ro.FlushBufferWhenFull {
-			ro.metrics = append(ro.metrics, metric)
-			tmpmetrics := make([]telegraf.Metric, len(ro.metrics))
-			copy(tmpmetrics, ro.metrics)
-			ro.metrics = make([]telegraf.Metric, 0)
-			err := ro.write(tmpmetrics)
-			if err != nil {
-				log.Printf("ERROR writing full metric buffer to output %s, %s",
-					ro.Name, err)
-				if len(ro.tmpmetrics) == FULL_METRIC_BUFFERS_LIMIT {
-					ro.mapI = 0
-					// overwrite one
-					ro.tmpmetrics[ro.mapI] = tmpmetrics
-					ro.mapI++
-				} else {
-					ro.tmpmetrics[ro.mapI] = tmpmetrics
-					ro.mapI++
-				}
-			}
-		} else {
-			log.Printf("WARNING: overwriting cached metrics, you may want to " +
-				"increase the metric_buffer_limit setting in your [agent] " +
-				"config if you do not wish to overwrite metrics.\n")
-			if ro.overwriteI == len(ro.metrics) {
-				ro.overwriteI = 0
-			}
-			ro.metrics[ro.overwriteI] = metric
-			ro.overwriteI++
+	// Filter any tagexclude/taginclude parameters before adding metric
+	if len(ro.Config.Filter.TagExclude) != 0 || len(ro.Config.Filter.TagInclude) != 0 {
+		// In order to filter out tags, we need to create a new metric, since
+		// metrics are immutable once created.
+		tags := metric.Tags()
+		fields := metric.Fields()
+		t := metric.Time()
+		name := metric.Name()
+		ro.Config.Filter.FilterTags(tags)
+		// error is not possible if creating from another metric, so ignore.
+		metric, _ = telegraf.NewMetric(name, tags, fields, t)
+	}
+
+	ro.metrics.Add(metric)
+	if ro.metrics.Len() == ro.MetricBatchSize {
+		batch := ro.metrics.Batch(ro.MetricBatchSize)
+		err := ro.write(batch)
+		if err != nil {
+			ro.failMetrics.Add(batch...)
 		}
 	}
 }
 
 // Write writes all cached points to this output.
 func (ro *RunningOutput) Write() error {
-	ro.Lock()
-	defer ro.Unlock()
-	err := ro.write(ro.metrics)
-	if err != nil {
-		return err
-	} else {
-		ro.metrics = make([]telegraf.Metric, 0)
-		ro.overwriteI = 0
+	if !ro.Quiet {
+		log.Printf("Output [%s] buffer fullness: %d / %d metrics. "+
+			"Total gathered metrics: %d. Total dropped metrics: %d.",
+			ro.Name,
+			ro.failMetrics.Len()+ro.metrics.Len(),
+			ro.MetricBufferLimit,
+			ro.metrics.Total(),
+			ro.metrics.Drops()+ro.failMetrics.Drops())
 	}
 
-	// Write any cached metric buffers that failed previously
-	for i, tmpmetrics := range ro.tmpmetrics {
-		if err := ro.write(tmpmetrics); err != nil {
-			return err
-		} else {
-			delete(ro.tmpmetrics, i)
+	var err error
+	if !ro.failMetrics.IsEmpty() {
+		bufLen := ro.failMetrics.Len()
+		// how many batches of failed writes we need to write.
+		nBatches := bufLen/ro.MetricBatchSize + 1
+		batchSize := ro.MetricBatchSize
+
+		for i := 0; i < nBatches; i++ {
+			// If it's the last batch, only grab the metrics that have not had
+			// a write attempt already (this is primarily to preserve order).
+			if i == nBatches-1 {
+				batchSize = bufLen % ro.MetricBatchSize
+			}
+			batch := ro.failMetrics.Batch(batchSize)
+			// If we've already failed previous writes, don't bother trying to
+			// write to this output again. We are not exiting the loop just so
+			// that we can rotate the metrics to preserve order.
+			if err == nil {
+				err = ro.write(batch)
+			}
+			if err != nil {
+				ro.failMetrics.Add(batch...)
+			}
 		}
 	}
 
+	batch := ro.metrics.Batch(ro.MetricBatchSize)
+	// see comment above about not trying to write to an already failed output.
+	// if ro.failMetrics is empty then err will always be nil at this point.
+	if err == nil {
+		err = ro.write(batch)
+	}
+	if err != nil {
+		ro.failMetrics.Add(batch...)
+		return err
+	}
 	return nil
 }
 
 func (ro *RunningOutput) write(metrics []telegraf.Metric) error {
+	if len(metrics) == 0 {
+		return nil
+	}
 	start := time.Now()
 	err := ro.Output.Write(metrics)
 	elapsed := time.Since(start)
 	if err == nil {
 		if !ro.Quiet {
-			log.Printf("Wrote %d metrics to output %s in %s\n",
-				len(metrics), ro.Name, elapsed)
+			log.Printf("Output [%s] wrote batch of %d metrics in %s\n",
+				ro.Name, len(metrics), elapsed)
 		}
 	}
 	return err

internal/models/running_output_test.go

@@ -2,7 +2,6 @@ package internal_models
 
 import (
 	"fmt"
-	"sort"
 	"sync"
 	"testing"
 
@@ -29,16 +28,100 @@ var next5 = []telegraf.Metric{
 	testutil.TestMetric(101, "metric10"),
 }
 
-// Test that we can write metrics with simple default setup.
-func TestRunningOutputDefault(t *testing.T) {
+// Benchmark adding metrics.
+func BenchmarkRunningOutputAddWrite(b *testing.B) {
 	conf := &OutputConfig{
 		Filter: Filter{
 			IsActive: false,
 		},
 	}
 
+	m := &perfOutput{}
+	ro := NewRunningOutput("test", m, conf, 1000, 10000)
+	ro.Quiet = true
+
+	for n := 0; n < b.N; n++ {
+		ro.AddMetric(first5[0])
+		ro.Write()
+	}
+}
+
+// Benchmark adding metrics.
+func BenchmarkRunningOutputAddWriteEvery100(b *testing.B) {
+	conf := &OutputConfig{
+		Filter: Filter{
+			IsActive: false,
+		},
+	}
+
+	m := &perfOutput{}
+	ro := NewRunningOutput("test", m, conf, 1000, 10000)
+	ro.Quiet = true
+
+	for n := 0; n < b.N; n++ {
+		ro.AddMetric(first5[0])
+		if n%100 == 0 {
+			ro.Write()
+		}
+	}
+}
+
+// Benchmark adding metrics.
+func BenchmarkRunningOutputAddFailWrites(b *testing.B) {
+	conf := &OutputConfig{
+		Filter: Filter{
+			IsActive: false,
+		},
+	}
+
+	m := &perfOutput{}
+	m.failWrite = true
+	ro := NewRunningOutput("test", m, conf, 1000, 10000)
+	ro.Quiet = true
+
+	for n := 0; n < b.N; n++ {
+		ro.AddMetric(first5[0])
+	}
+}
+
+// Test that NameDrop filters ger properly applied.
+func TestRunningOutput_DropFilter(t *testing.T) {
+	conf := &OutputConfig{
+		Filter: Filter{
+			IsActive: true,
+			NameDrop: []string{"metric1", "metric2"},
+		},
+	}
+	assert.NoError(t, conf.Filter.CompileFilter())
+
 	m := &mockOutput{}
-	ro := NewRunningOutput("test", m, conf)
+	ro := NewRunningOutput("test", m, conf, 1000, 10000)
+
+	for _, metric := range first5 {
+		ro.AddMetric(metric)
+	}
+	for _, metric := range next5 {
+		ro.AddMetric(metric)
+	}
+	assert.Len(t, m.Metrics(), 0)
+
+	err := ro.Write()
+	assert.NoError(t, err)
+	assert.Len(t, m.Metrics(), 8)
+}
+
+// Test that NameDrop filters without a match do nothing.
+func TestRunningOutput_PassFilter(t *testing.T) {
+	conf := &OutputConfig{
+		Filter: Filter{
+			IsActive: true,
+			NameDrop: []string{"metric1000", "foo*"},
+		},
+	}
+	assert.NoError(t, conf.Filter.CompileFilter())
+
+	m := &mockOutput{}
+	ro := NewRunningOutput("test", m, conf, 1000, 10000)
 
 	for _, metric := range first5 {
 		ro.AddMetric(metric)
@@ -53,41 +136,96 @@ func TestRunningOutputDefault(t *testing.T) {
 	assert.Len(t, m.Metrics(), 10)
 }
 
-// Test that the first metric gets overwritten if there is a buffer overflow.
-func TestRunningOutputOverwrite(t *testing.T) {
+// Test that tags are properly included
+func TestRunningOutput_TagIncludeNoMatch(t *testing.T) {
 	conf := &OutputConfig{
 		Filter: Filter{
-			IsActive: false,
+			IsActive:   true,
+			TagInclude: []string{"nothing*"},
 		},
 	}
+	assert.NoError(t, conf.Filter.CompileFilter())
 
 	m := &mockOutput{}
-	ro := NewRunningOutput("test", m, conf)
-	ro.MetricBufferLimit = 4
+	ro := NewRunningOutput("test", m, conf, 1000, 10000)
 
-	for _, metric := range first5 {
-		ro.AddMetric(metric)
-	}
-	require.Len(t, m.Metrics(), 0)
+	ro.AddMetric(first5[0])
+	assert.Len(t, m.Metrics(), 0)
 
 	err := ro.Write()
-	require.NoError(t, err)
-	require.Len(t, m.Metrics(), 4)
-
-	var expected, actual []string
-	for i, exp := range first5[1:] {
-		expected = append(expected, exp.String())
-		actual = append(actual, m.Metrics()[i].String())
-	}
-
-	sort.Strings(expected)
-	sort.Strings(actual)
-
-	assert.Equal(t, expected, actual)
+	assert.NoError(t, err)
+	assert.Len(t, m.Metrics(), 1)
+	assert.Empty(t, m.Metrics()[0].Tags())
 }
 
-// Test that multiple buffer overflows are handled properly.
-func TestRunningOutputMultiOverwrite(t *testing.T) {
+// Test that tags are properly excluded
+func TestRunningOutput_TagExcludeMatch(t *testing.T) {
+	conf := &OutputConfig{
+		Filter: Filter{
+			IsActive:   true,
+			TagExclude: []string{"tag*"},
+		},
+	}
+	assert.NoError(t, conf.Filter.CompileFilter())
+
+	m := &mockOutput{}
+	ro := NewRunningOutput("test", m, conf, 1000, 10000)
+
+	ro.AddMetric(first5[0])
+	assert.Len(t, m.Metrics(), 0)
+
+	err := ro.Write()
+	assert.NoError(t, err)
+	assert.Len(t, m.Metrics(), 1)
+	assert.Len(t, m.Metrics()[0].Tags(), 0)
+}
+
+// Test that tags are properly Excluded
+func TestRunningOutput_TagExcludeNoMatch(t *testing.T) {
+	conf := &OutputConfig{
+		Filter: Filter{
+			IsActive:   true,
+			TagExclude: []string{"nothing*"},
+		},
+	}
+	assert.NoError(t, conf.Filter.CompileFilter())
+
+	m := &mockOutput{}
+	ro := NewRunningOutput("test", m, conf, 1000, 10000)
+
+	ro.AddMetric(first5[0])
+	assert.Len(t, m.Metrics(), 0)
+
+	err := ro.Write()
+	assert.NoError(t, err)
+	assert.Len(t, m.Metrics(), 1)
+	assert.Len(t, m.Metrics()[0].Tags(), 1)
+}
+
+// Test that tags are properly included
+func TestRunningOutput_TagIncludeMatch(t *testing.T) {
+	conf := &OutputConfig{
+		Filter: Filter{
+			IsActive:   true,
+			TagInclude: []string{"tag*"},
+		},
+	}
+	assert.NoError(t, conf.Filter.CompileFilter())
+
+	m := &mockOutput{}
+	ro := NewRunningOutput("test", m, conf, 1000, 10000)
+
+	ro.AddMetric(first5[0])
+	assert.Len(t, m.Metrics(), 0)
+
+	err := ro.Write()
+	assert.NoError(t, err)
+	assert.Len(t, m.Metrics(), 1)
+	assert.Len(t, m.Metrics()[0].Tags(), 1)
+}
+
+// Test that we can write metrics with simple default setup.
+func TestRunningOutputDefault(t *testing.T) {
 	conf := &OutputConfig{
 		Filter: Filter{
 			IsActive: false,
@@ -95,8 +233,7 @@ func TestRunningOutputMultiOverwrite(t *testing.T) {
 	}
 
 	m := &mockOutput{}
-	ro := NewRunningOutput("test", m, conf)
-	ro.MetricBufferLimit = 3
+	ro := NewRunningOutput("test", m, conf, 1000, 10000)
 
 	for _, metric := range first5 {
 		ro.AddMetric(metric)
@@ -104,22 +241,11 @@ func TestRunningOutputMultiOverwrite(t *testing.T) {
 	for _, metric := range next5 {
 		ro.AddMetric(metric)
 	}
-	require.Len(t, m.Metrics(), 0)
+	assert.Len(t, m.Metrics(), 0)
 
 	err := ro.Write()
-	require.NoError(t, err)
-	require.Len(t, m.Metrics(), 3)
-
-	var expected, actual []string
-	for i, exp := range next5[2:] {
-		expected = append(expected, exp.String())
-		actual = append(actual, m.Metrics()[i].String())
-	}
-
-	sort.Strings(expected)
-	sort.Strings(actual)
-
-	assert.Equal(t, expected, actual)
+	assert.NoError(t, err)
+	assert.Len(t, m.Metrics(), 10)
 }
 
 // Test that running output doesn't flush until it's full when
@@ -132,11 +258,9 @@ func TestRunningOutputFlushWhenFull(t *testing.T) {
 	}
 
 	m := &mockOutput{}
-	ro := NewRunningOutput("test", m, conf)
-	ro.FlushBufferWhenFull = true
-	ro.MetricBufferLimit = 5
+	ro := NewRunningOutput("test", m, conf, 6, 10)
 
-	// Fill buffer to limit
+	// Fill buffer to 1 under limit
 	for _, metric := range first5 {
 		ro.AddMetric(metric)
 	}
@@ -165,9 +289,7 @@ func TestRunningOutputMultiFlushWhenFull(t *testing.T) {
 	}
 
 	m := &mockOutput{}
-	ro := NewRunningOutput("test", m, conf)
-	ro.FlushBufferWhenFull = true
-	ro.MetricBufferLimit = 4
+	ro := NewRunningOutput("test", m, conf, 4, 12)
 
 	// Fill buffer past limit twive
 	for _, metric := range first5 {
@@ -177,7 +299,7 @@ func TestRunningOutputMultiFlushWhenFull(t *testing.T) {
 		ro.AddMetric(metric)
 	}
 	// flushed twice
-	assert.Len(t, m.Metrics(), 10)
+	assert.Len(t, m.Metrics(), 8)
 }
 
 func TestRunningOutputWriteFail(t *testing.T) {
@@ -189,11 +311,9 @@ func TestRunningOutputWriteFail(t *testing.T) {
 
 	m := &mockOutput{}
 	m.failWrite = true
-	ro := NewRunningOutput("test", m, conf)
-	ro.FlushBufferWhenFull = true
-	ro.MetricBufferLimit = 4
+	ro := NewRunningOutput("test", m, conf, 4, 12)
 
-	// Fill buffer past limit twice
+	// Fill buffer to limit twice
 	for _, metric := range first5 {
 		ro.AddMetric(metric)
 	}
@@ -216,6 +336,161 @@ func TestRunningOutputWriteFail(t *testing.T) {
 	assert.Len(t, m.Metrics(), 10)
 }
 
+// Verify that the order of points is preserved during a write failure.
+func TestRunningOutputWriteFailOrder(t *testing.T) {
+	conf := &OutputConfig{
+		Filter: Filter{
+			IsActive: false,
+		},
+	}
+
+	m := &mockOutput{}
+	m.failWrite = true
+	ro := NewRunningOutput("test", m, conf, 100, 1000)
+
+	// add 5 metrics
+	for _, metric := range first5 {
+		ro.AddMetric(metric)
+	}
+	// no successful flush yet
+	assert.Len(t, m.Metrics(), 0)
+
+	// Write fails
+	err := ro.Write()
+	require.Error(t, err)
+	// no successful flush yet
+	assert.Len(t, m.Metrics(), 0)
+
+	m.failWrite = false
+	// add 5 more metrics
+	for _, metric := range next5 {
+		ro.AddMetric(metric)
+	}
+	err = ro.Write()
+	require.NoError(t, err)
+
+	// Verify that 10 metrics were written
+	assert.Len(t, m.Metrics(), 10)
+	// Verify that they are in order
+	expected := append(first5, next5...)
+	assert.Equal(t, expected, m.Metrics())
+}
+
+// Verify that the order of points is preserved during many write failures.
+func TestRunningOutputWriteFailOrder2(t *testing.T) {
+	conf := &OutputConfig{
+		Filter: Filter{
+			IsActive: false,
+		},
+	}
+
+	m := &mockOutput{}
+	m.failWrite = true
+	ro := NewRunningOutput("test", m, conf, 5, 100)
+
+	// add 5 metrics
+	for _, metric := range first5 {
+		ro.AddMetric(metric)
+	}
+	// Write fails
+	err := ro.Write()
+	require.Error(t, err)
+	// no successful flush yet
+	assert.Len(t, m.Metrics(), 0)
+
+	// add 5 metrics
+	for _, metric := range next5 {
+		ro.AddMetric(metric)
+	}
+	// Write fails
+	err = ro.Write()
+	require.Error(t, err)
+	// no successful flush yet
+	assert.Len(t, m.Metrics(), 0)
+
+	// add 5 metrics
+	for _, metric := range first5 {
+		ro.AddMetric(metric)
+	}
+	// Write fails
+	err = ro.Write()
+	require.Error(t, err)
+	// no successful flush yet
+	assert.Len(t, m.Metrics(), 0)
+
+	// add 5 metrics
+	for _, metric := range next5 {
+		ro.AddMetric(metric)
+	}
+	// Write fails
+	err = ro.Write()
+	require.Error(t, err)
+	// no successful flush yet
+	assert.Len(t, m.Metrics(), 0)
+
+	m.failWrite = false
+	err = ro.Write()
+	require.NoError(t, err)
+
+	// Verify that 10 metrics were written
+	assert.Len(t, m.Metrics(), 20)
+	// Verify that they are in order
+	expected := append(first5, next5...)
+	expected = append(expected, first5...)
+	expected = append(expected, next5...)
+	assert.Equal(t, expected, m.Metrics())
+}
+
+// Verify that the order of points is preserved when there is a remainder
+// of points for the batch.
+//
+// ie, with a batch size of 5:
+//
+//     1 2 3 4 5 6 <-- order, failed points
+//     6 1 2 3 4 5 <-- order, after 1st write failure (1 2 3 4 5 was batch)
+//     1 2 3 4 5 6 <-- order, after 2nd write failure, (6 was batch)
+//
+func TestRunningOutputWriteFailOrder3(t *testing.T) {
+	conf := &OutputConfig{
+		Filter: Filter{
+			IsActive: false,
+		},
+	}
+
+	m := &mockOutput{}
+	m.failWrite = true
+	ro := NewRunningOutput("test", m, conf, 5, 1000)
+
+	// add 5 metrics
+	for _, metric := range first5 {
+		ro.AddMetric(metric)
+	}
+	// no successful flush yet
+	assert.Len(t, m.Metrics(), 0)
+
+	// Write fails
+	err := ro.Write()
+	require.Error(t, err)
+	// no successful flush yet
+	assert.Len(t, m.Metrics(), 0)
+
+	// add and attempt to write a single metric:
+	ro.AddMetric(next5[0])
+	err = ro.Write()
+	require.Error(t, err)
+
+	// unset fail and write metrics
+	m.failWrite = false
+	err = ro.Write()
+	require.NoError(t, err)
+
+	// Verify that 6 metrics were written
+	assert.Len(t, m.Metrics(), 6)
+	// Verify that they are in order
+	expected := append(first5, next5[0])
+	assert.Equal(t, expected, m.Metrics())
+}
+
 type mockOutput struct {
 	sync.Mutex
 
@@ -263,3 +538,31 @@ func (m *mockOutput) Metrics() []telegraf.Metric {
 	defer m.Unlock()
 	return m.metrics
 }
+
+type perfOutput struct {
+	// if true, mock a write failure
+	failWrite bool
+}
+
+func (m *perfOutput) Connect() error {
+	return nil
+}
+
+func (m *perfOutput) Close() error {
+	return nil
+}
+
+func (m *perfOutput) Description() string {
+	return ""
+}
+
+func (m *perfOutput) SampleConfig() string {
+	return ""
+}
+
+func (m *perfOutput) Write(metrics []telegraf.Metric) error {
+	if m.failWrite {
+		return fmt.Errorf("Failed Write!")
+	}
+	return nil
+}

plugins/inputs/EXAMPLE_README.md

@@ -30,8 +30,6 @@ The example plugin gathers metrics about example things
 
 ### Example Output:
 
-Give an example `-test` output here
-
 ```
 $ ./telegraf -config telegraf.conf -input-filter example -test
 measurement1,tag1=foo,tag2=bar field1=1i,field2=2.1 1453831884664956455

plugins/inputs/all/all.go

@@ -4,6 +4,11 @@ import (
 	_ "github.com/influxdata/telegraf/plugins/inputs/aerospike"
 	_ "github.com/influxdata/telegraf/plugins/inputs/apache"
 	_ "github.com/influxdata/telegraf/plugins/inputs/bcache"
+	_ "github.com/influxdata/telegraf/plugins/inputs/cassandra"
+	_ "github.com/influxdata/telegraf/plugins/inputs/ceph"
+	_ "github.com/influxdata/telegraf/plugins/inputs/chrony"
+	_ "github.com/influxdata/telegraf/plugins/inputs/cloudwatch"
+	_ "github.com/influxdata/telegraf/plugins/inputs/couchbase"
 	_ "github.com/influxdata/telegraf/plugins/inputs/couchdb"
 	_ "github.com/influxdata/telegraf/plugins/inputs/disque"
 	_ "github.com/influxdata/telegraf/plugins/inputs/dns_query"
@@ -11,10 +16,13 @@ import (
 	_ "github.com/influxdata/telegraf/plugins/inputs/dovecot"
 	_ "github.com/influxdata/telegraf/plugins/inputs/elasticsearch"
 	_ "github.com/influxdata/telegraf/plugins/inputs/exec"
+	_ "github.com/influxdata/telegraf/plugins/inputs/filestat"
 	_ "github.com/influxdata/telegraf/plugins/inputs/github_webhooks"
 	_ "github.com/influxdata/telegraf/plugins/inputs/haproxy"
+	_ "github.com/influxdata/telegraf/plugins/inputs/http_response"
 	_ "github.com/influxdata/telegraf/plugins/inputs/httpjson"
 	_ "github.com/influxdata/telegraf/plugins/inputs/influxdb"
+	_ "github.com/influxdata/telegraf/plugins/inputs/ipmi_sensor"
 	_ "github.com/influxdata/telegraf/plugins/inputs/jolokia"
 	_ "github.com/influxdata/telegraf/plugins/inputs/kafka_consumer"
 	_ "github.com/influxdata/telegraf/plugins/inputs/leofs"
@@ -29,10 +37,13 @@ import (
 	_ "github.com/influxdata/telegraf/plugins/inputs/net_response"
 	_ "github.com/influxdata/telegraf/plugins/inputs/nginx"
 	_ "github.com/influxdata/telegraf/plugins/inputs/nsq"
+	_ "github.com/influxdata/telegraf/plugins/inputs/nstat"
+	_ "github.com/influxdata/telegraf/plugins/inputs/ntpq"
 	_ "github.com/influxdata/telegraf/plugins/inputs/passenger"
 	_ "github.com/influxdata/telegraf/plugins/inputs/phpfpm"
 	_ "github.com/influxdata/telegraf/plugins/inputs/ping"
 	_ "github.com/influxdata/telegraf/plugins/inputs/postgresql"
+	_ "github.com/influxdata/telegraf/plugins/inputs/postgresql_extensible"
 	_ "github.com/influxdata/telegraf/plugins/inputs/powerdns"
 	_ "github.com/influxdata/telegraf/plugins/inputs/procstat"
 	_ "github.com/influxdata/telegraf/plugins/inputs/prometheus"
@@ -46,9 +57,14 @@ import (
 	_ "github.com/influxdata/telegraf/plugins/inputs/snmp"
 	_ "github.com/influxdata/telegraf/plugins/inputs/sqlserver"
 	_ "github.com/influxdata/telegraf/plugins/inputs/statsd"
+	_ "github.com/influxdata/telegraf/plugins/inputs/sysstat"
 	_ "github.com/influxdata/telegraf/plugins/inputs/system"
+	_ "github.com/influxdata/telegraf/plugins/inputs/tail"
+	_ "github.com/influxdata/telegraf/plugins/inputs/tcp_listener"
 	_ "github.com/influxdata/telegraf/plugins/inputs/trig"
 	_ "github.com/influxdata/telegraf/plugins/inputs/twemproxy"
+	_ "github.com/influxdata/telegraf/plugins/inputs/udp_listener"
+	_ "github.com/influxdata/telegraf/plugins/inputs/varnish"
 	_ "github.com/influxdata/telegraf/plugins/inputs/win_perf_counters"
 	_ "github.com/influxdata/telegraf/plugins/inputs/zfs"
 	_ "github.com/influxdata/telegraf/plugins/inputs/zookeeper"

plugins/inputs/apache/apache.go

@@ -58,7 +58,10 @@ var tr = &http.Transport{
 	ResponseHeaderTimeout: time.Duration(3 * time.Second),
 }
 
-var client = &http.Client{Transport: tr}
+var client = &http.Client{
+	Transport: tr,
+	Timeout:   time.Duration(4 * time.Second),
+}
 
 func (n *Apache) gatherUrl(addr *url.URL, acc telegraf.Accumulator) error {
 	resp, err := client.Get(addr.String())

plugins/inputs/cassandra/README.md

@@ -0,0 +1,125 @@
+# Telegraf plugin: Cassandra
+
+#### Plugin arguments:
+- **context** string: Context root used for jolokia url
+- **servers** []string: List of servers with the format "<user:passwd@><host>:port"
+- **metrics** []string: List of Jmx paths that identify mbeans attributes
+
+#### Description
+
+The Cassandra plugin collects Cassandra/JVM metrics exposed as MBean's attributes through jolokia REST endpoint. All metrics are collected for each server configured.
+
+See: https://jolokia.org/ and [Cassandra Documentation](http://docs.datastax.com/en/cassandra/3.x/cassandra/operations/monitoringCassandraTOC.html)
+
+# Measurements:
+Cassandra plugin produces one or more measurements for each metric configured, adding Server's name  as `host` tag. More than one measurement is generated when querying table metrics with a wildcard for the keyspace or table name.
+
+Given a configuration like:
+
+```toml
+[[inputs.cassandra]]
+  context = "/jolokia/read"
+  servers = [":8778"]
+  metrics = ["/java.lang:type=Memory/HeapMemoryUsage"]
+```
+
+The collected metrics will be:
+
+```
+javaMemory,host=myHost,mname=HeapMemoryUsage HeapMemoryUsage_committed=1040187392,HeapMemoryUsage_init=1050673152,HeapMemoryUsage_max=1040187392,HeapMemoryUsage_used=368155000 1459551767230567084
+```
+
+# Useful Metrics:
+
+Here is a list of metrics that might be useful to monitor your cassandra cluster. This was put together from multiple sources on the web.
+
+- [How to monitor Cassandra performance metrics](https://www.datadoghq.com/blog/how-to-monitor-cassandra-performance-metrics)
+- [Cassandra Documentation](http://docs.datastax.com/en/cassandra/3.x/cassandra/operations/monitoringCassandraTOC.html)
+
+####measurement = javaGarbageCollector
+
+- /java.lang:type=GarbageCollector,name=ConcurrentMarkSweep/CollectionTime       
+- /java.lang:type=GarbageCollector,name=ConcurrentMarkSweep/CollectionCount
+- /java.lang:type=GarbageCollector,name=ParNew/CollectionTime          
+- /java.lang:type=GarbageCollector,name=ParNew/CollectionCount
+
+####measurement = javaMemory
+
+- /java.lang:type=Memory/HeapMemoryUsage
+- /java.lang:type=Memory/NonHeapMemoryUsage
+
+####measurement = cassandraCache
+
+- /org.apache.cassandra.metrics:type=Cache,scope=KeyCache,name=Hit
+- /org.apache.cassandra.metrics:type=Cache,scope=KeyCache,name=Requests
+- /org.apache.cassandra.metrics:type=Cache,scope=KeyCache,name=Entries
+- /org.apache.cassandra.metrics:type=Cache,scope=KeyCache,name=Size 
+- /org.apache.cassandra.metrics:type=Cache,scope=KeyCache,name=Capacity 
+- /org.apache.cassandra.metrics:type=Cache,scope=RowCache,name=Hit
+- /org.apache.cassandra.metrics:type=Cache,scope=RowCache,name=Requests     
+- /org.apache.cassandra.metrics:type=Cache,scope=RowCache,name=Entries
+- /org.apache.cassandra.metrics:type=Cache,scope=RowCache,name=Size
+- /org.apache.cassandra.metrics:type=Cache,scope=RowCache,name=Capacity
+
+####measurement = cassandraClient
+
+- /org.apache.cassandra.metrics:type=Client,name=connectedNativeClients
+
+####measurement = cassandraClientRequest
+
+- /org.apache.cassandra.metrics:type=ClientRequest,scope=Read,name=TotalLatency        
+- /org.apache.cassandra.metrics:type=ClientRequest,scope=Write,name=TotalLatency         
+- /org.apache.cassandra.metrics:type=ClientRequest,scope=Read,name=Latency       
+- /org.apache.cassandra.metrics:type=ClientRequest,scope=Write,name=Latency        
+- /org.apache.cassandra.metrics:type=ClientRequest,scope=Read,name=Timeouts       
+- /org.apache.cassandra.metrics:type=ClientRequest,scope=Write,name=Timeouts
+- /org.apache.cassandra.metrics:type=ClientRequest,scope=Read,name=Unavailables        
+- /org.apache.cassandra.metrics:type=ClientRequest,scope=Write,name=Unavailables         
+- /org.apache.cassandra.metrics:type=ClientRequest,scope=Read,name=Failures         
+- /org.apache.cassandra.metrics:type=ClientRequest,scope=Write,name=Failures 
+
+####measurement = cassandraCommitLog
+
+- /org.apache.cassandra.metrics:type=CommitLog,name=PendingTasks        
+- /org.apache.cassandra.metrics:type=CommitLog,name=TotalCommitLogSize
+
+####measurement = cassandraCompaction
+
+- /org.apache.cassandra.metrics:type=Compaction,name=CompletedTask
+- /org.apache.cassandra.metrics:type=Compaction,name=PendingTasks    
+- /org.apache.cassandra.metrics:type=Compaction,name=TotalCompactionsCompleted
+- /org.apache.cassandra.metrics:type=Compaction,name=BytesCompacted
+
+####measurement = cassandraStorage
+
+- /org.apache.cassandra.metrics:type=Storage,name=Load
+-  /org.apache.cassandra.metrics:type=Storage,name=Exceptions 
+
+####measurement = cassandraTable
+Using wildcards for "keyspace" and "scope" can create a lot of series as metrics will be reported for every table and keyspace including internal system tables. Specify a keyspace name and/or a table name to limit them.
+
+- /org.apache.cassandra.metrics:type=Table,keyspace=\*,scope=\*,name=LiveDiskSpaceUsed
+- /org.apache.cassandra.metrics:type=Table,keyspace=\*,scope=\*,name=TotalDiskSpaceUsed
+- /org.apache.cassandra.metrics:type=Table,keyspace=\*,scope=\*,name=ReadLatency
+- /org.apache.cassandra.metrics:type=Table,keyspace=\*,scope=\*,name=CoordinatorReadLatency 
+- /org.apache.cassandra.metrics:type=Table,keyspace=\*,scope=\*,name=WriteLatency 
+- /org.apache.cassandra.metrics:type=Table,keyspace=\*,scope=\*,name=ReadTotalLatency 
+- /org.apache.cassandra.metrics:type=Table,keyspace=\*,scope=\*,name=WriteTotalLatency              
+
+
+####measurement = cassandraThreadPools
+
+- /org.apache.cassandra.metrics:type=ThreadPools,path=internal,scope=CompactionExecutor,name=ActiveTasks          
+- /org.apache.cassandra.metrics:type=ThreadPools,path=internal,scope=AntiEntropyStage,name=ActiveTasks         
+-  /org.apache.cassandra.metrics:type=ThreadPools,path=request,scope=CounterMutationStage,name=PendingTasks
+- /org.apache.cassandra.metrics:type=ThreadPools,path=request,scope=CounterMutationStage,name=CurrentlyBlockedTasks        
+- /org.apache.cassandra.metrics:type=ThreadPools,path=request,scope=MutationStage,name=PendingTasks
+- /org.apache.cassandra.metrics:type=ThreadPools,path=request,scope=MutationStage,name=CurrentlyBlockedTasks        
+- /org.apache.cassandra.metrics:type=ThreadPools,path=request,scope=ReadRepairStage,name=PendingTasks
+- /org.apache.cassandra.metrics:type=ThreadPools,path=request,scope=ReadRepairStage,name=CurrentlyBlockedTasks        
+- /org.apache.cassandra.metrics:type=ThreadPools,path=request,scope=ReadStage,name=PendingTasks
+- /org.apache.cassandra.metrics:type=ThreadPools,path=request,scope=ReadStage,name=CurrentlyBlockedTasks          
+-  /org.apache.cassandra.metrics:type=ThreadPools,path=request,scope=RequestResponseStage,name=PendingTasks                  
+- /org.apache.cassandra.metrics:type=ThreadPools,path=request,scope=RequestResponseStage,name=CurrentlyBlockedTasks 
+
+

plugins/inputs/cassandra/cassandra.go

@@ -0,0 +1,309 @@
+package cassandra
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"github.com/influxdata/telegraf"
+	"github.com/influxdata/telegraf/plugins/inputs"
+	"io/ioutil"
+	"log"
+	"net/http"
+	"net/url"
+	"strings"
+)
+
+type JolokiaClient interface {
+	MakeRequest(req *http.Request) (*http.Response, error)
+}
+
+type JolokiaClientImpl struct {
+	client *http.Client
+}
+
+func (c JolokiaClientImpl) MakeRequest(req *http.Request) (*http.Response, error) {
+	return c.client.Do(req)
+}
+
+type Cassandra struct {
+	jClient JolokiaClient
+	Context string
+	Servers []string
+	Metrics []string
+}
+
+type javaMetric struct {
+	host   string
+	metric string
+	acc    telegraf.Accumulator
+}
+
+type cassandraMetric struct {
+	host   string
+	metric string
+	acc    telegraf.Accumulator
+}
+
+type jmxMetric interface {
+	addTagsFields(out map[string]interface{})
+}
+
+func newJavaMetric(host string, metric string,
+	acc telegraf.Accumulator) *javaMetric {
+	return &javaMetric{host: host, metric: metric, acc: acc}
+}
+
+func newCassandraMetric(host string, metric string,
+	acc telegraf.Accumulator) *cassandraMetric {
+	return &cassandraMetric{host: host, metric: metric, acc: acc}
+}
+
+func addValuesAsFields(values map[string]interface{}, fields map[string]interface{},
+	mname string) {
+	for k, v := range values {
+		if v != nil {
+			fields[mname+"_"+k] = v
+		}
+	}
+}
+
+func parseJmxMetricRequest(mbean string) map[string]string {
+	tokens := make(map[string]string)
+	classAndPairs := strings.Split(mbean, ":")
+	if classAndPairs[0] == "org.apache.cassandra.metrics" {
+		tokens["class"] = "cassandra"
+	} else if classAndPairs[0] == "java.lang" {
+		tokens["class"] = "java"
+	} else {
+		return tokens
+	}
+	pairs := strings.Split(classAndPairs[1], ",")
+	for _, pair := range pairs {
+		p := strings.Split(pair, "=")
+		tokens[p[0]] = p[1]
+	}
+	return tokens
+}
+
+func addTokensToTags(tokens map[string]string, tags map[string]string) {
+	for k, v := range tokens {
+		if k == "name" {
+			tags["mname"] = v // name seems to a reserved word in influxdb
+		} else if k == "class" || k == "type" {
+			continue // class and type are used in the metric name
+		} else {
+			tags[k] = v
+		}
+	}
+}
+
+func (j javaMetric) addTagsFields(out map[string]interface{}) {
+	tags := make(map[string]string)
+	fields := make(map[string]interface{})
+
+	a := out["request"].(map[string]interface{})
+	attribute := a["attribute"].(string)
+	mbean := a["mbean"].(string)
+
+	tokens := parseJmxMetricRequest(mbean)
+	addTokensToTags(tokens, tags)
+	tags["cassandra_host"] = j.host
+
+	if _, ok := tags["mname"]; !ok {
+		//Queries for a single value will not return a "name" tag in the response.
+		tags["mname"] = attribute
+	}
+
+	if values, ok := out["value"]; ok {
+		switch t := values.(type) {
+		case map[string]interface{}:
+			addValuesAsFields(values.(map[string]interface{}), fields, attribute)
+		case interface{}:
+			fields[attribute] = t
+		}
+		j.acc.AddFields(tokens["class"]+tokens["type"], fields, tags)
+	} else {
+		fmt.Printf("Missing key 'value' in '%s' output response\n%v\n",
+			j.metric, out)
+	}
+}
+
+func addCassandraMetric(mbean string, c cassandraMetric,
+	values map[string]interface{}) {
+
+	tags := make(map[string]string)
+	fields := make(map[string]interface{})
+	tokens := parseJmxMetricRequest(mbean)
+	addTokensToTags(tokens, tags)
+	tags["cassandra_host"] = c.host
+	addValuesAsFields(values, fields, tags["mname"])
+	c.acc.AddFields(tokens["class"]+tokens["type"], fields, tags)
+
+}
+
+func (c cassandraMetric) addTagsFields(out map[string]interface{}) {
+
+	r := out["request"]
+
+	tokens := parseJmxMetricRequest(r.(map[string]interface{})["mbean"].(string))
+	// Requests with wildcards for keyspace or table names will return nested
+	// maps in the json response
+	if tokens["type"] == "Table" && (tokens["keyspace"] == "*" ||
+		tokens["scope"] == "*") {
+		if valuesMap, ok := out["value"]; ok {
+			for k, v := range valuesMap.(map[string]interface{}) {
+				addCassandraMetric(k, c, v.(map[string]interface{}))
+			}
+		} else {
+			fmt.Printf("Missing key 'value' in '%s' output response\n%v\n",
+				c.metric, out)
+			return
+		}
+	} else {
+		if values, ok := out["value"]; ok {
+			addCassandraMetric(r.(map[string]interface{})["mbean"].(string),
+				c, values.(map[string]interface{}))
+		} else {
+			fmt.Printf("Missing key 'value' in '%s' output response\n%v\n",
+				c.metric, out)
+			return
+		}
+	}
+}
+
+func (j *Cassandra) SampleConfig() string {
+	return `
+  # This is the context root used to compose the jolokia url
+  context = "/jolokia/read"
+  ## List of cassandra servers exposing jolokia read service
+  servers = ["myuser:mypassword@10.10.10.1:8778","10.10.10.2:8778",":8778"]
+  ## List of metrics collected on above servers
+  ## Each metric consists of a jmx path.
+  ## This will collect all heap memory usage metrics from the jvm and
+  ## ReadLatency metrics for all keyspaces and tables.
+  ## "type=Table" in the query works with Cassandra3.0. Older versions might
+  ## need to use "type=ColumnFamily"
+  metrics  = [
+    "/java.lang:type=Memory/HeapMemoryUsage",
+    "/org.apache.cassandra.metrics:type=Table,keyspace=*,scope=*,name=ReadLatency"
+  ]
+`
+}
+
+func (j *Cassandra) Description() string {
+	return "Read Cassandra metrics through Jolokia"
+}
+
+func (j *Cassandra) getAttr(requestUrl *url.URL) (map[string]interface{}, error) {
+	// Create + send request
+	req, err := http.NewRequest("GET", requestUrl.String(), nil)
+	if err != nil {
+		return nil, err
+	}
+
+	resp, err := j.jClient.MakeRequest(req)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	// Process response
+	if resp.StatusCode != http.StatusOK {
+		err = fmt.Errorf("Response from url \"%s\" has status code %d (%s), expected %d (%s)",
+			requestUrl,
+			resp.StatusCode,
+			http.StatusText(resp.StatusCode),
+			http.StatusOK,
+			http.StatusText(http.StatusOK))
+		return nil, err
+	}
+
+	// read body
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return nil, err
+	}
+
+	// Unmarshal json
+	var jsonOut map[string]interface{}
+	if err = json.Unmarshal([]byte(body), &jsonOut); err != nil {
+		return nil, errors.New("Error decoding JSON response")
+	}
+
+	return jsonOut, nil
+}
+
+func parseServerTokens(server string) map[string]string {
+	serverTokens := make(map[string]string)
+
+	hostAndUser := strings.Split(server, "@")
+	hostPort := ""
+	userPasswd := ""
+	if len(hostAndUser) == 2 {
+		hostPort = hostAndUser[1]
+		userPasswd = hostAndUser[0]
+	} else {
+		hostPort = hostAndUser[0]
+	}
+	hostTokens := strings.Split(hostPort, ":")
+	serverTokens["host"] = hostTokens[0]
+	serverTokens["port"] = hostTokens[1]
+
+	if userPasswd != "" {
+		userTokens := strings.Split(userPasswd, ":")
+		serverTokens["user"] = userTokens[0]
+		serverTokens["passwd"] = userTokens[1]
+	}
+	return serverTokens
+}
+
+func (c *Cassandra) Gather(acc telegraf.Accumulator) error {
+	context := c.Context
+	servers := c.Servers
+	metrics := c.Metrics
+
+	for _, server := range servers {
+		for _, metric := range metrics {
+			serverTokens := parseServerTokens(server)
+
+			var m jmxMetric
+			if strings.HasPrefix(metric, "/java.lang:") {
+				m = newJavaMetric(serverTokens["host"], metric, acc)
+			} else if strings.HasPrefix(metric,
+				"/org.apache.cassandra.metrics:") {
+				m = newCassandraMetric(serverTokens["host"], metric, acc)
+			} else {
+				// unsupported metric type
+				log.Printf("Unsupported Cassandra metric [%s], skipping",
+					metric)
+				continue
+			}
+
+			// Prepare URL
+			requestUrl, err := url.Parse("http://" + serverTokens["host"] + ":" +
+				serverTokens["port"] + context + metric)
+			if err != nil {
+				return err
+			}
+			if serverTokens["user"] != "" && serverTokens["passwd"] != "" {
+				requestUrl.User = url.UserPassword(serverTokens["user"],
+					serverTokens["passwd"])
+			}
+			fmt.Printf("host %s url %s\n", serverTokens["host"], requestUrl)
+
+			out, err := c.getAttr(requestUrl)
+			if out["status"] != 200.0 {
+				fmt.Printf("URL returned with status %v\n", out["status"])
+				continue
+			}
+			m.addTagsFields(out)
+		}
+	}
+	return nil
+}
+
+func init() {
+	inputs.Add("cassandra", func() telegraf.Input {
+		return &Cassandra{jClient: &JolokiaClientImpl{client: &http.Client{}}}
+	})
+}

plugins/inputs/cassandra/cassandra_test.go

@@ -0,0 +1,286 @@
+package cassandra
+
+import (
+	_ "fmt"
+	"io/ioutil"
+	"net/http"
+	"strings"
+	"testing"
+
+	"github.com/influxdata/telegraf/testutil"
+	"github.com/stretchr/testify/assert"
+	_ "github.com/stretchr/testify/require"
+)
+
+const validJavaMultiValueJSON = `
+{
+  "request":{
+    "mbean":"java.lang:type=Memory",
+    "attribute":"HeapMemoryUsage",
+    "type":"read"
+  },
+  "value":{
+    "init":67108864,
+    "committed":456130560,
+    "max":477626368,
+    "used":203288528
+  },
+  "timestamp":1446129191,
+  "status":200
+}`
+
+const validCassandraMultiValueJSON = `
+{
+	"request": {
+		"mbean": "org.apache.cassandra.metrics:keyspace=test_keyspace1,name=ReadLatency,scope=test_table,type=Table",
+		"type": "read"},
+	"status": 200,
+	"timestamp": 1458089229,
+	"value": {
+		"999thPercentile": 20.0,
+		"99thPercentile": 10.0,
+		"Count": 400,
+		"DurationUnit": "microseconds",
+		"Max": 30.0,
+		"Mean": null,
+		"MeanRate": 3.0,
+		"Min": 1.0,
+		"RateUnit": "events/second",
+		"StdDev": null
+	}
+}`
+
+const validCassandraNestedMultiValueJSON = `
+{
+	"request": {
+		"mbean": "org.apache.cassandra.metrics:keyspace=test_keyspace1,name=ReadLatency,scope=*,type=Table",
+		"type": "read"},
+    "status": 200,
+    "timestamp": 1458089184,
+    "value": {
+		"org.apache.cassandra.metrics:keyspace=test_keyspace1,name=ReadLatency,scope=test_table1,type=Table":
+		{  	"999thPercentile": 1.0,
+			"Count": 100,
+			"DurationUnit": "microseconds",
+			"OneMinuteRate": 1.0,
+			"RateUnit": "events/second",
+			"StdDev": null
+		},
+		"org.apache.cassandra.metrics:keyspace=test_keyspace2,name=ReadLatency,scope=test_table2,type=Table":
+		{  	"999thPercentile": 2.0,
+			"Count": 200,
+			"DurationUnit": "microseconds",
+			"OneMinuteRate": 2.0,
+			"RateUnit": "events/second",
+			"StdDev": null
+		}
+	}
+}`
+
+const validSingleValueJSON = `
+{
+  "request":{
+    "path":"used",
+    "mbean":"java.lang:type=Memory",
+    "attribute":"HeapMemoryUsage",
+    "type":"read"
+  },
+  "value":209274376,
+  "timestamp":1446129256,
+  "status":200
+}`
+
+const validJavaMultiTypeJSON = `
+{
+   "request":{
+	   "mbean":"java.lang:name=ConcurrentMarkSweep,type=GarbageCollector",
+	   "attribute":"CollectionCount",
+	   "type":"read"
+   },
+   "value":1,
+   "timestamp":1459316570,
+   "status":200
+}`
+
+const invalidJSON = "I don't think this is JSON"
+
+const empty = ""
+
+var Servers = []string{"10.10.10.10:8778"}
+var AuthServers = []string{"user:passwd@10.10.10.10:8778"}
+var MultipleServers = []string{"10.10.10.10:8778", "10.10.10.11:8778"}
+var HeapMetric = "/java.lang:type=Memory/HeapMemoryUsage"
+var ReadLatencyMetric = "/org.apache.cassandra.metrics:type=Table,keyspace=test_keyspace1,scope=test_table,name=ReadLatency"
+var NestedReadLatencyMetric = "/org.apache.cassandra.metrics:type=Table,keyspace=test_keyspace1,scope=*,name=ReadLatency"
+var GarbageCollectorMetric1 = "/java.lang:type=GarbageCollector,name=ConcurrentMarkSweep/CollectionCount"
+var GarbageCollectorMetric2 = "/java.lang:type=GarbageCollector,name=ConcurrentMarkSweep/CollectionTime"
+var Context = "/jolokia/read"
+
+type jolokiaClientStub struct {
+	responseBody string
+	statusCode   int
+}
+
+func (c jolokiaClientStub) MakeRequest(req *http.Request) (*http.Response, error) {
+	resp := http.Response{}
+	resp.StatusCode = c.statusCode
+	resp.Body = ioutil.NopCloser(strings.NewReader(c.responseBody))
+	return &resp, nil
+}
+
+// Generates a pointer to an HttpJson object that uses a mock HTTP client.
+// Parameters:
+//     response  : Body of the response that the mock HTTP client should return
+//     statusCode: HTTP status code the mock HTTP client should return
+//
+// Returns:
+//     *HttpJson: Pointer to an HttpJson object that uses the generated mock HTTP client
+func genJolokiaClientStub(response string, statusCode int, servers []string, metrics []string) *Cassandra {
+	return &Cassandra{
+		jClient: jolokiaClientStub{responseBody: response, statusCode: statusCode},
+		Context: Context,
+		Servers: servers,
+		Metrics: metrics,
+	}
+}
+
+// Test that the proper values are ignored or collected for class=Java
+func TestHttpJsonJavaMultiValue(t *testing.T) {
+	cassandra := genJolokiaClientStub(validJavaMultiValueJSON, 200,
+		MultipleServers, []string{HeapMetric})
+
+	var acc testutil.Accumulator
+	acc.SetDebug(true)
+	err := cassandra.Gather(&acc)
+
+	assert.Nil(t, err)
+	assert.Equal(t, 2, len(acc.Metrics))
+
+	fields := map[string]interface{}{
+		"HeapMemoryUsage_init":      67108864.0,
+		"HeapMemoryUsage_committed": 456130560.0,
+		"HeapMemoryUsage_max":       477626368.0,
+		"HeapMemoryUsage_used":      203288528.0,
+	}
+	tags1 := map[string]string{
+		"cassandra_host": "10.10.10.10",
+		"mname":          "HeapMemoryUsage",
+	}
+
+	tags2 := map[string]string{
+		"cassandra_host": "10.10.10.11",
+		"mname":          "HeapMemoryUsage",
+	}
+	acc.AssertContainsTaggedFields(t, "javaMemory", fields, tags1)
+	acc.AssertContainsTaggedFields(t, "javaMemory", fields, tags2)
+}
+
+func TestHttpJsonJavaMultiType(t *testing.T) {
+	cassandra := genJolokiaClientStub(validJavaMultiTypeJSON, 200, AuthServers, []string{GarbageCollectorMetric1, GarbageCollectorMetric2})
+
+	var acc testutil.Accumulator
+	acc.SetDebug(true)
+	err := cassandra.Gather(&acc)
+
+	assert.Nil(t, err)
+	assert.Equal(t, 2, len(acc.Metrics))
+
+	fields := map[string]interface{}{
+		"CollectionCount": 1.0,
+	}
+
+	tags := map[string]string{
+		"cassandra_host": "10.10.10.10",
+		"mname":          "ConcurrentMarkSweep",
+	}
+	acc.AssertContainsTaggedFields(t, "javaGarbageCollector", fields, tags)
+}
+
+// Test that the proper values are ignored or collected
+func TestHttpJsonOn404(t *testing.T) {
+
+	jolokia := genJolokiaClientStub(validJavaMultiValueJSON, 404, Servers,
+		[]string{HeapMetric})
+
+	var acc testutil.Accumulator
+	err := jolokia.Gather(&acc)
+
+	assert.Nil(t, err)
+	assert.Equal(t, 0, len(acc.Metrics))
+}
+
+// Test that the proper values are ignored or collected for class=Cassandra
+func TestHttpJsonCassandraMultiValue(t *testing.T) {
+	cassandra := genJolokiaClientStub(validCassandraMultiValueJSON, 200, Servers, []string{ReadLatencyMetric})
+
+	var acc testutil.Accumulator
+	err := cassandra.Gather(&acc)
+
+	assert.Nil(t, err)
+	assert.Equal(t, 1, len(acc.Metrics))
+
+	fields := map[string]interface{}{
+		"ReadLatency_999thPercentile": 20.0,
+		"ReadLatency_99thPercentile":  10.0,
+		"ReadLatency_Count":           400.0,
+		"ReadLatency_DurationUnit":    "microseconds",
+		"ReadLatency_Max":             30.0,
+		"ReadLatency_MeanRate":        3.0,
+		"ReadLatency_Min":             1.0,
+		"ReadLatency_RateUnit":        "events/second",
+	}
+
+	tags := map[string]string{
+		"cassandra_host": "10.10.10.10",
+		"mname":          "ReadLatency",
+		"keyspace":       "test_keyspace1",
+		"scope":          "test_table",
+	}
+	acc.AssertContainsTaggedFields(t, "cassandraTable", fields, tags)
+}
+
+// Test that the proper values are ignored or collected for class=Cassandra with
+// nested values
+func TestHttpJsonCassandraNestedMultiValue(t *testing.T) {
+	cassandra := genJolokiaClientStub(validCassandraNestedMultiValueJSON, 200, Servers, []string{NestedReadLatencyMetric})
+
+	var acc testutil.Accumulator
+	acc.SetDebug(true)
+	err := cassandra.Gather(&acc)
+
+	assert.Nil(t, err)
+	assert.Equal(t, 2, len(acc.Metrics))
+
+	fields1 := map[string]interface{}{
+		"ReadLatency_999thPercentile": 1.0,
+		"ReadLatency_Count":           100.0,
+		"ReadLatency_DurationUnit":    "microseconds",
+		"ReadLatency_OneMinuteRate":   1.0,
+		"ReadLatency_RateUnit":        "events/second",
+	}
+
+	fields2 := map[string]interface{}{
+		"ReadLatency_999thPercentile": 2.0,
+		"ReadLatency_Count":           200.0,
+		"ReadLatency_DurationUnit":    "microseconds",
+		"ReadLatency_OneMinuteRate":   2.0,
+		"ReadLatency_RateUnit":        "events/second",
+	}
+
+	tags1 := map[string]string{
+		"cassandra_host": "10.10.10.10",
+		"mname":          "ReadLatency",
+		"keyspace":       "test_keyspace1",
+		"scope":          "test_table1",
+	}
+
+	tags2 := map[string]string{
+		"cassandra_host": "10.10.10.10",
+		"mname":          "ReadLatency",
+		"keyspace":       "test_keyspace2",
+		"scope":          "test_table2",
+	}
+
+	acc.AssertContainsTaggedFields(t, "cassandraTable", fields1, tags1)
+	acc.AssertContainsTaggedFields(t, "cassandraTable", fields2, tags2)
+}

plugins/inputs/ceph/README.md

@@ -0,0 +1,109 @@
+# Ceph Storage Input Plugin
+
+Collects performance metrics from the MON and OSD nodes in a Ceph storage cluster.  
+
+The plugin works by scanning the configured SocketDir for OSD and MON socket files.  When it finds
+a MON socket, it runs **ceph --admin-daemon $file perfcounters_dump**. For OSDs it runs **ceph --admin-daemon $file perf dump** 
+
+The resulting JSON is parsed and grouped into collections, based on top-level key.  Top-level keys are
+used as collection tags, and all sub-keys are flattened. For example:
+
+```
+ { 
+   "paxos": { 
+     "refresh": 9363435,
+     "refresh_latency": { 
+       "avgcount": 9363435,
+       "sum": 5378.794002000
+     }
+   }
+ }
+```
+
+Would be parsed into the following metrics, all of which would be tagged with collection=paxos:
+
+ - refresh = 9363435
+ - refresh_latency.avgcount: 9363435
+ - refresh_latency.sum: 5378.794002000
+
+
+### Configuration:
+
+```
+# Collects performance metrics from the MON and OSD nodes in a Ceph storage cluster.
+[[inputs.ceph]]
+  ## All configuration values are optional, defaults are shown below
+
+  ## location of ceph binary
+  ceph_binary = "/usr/bin/ceph"
+
+  ## directory in which to look for socket files
+  socket_dir = "/var/run/ceph"
+
+  ## prefix of MON and OSD socket files, used to determine socket type
+  mon_prefix = "ceph-mon"
+  osd_prefix = "ceph-osd"
+
+  ## suffix used to identify socket files
+  socket_suffix = "asok"
+```
+
+### Measurements & Fields:
+
+All fields are collected under the **ceph** measurement and stored as float64s. For a full list of fields, see the sample perf dumps in ceph_test.go. 
+
+
+### Tags:
+
+All measurements will have the following tags:
+
+- type: either 'osd' or 'mon' to indicate which type of node was queried
+- id: a unique string identifier, parsed from the socket file name for the node
+- collection: the top-level key under which these fields were reported. Possible values are:
+  - for MON nodes:
+    - cluster
+    - leveldb
+    - mon
+    - paxos
+    - throttle-mon_client_bytes
+    - throttle-mon_daemon_bytes
+    - throttle-msgr_dispatch_throttler-mon
+  - for OSD nodes:
+    - WBThrottle
+    - filestore
+    - leveldb
+    - mutex-FileJournal::completions_lock
+    - mutex-FileJournal::finisher_lock
+    - mutex-FileJournal::write_lock
+    - mutex-FileJournal::writeq_lock
+    - mutex-JOS::ApplyManager::apply_lock
+    - mutex-JOS::ApplyManager::com_lock
+    - mutex-JOS::SubmitManager::lock
+    - mutex-WBThrottle::lock
+    - objecter
+    - osd
+    - recoverystate_perf
+    - throttle-filestore_bytes
+    - throttle-filestore_ops
+    - throttle-msgr_dispatch_throttler-client
+    - throttle-msgr_dispatch_throttler-cluster
+    - throttle-msgr_dispatch_throttler-hb_back_server
+    - throttle-msgr_dispatch_throttler-hb_front_serve
+    - throttle-msgr_dispatch_throttler-hbclient
+    - throttle-msgr_dispatch_throttler-ms_objecter
+    - throttle-objecter_bytes
+    - throttle-objecter_ops
+    - throttle-osd_client_bytes
+    - throttle-osd_client_messages
+ 
+
+### Example Output:
+
+<pre>
+telegraf -test -config /etc/telegraf/telegraf.conf -config-directory /etc/telegraf/telegraf.d  -input-filter ceph
+* Plugin: ceph, Collection 1
+> ceph,collection=paxos, id=node-2,role=openstack,type=mon accept_timeout=0,begin=14931264,begin_bytes.avgcount=14931264,begin_bytes.sum=180309683362,begin_keys.avgcount=0,begin_keys.sum=0,begin_latency.avgcount=14931264,begin_latency.sum=9293.29589,collect=1,collect_bytes.avgcount=1,collect_bytes.sum=24,collect_keys.avgcount=1,collect_keys.sum=1,collect_latency.avgcount=1,collect_latency.sum=0.00028,collect_timeout=0,collect_uncommitted=0,commit=14931264,commit_bytes.avgcount=0,commit_bytes.sum=0,commit_keys.avgcount=0,commit_keys.sum=0,commit_latency.avgcount=0,commit_latency.sum=0,lease_ack_timeout=0,lease_timeout=0,new_pn=0,new_pn_latency.avgcount=0,new_pn_latency.sum=0,refresh=14931264,refresh_latency.avgcount=14931264,refresh_latency.sum=8706.98498,restart=4,share_state=0,share_state_bytes.avgcount=0,share_state_bytes.sum=0,share_state_keys.avgcount=0,share_state_keys.sum=0,start_leader=0,start_peon=1,store_state=14931264,store_state_bytes.avgcount=14931264,store_state_bytes.sum=353119959211,store_state_keys.avgcount=14931264,store_state_keys.sum=289807523,store_state_latency.avgcount=14931264,store_state_latency.sum=10952.835724 1462821234814535148
+> ceph,collection=throttle-mon_client_bytes,id=node-2,type=mon get=1413017,get_or_fail_fail=0,get_or_fail_success=0,get_sum=71211705,max=104857600,put=1413013,put_sum=71211459,take=0,take_sum=0,val=246,wait.avgcount=0,wait.sum=0 1462821234814737219
+> ceph,collection=throttle-mon_daemon_bytes,id=node-2,type=mon get=4058121,get_or_fail_fail=0,get_or_fail_success=0,get_sum=6027348117,max=419430400,put=4058121,put_sum=6027348117,take=0,take_sum=0,val=0,wait.avgcount=0,wait.sum=0 1462821234814815661
+> ceph,collection=throttle-msgr_dispatch_throttler-mon,id=node-2,type=mon get=54276277,get_or_fail_fail=0,get_or_fail_success=0,get_sum=370232877040,max=104857600,put=54276277,put_sum=370232877040,take=0,take_sum=0,val=0,wait.avgcount=0,wait.sum=0 1462821234814872064
+</pre>

plugins/inputs/ceph/ceph.go

@@ -0,0 +1,249 @@
+package ceph
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"github.com/influxdata/telegraf"
+	"github.com/influxdata/telegraf/plugins/inputs"
+	"io/ioutil"
+	"log"
+	"os/exec"
+	"path/filepath"
+	"strings"
+)
+
+const (
+	measurement = "ceph"
+	typeMon     = "monitor"
+	typeOsd     = "osd"
+	osdPrefix   = "ceph-osd"
+	monPrefix   = "ceph-mon"
+	sockSuffix  = "asok"
+)
+
+type Ceph struct {
+	CephBinary   string
+	OsdPrefix    string
+	MonPrefix    string
+	SocketDir    string
+	SocketSuffix string
+}
+
+func (c *Ceph) setDefaults() {
+	if c.CephBinary == "" {
+		c.CephBinary = "/usr/bin/ceph"
+	}
+
+	if c.OsdPrefix == "" {
+		c.OsdPrefix = osdPrefix
+	}
+
+	if c.MonPrefix == "" {
+		c.MonPrefix = monPrefix
+	}
+
+	if c.SocketDir == "" {
+		c.SocketDir = "/var/run/ceph"
+	}
+
+	if c.SocketSuffix == "" {
+		c.SocketSuffix = sockSuffix
+	}
+}
+
+func (c *Ceph) Description() string {
+	return "Collects performance metrics from the MON and OSD nodes in a Ceph storage cluster."
+}
+
+var sampleConfig = `
+  ## All configuration values are optional, defaults are shown below
+
+  ## location of ceph binary
+  ceph_binary = "/usr/bin/ceph"
+
+  ## directory in which to look for socket files
+  socket_dir = "/var/run/ceph"
+
+  ## prefix of MON and OSD socket files, used to determine socket type
+  mon_prefix = "ceph-mon"
+  osd_prefix = "ceph-osd"
+
+  ## suffix used to identify socket files
+  socket_suffix = "asok"
+`
+
+func (c *Ceph) SampleConfig() string {
+	return sampleConfig
+}
+
+func (c *Ceph) Gather(acc telegraf.Accumulator) error {
+	c.setDefaults()
+	sockets, err := findSockets(c)
+	if err != nil {
+		return fmt.Errorf("failed to find sockets at path '%s': %v", c.SocketDir, err)
+	}
+
+	for _, s := range sockets {
+		dump, err := perfDump(c.CephBinary, s)
+		if err != nil {
+			log.Printf("error reading from socket '%s': %v", s.socket, err)
+			continue
+		}
+		data, err := parseDump(dump)
+		if err != nil {
+			log.Printf("error parsing dump from socket '%s': %v", s.socket, err)
+			continue
+		}
+		for tag, metrics := range *data {
+			acc.AddFields(measurement,
+				map[string]interface{}(metrics),
+				map[string]string{"type": s.sockType, "id": s.sockId, "collection": tag})
+		}
+	}
+	return nil
+}
+
+func init() {
+	inputs.Add(measurement, func() telegraf.Input { return &Ceph{} })
+}
+
+var perfDump = func(binary string, socket *socket) (string, error) {
+	cmdArgs := []string{"--admin-daemon", socket.socket}
+	if socket.sockType == typeOsd {
+		cmdArgs = append(cmdArgs, "perf", "dump")
+	} else if socket.sockType == typeMon {
+		cmdArgs = append(cmdArgs, "perfcounters_dump")
+	} else {
+		return "", fmt.Errorf("ignoring unknown socket type: %s", socket.sockType)
+	}
+
+	cmd := exec.Command(binary, cmdArgs...)
+	var out bytes.Buffer
+	cmd.Stdout = &out
+	err := cmd.Run()
+	if err != nil {
+		return "", fmt.Errorf("error running ceph dump: %s", err)
+	}
+
+	return out.String(), nil
+}
+
+var findSockets = func(c *Ceph) ([]*socket, error) {
+	listing, err := ioutil.ReadDir(c.SocketDir)
+	if err != nil {
+		return []*socket{}, fmt.Errorf("Failed to read socket directory '%s': %v", c.SocketDir, err)
+	}
+	sockets := make([]*socket, 0, len(listing))
+	for _, info := range listing {
+		f := info.Name()
+		var sockType string
+		var sockPrefix string
+		if strings.HasPrefix(f, c.MonPrefix) {
+			sockType = typeMon
+			sockPrefix = monPrefix
+		}
+		if strings.HasPrefix(f, c.OsdPrefix) {
+			sockType = typeOsd
+			sockPrefix = osdPrefix
+
+		}
+		if sockType == typeOsd || sockType == typeMon {
+			path := filepath.Join(c.SocketDir, f)
+			sockets = append(sockets, &socket{parseSockId(f, sockPrefix, c.SocketSuffix), sockType, path})
+		}
+	}
+	return sockets, nil
+}
+
+func parseSockId(fname, prefix, suffix string) string {
+	s := fname
+	s = strings.TrimPrefix(s, prefix)
+	s = strings.TrimSuffix(s, suffix)
+	s = strings.Trim(s, ".-_")
+	return s
+}
+
+type socket struct {
+	sockId   string
+	sockType string
+	socket   string
+}
+
+type metric struct {
+	pathStack []string // lifo stack of name components
+	value     float64
+}
+
+// Pops names of pathStack to build the flattened name for a metric
+func (m *metric) name() string {
+	buf := bytes.Buffer{}
+	for i := len(m.pathStack) - 1; i >= 0; i-- {
+		if buf.Len() > 0 {
+			buf.WriteString(".")
+		}
+		buf.WriteString(m.pathStack[i])
+	}
+	return buf.String()
+}
+
+type metricMap map[string]interface{}
+
+type taggedMetricMap map[string]metricMap
+
+// Parses a raw JSON string into a taggedMetricMap
+// Delegates the actual parsing to newTaggedMetricMap(..)
+func parseDump(dump string) (*taggedMetricMap, error) {
+	data := make(map[string]interface{})
+	err := json.Unmarshal([]byte(dump), &data)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse json: '%s': %v", dump, err)
+	}
+
+	tmm := newTaggedMetricMap(data)
+
+	if err != nil {
+		return nil, fmt.Errorf("failed to tag dataset: '%v': %v", tmm, err)
+	}
+
+	return tmm, nil
+}
+
+// Builds a TaggedMetricMap out of a generic string map.
+// The top-level key is used as a tag and all sub-keys are flattened into metrics
+func newTaggedMetricMap(data map[string]interface{}) *taggedMetricMap {
+	tmm := make(taggedMetricMap)
+	for tag, datapoints := range data {
+		mm := make(metricMap)
+		for _, m := range flatten(datapoints) {
+			mm[m.name()] = m.value
+		}
+		tmm[tag] = mm
+	}
+	return &tmm
+}
+
+// Recursively flattens any k-v hierarchy present in data.
+// Nested keys are flattened into ordered slices associated with a metric value.
+// The key slices are treated as stacks, and are expected to be reversed and concatenated
+// when passed as metrics to the accumulator. (see (*metric).name())
+func flatten(data interface{}) []*metric {
+	var metrics []*metric
+
+	switch val := data.(type) {
+	case float64:
+		metrics = []*metric{&metric{make([]string, 0, 1), val}}
+	case map[string]interface{}:
+		metrics = make([]*metric, 0, len(val))
+		for k, v := range val {
+			for _, m := range flatten(v) {
+				m.pathStack = append(m.pathStack, k)
+				metrics = append(metrics, m)
+			}
+		}
+	default:
+		log.Printf("Ignoring unexpected type '%T' for value %v", val, val)
+	}
+
+	return metrics
+}

plugins/inputs/ceph/ceph_test.go

@@ -0,0 +1,682 @@
+package ceph
+
+import (
+	"fmt"
+	"github.com/influxdata/telegraf/testutil"
+	"github.com/stretchr/testify/assert"
+	"io/ioutil"
+	"os"
+	"path"
+	"strconv"
+	"strings"
+	"testing"
+)
+
+const (
+	epsilon = float64(0.00000001)
+)
+
+func TestParseSockId(t *testing.T) {
+	s := parseSockId(sockFile(osdPrefix, 1), osdPrefix, sockSuffix)
+	assert.Equal(t, s, "1")
+}
+
+func TestParseMonDump(t *testing.T) {
+	dump, err := parseDump(monPerfDump)
+	assert.NoError(t, err)
+	assert.InEpsilon(t, 5678670180, (*dump)["cluster"]["osd_kb_used"], epsilon)
+	assert.InEpsilon(t, 6866.540527000, (*dump)["paxos"]["store_state_latency.sum"], epsilon)
+}
+
+func TestParseOsdDump(t *testing.T) {
+	dump, err := parseDump(osdPerfDump)
+	assert.NoError(t, err)
+	assert.InEpsilon(t, 552132.109360000, (*dump)["filestore"]["commitcycle_interval.sum"], epsilon)
+	assert.Equal(t, float64(0), (*dump)["mutex-FileJournal::finisher_lock"]["wait.avgcount"])
+}
+
+func TestGather(t *testing.T) {
+	saveFind := findSockets
+	saveDump := perfDump
+	defer func() {
+		findSockets = saveFind
+		perfDump = saveDump
+	}()
+
+	findSockets = func(c *Ceph) ([]*socket, error) {
+		return []*socket{&socket{"osd.1", typeOsd, ""}}, nil
+	}
+
+	perfDump = func(binary string, s *socket) (string, error) {
+		return osdPerfDump, nil
+	}
+
+	acc := &testutil.Accumulator{}
+	c := &Ceph{}
+	c.Gather(acc)
+
+}
+
+func TestFindSockets(t *testing.T) {
+	tmpdir, err := ioutil.TempDir("", "socktest")
+	assert.NoError(t, err)
+	defer func() {
+		err := os.Remove(tmpdir)
+		assert.NoError(t, err)
+	}()
+	c := &Ceph{
+		CephBinary: "foo",
+		SocketDir:  tmpdir,
+	}
+
+	c.setDefaults()
+
+	for _, st := range sockTestParams {
+		createTestFiles(tmpdir, st)
+
+		sockets, err := findSockets(c)
+		assert.NoError(t, err)
+
+		for i := 1; i <= st.osds; i++ {
+			assertFoundSocket(t, tmpdir, typeOsd, i, sockets)
+		}
+
+		for i := 1; i <= st.mons; i++ {
+			assertFoundSocket(t, tmpdir, typeMon, i, sockets)
+		}
+		cleanupTestFiles(tmpdir, st)
+	}
+}
+
+func assertFoundSocket(t *testing.T, dir, sockType string, i int, sockets []*socket) {
+	var prefix string
+	if sockType == typeOsd {
+		prefix = osdPrefix
+	} else {
+		prefix = monPrefix
+	}
+	expected := path.Join(dir, sockFile(prefix, i))
+	found := false
+	for _, s := range sockets {
+		fmt.Printf("Checking %s\n", s.socket)
+		if s.socket == expected {
+			found = true
+			assert.Equal(t, s.sockType, sockType, "Unexpected socket type for '%s'", s)
+			assert.Equal(t, s.sockId, strconv.Itoa(i))
+		}
+	}
+	assert.True(t, found, "Did not find socket: %s", expected)
+}
+
+func sockFile(prefix string, i int) string {
+	return strings.Join([]string{prefix, strconv.Itoa(i), sockSuffix}, ".")
+}
+
+func createTestFiles(dir string, st *SockTest) {
+	writeFile := func(prefix string, i int) {
+		f := sockFile(prefix, i)
+		fpath := path.Join(dir, f)
+		ioutil.WriteFile(fpath, []byte(""), 0777)
+	}
+	tstFileApply(st, writeFile)
+}
+
+func cleanupTestFiles(dir string, st *SockTest) {
+	rmFile := func(prefix string, i int) {
+		f := sockFile(prefix, i)
+		fpath := path.Join(dir, f)
+		err := os.Remove(fpath)
+		if err != nil {
+			fmt.Printf("Error removing test file %s: %v\n", fpath, err)
+		}
+	}
+	tstFileApply(st, rmFile)
+}
+
+func tstFileApply(st *SockTest, fn func(prefix string, i int)) {
+	for i := 1; i <= st.osds; i++ {
+		fn(osdPrefix, i)
+	}
+	for i := 1; i <= st.mons; i++ {
+		fn(monPrefix, i)
+	}
+}
+
+type SockTest struct {
+	osds int
+	mons int
+}
+
+var sockTestParams = []*SockTest{
+	&SockTest{
+		osds: 2,
+		mons: 2,
+	},
+	&SockTest{
+		mons: 1,
+	},
+	&SockTest{
+		osds: 1,
+	},
+	&SockTest{},
+}
+
+var monPerfDump = `
+{ "cluster": { "num_mon": 2,
+      "num_mon_quorum": 2,
+      "num_osd": 26,
+      "num_osd_up": 26,
+      "num_osd_in": 26,
+      "osd_epoch": 3306,
+      "osd_kb": 11487846448,
+      "osd_kb_used": 5678670180,
+      "osd_kb_avail": 5809176268,
+      "num_pool": 12,
+      "num_pg": 768,
+      "num_pg_active_clean": 768,
+      "num_pg_active": 768,
+      "num_pg_peering": 0,
+      "num_object": 397616,
+      "num_object_degraded": 0,
+      "num_object_unfound": 0,
+      "num_bytes": 2917848227467,
+      "num_mds_up": 0,
+      "num_mds_in": 0,
+      "num_mds_failed": 0,
+      "mds_epoch": 1},
+  "leveldb": { "leveldb_get": 321950312,
+      "leveldb_transaction": 18729922,
+      "leveldb_compact": 0,
+      "leveldb_compact_range": 74141,
+      "leveldb_compact_queue_merge": 0,
+      "leveldb_compact_queue_len": 0},
+  "mon": {},
+  "paxos": { "start_leader": 0,
+      "start_peon": 1,
+      "restart": 4,
+      "refresh": 9363435,
+      "refresh_latency": { "avgcount": 9363435,
+          "sum": 5378.794002000},
+      "begin": 9363435,
+      "begin_keys": { "avgcount": 0,
+          "sum": 0},
+      "begin_bytes": { "avgcount": 9363435,
+          "sum": 110468605489},
+      "begin_latency": { "avgcount": 9363435,
+          "sum": 5850.060682000},
+      "commit": 9363435,
+      "commit_keys": { "avgcount": 0,
+          "sum": 0},
+      "commit_bytes": { "avgcount": 0,
+          "sum": 0},
+      "commit_latency": { "avgcount": 0,
+          "sum": 0.000000000},
+      "collect": 1,
+      "collect_keys": { "avgcount": 1,
+          "sum": 1},
+      "collect_bytes": { "avgcount": 1,
+          "sum": 24},
+      "collect_latency": { "avgcount": 1,
+          "sum": 0.000280000},
+      "collect_uncommitted": 0,
+      "collect_timeout": 0,
+      "accept_timeout": 0,
+      "lease_ack_timeout": 0,
+      "lease_timeout": 0,
+      "store_state": 9363435,
+      "store_state_keys": { "avgcount": 9363435,
+          "sum": 176572789},
+      "store_state_bytes": { "avgcount": 9363435,
+          "sum": 216355887217},
+      "store_state_latency": { "avgcount": 9363435,
+          "sum": 6866.540527000},
+      "share_state": 0,
+      "share_state_keys": { "avgcount": 0,
+          "sum": 0},
+      "share_state_bytes": { "avgcount": 0,
+          "sum": 0},
+      "new_pn": 0,
+      "new_pn_latency": { "avgcount": 0,
+          "sum": 0.000000000}},
+  "throttle-mon_client_bytes": { "val": 246,
+      "max": 104857600,
+      "get": 896030,
+      "get_sum": 45854374,
+      "get_or_fail_fail": 0,
+      "get_or_fail_success": 0,
+      "take": 0,
+      "take_sum": 0,
+      "put": 896026,
+      "put_sum": 45854128,
+      "wait": { "avgcount": 0,
+          "sum": 0.000000000}},
+  "throttle-mon_daemon_bytes": { "val": 0,
+      "max": 419430400,
+      "get": 2773768,
+      "get_sum": 3627676976,
+      "get_or_fail_fail": 0,
+      "get_or_fail_success": 0,
+      "take": 0,
+      "take_sum": 0,
+      "put": 2773768,
+      "put_sum": 3627676976,
+      "wait": { "avgcount": 0,
+          "sum": 0.000000000}},
+  "throttle-msgr_dispatch_throttler-mon": { "val": 0,
+      "max": 104857600,
+      "get": 34504949,
+      "get_sum": 226860281124,
+      "get_or_fail_fail": 0,
+      "get_or_fail_success": 0,
+      "take": 0,
+      "take_sum": 0,
+      "put": 34504949,
+      "put_sum": 226860281124,
+      "wait": { "avgcount": 0,
+          "sum": 0.000000000}}}
+`
+
+var osdPerfDump = `
+{ "WBThrottle": { "bytes_dirtied": 28405539,
+      "bytes_wb": 0,
+      "ios_dirtied": 93,
+      "ios_wb": 0,
+      "inodes_dirtied": 86,
+      "inodes_wb": 0},
+  "filestore": { "journal_queue_max_ops": 0,
+      "journal_queue_ops": 0,
+      "journal_ops": 1108008,
+      "journal_queue_max_bytes": 0,
+      "journal_queue_bytes": 0,
+      "journal_bytes": 73233416196,
+      "journal_latency": { "avgcount": 1108008,
+          "sum": 290.981036000},
+      "journal_wr": 1091866,
+      "journal_wr_bytes": { "avgcount": 1091866,
+          "sum": 74925682688},
+      "journal_full": 0,
+      "committing": 0,
+      "commitcycle": 110389,
+      "commitcycle_interval": { "avgcount": 110389,
+          "sum": 552132.109360000},
+      "commitcycle_latency": { "avgcount": 110389,
+          "sum": 178.657804000},
+      "op_queue_max_ops": 50,
+      "op_queue_ops": 0,
+      "ops": 1108008,
+      "op_queue_max_bytes": 104857600,
+      "op_queue_bytes": 0,
+      "bytes": 73226768148,
+      "apply_latency": { "avgcount": 1108008,
+          "sum": 947.742722000},
+      "queue_transaction_latency_avg": { "avgcount": 1108008,
+          "sum": 0.511327000}},
+  "leveldb": { "leveldb_get": 4361221,
+      "leveldb_transaction": 4351276,
+      "leveldb_compact": 0,
+      "leveldb_compact_range": 0,
+      "leveldb_compact_queue_merge": 0,
+      "leveldb_compact_queue_len": 0},
+  "mutex-FileJournal::completions_lock": { "wait": { "avgcount": 0,
+          "sum": 0.000000000}},
+  "mutex-FileJournal::finisher_lock": { "wait": { "avgcount": 0,
+          "sum": 0.000000000}},
+  "mutex-FileJournal::write_lock": { "wait": { "avgcount": 0,
+          "sum": 0.000000000}},
+  "mutex-FileJournal::writeq_lock": { "wait": { "avgcount": 0,
+          "sum": 0.000000000}},
+  "mutex-JOS::ApplyManager::apply_lock": { "wait": { "avgcount": 0,
+          "sum": 0.000000000}},
+  "mutex-JOS::ApplyManager::com_lock": { "wait": { "avgcount": 0,
+          "sum": 0.000000000}},
+  "mutex-JOS::SubmitManager::lock": { "wait": { "avgcount": 0,
+          "sum": 0.000000000}},
+  "mutex-WBThrottle::lock": { "wait": { "avgcount": 0,
+          "sum": 0.000000000}},
+  "objecter": { "op_active": 0,
+      "op_laggy": 0,
+      "op_send": 0,
+      "op_send_bytes": 0,
+      "op_resend": 0,
+      "op_ack": 0,
+      "op_commit": 0,
+      "op": 0,
+      "op_r": 0,
+      "op_w": 0,
+      "op_rmw": 0,
+      "op_pg": 0,
+      "osdop_stat": 0,
+      "osdop_create": 0,
+      "osdop_read": 0,
+      "osdop_write": 0,
+      "osdop_writefull": 0,
+      "osdop_append": 0,
+      "osdop_zero": 0,
+      "osdop_truncate": 0,
+      "osdop_delete": 0,
+      "osdop_mapext": 0,
+      "osdop_sparse_read": 0,
+      "osdop_clonerange": 0,
+      "osdop_getxattr": 0,
+      "osdop_setxattr": 0,
+      "osdop_cmpxattr": 0,
+      "osdop_rmxattr": 0,
+      "osdop_resetxattrs": 0,
+      "osdop_tmap_up": 0,
+      "osdop_tmap_put": 0,
+      "osdop_tmap_get": 0,
+      "osdop_call": 0,
+      "osdop_watch": 0,
+      "osdop_notify": 0,
+      "osdop_src_cmpxattr": 0,
+      "osdop_pgls": 0,
+      "osdop_pgls_filter": 0,
+      "osdop_other": 0,
+      "linger_active": 0,
+      "linger_send": 0,
+      "linger_resend": 0,
+      "poolop_active": 0,
+      "poolop_send": 0,
+      "poolop_resend": 0,
+      "poolstat_active": 0,
+      "poolstat_send": 0,
+      "poolstat_resend": 0,
+      "statfs_active": 0,
+      "statfs_send": 0,
+      "statfs_resend": 0,
+      "command_active": 0,
+      "command_send": 0,
+      "command_resend": 0,
+      "map_epoch": 3300,
+      "map_full": 0,
+      "map_inc": 3293,
+      "osd_sessions": 0,
+      "osd_session_open": 0,
+      "osd_session_close": 0,
+      "osd_laggy": 0},
+  "osd": { "opq": 0,
+      "op_wip": 0,
+      "op": 23939,
+      "op_in_bytes": 1245903961,
+      "op_out_bytes": 29103083856,
+      "op_latency": { "avgcount": 23939,
+          "sum": 440.192015000},
+      "op_process_latency": { "avgcount": 23939,
+          "sum": 30.170685000},
+      "op_r": 23112,
+      "op_r_out_bytes": 29103056146,
+      "op_r_latency": { "avgcount": 23112,
+          "sum": 19.373526000},
+      "op_r_process_latency": { "avgcount": 23112,
+          "sum": 14.625928000},
+      "op_w": 549,
+      "op_w_in_bytes": 1245804358,
+      "op_w_rlat": { "avgcount": 549,
+          "sum": 17.022299000},
+      "op_w_latency": { "avgcount": 549,
+          "sum": 418.494610000},
+      "op_w_process_latency": { "avgcount": 549,
+          "sum": 13.316555000},
+      "op_rw": 278,
+      "op_rw_in_bytes": 99603,
+      "op_rw_out_bytes": 27710,
+      "op_rw_rlat": { "avgcount": 278,
+          "sum": 2.213785000},
+      "op_rw_latency": { "avgcount": 278,
+          "sum": 2.323879000},
+      "op_rw_process_latency": { "avgcount": 278,
+          "sum": 2.228202000},
+      "subop": 1074774,
+      "subop_in_bytes": 26841811636,
+      "subop_latency": { "avgcount": 1074774,
+          "sum": 745.509160000},
+      "subop_w": 0,
+      "subop_w_in_bytes": 26841811636,
+      "subop_w_latency": { "avgcount": 1074774,
+          "sum": 745.509160000},
+      "subop_pull": 0,
+      "subop_pull_latency": { "avgcount": 0,
+          "sum": 0.000000000},
+      "subop_push": 0,
+      "subop_push_in_bytes": 0,
+      "subop_push_latency": { "avgcount": 0,
+          "sum": 0.000000000},
+      "pull": 0,
+      "push": 28,
+      "push_out_bytes": 103483392,
+      "push_in": 0,
+      "push_in_bytes": 0,
+      "recovery_ops": 15,
+      "loadavg": 202,
+      "buffer_bytes": 0,
+      "numpg": 18,
+      "numpg_primary": 8,
+      "numpg_replica": 10,
+      "numpg_stray": 0,
+      "heartbeat_to_peers": 10,
+      "heartbeat_from_peers": 0,
+      "map_messages": 7413,
+      "map_message_epochs": 9792,
+      "map_message_epoch_dups": 10105,
+      "messages_delayed_for_map": 83,
+      "stat_bytes": 102123175936,
+      "stat_bytes_used": 49961820160,
+      "stat_bytes_avail": 52161355776,
+      "copyfrom": 0,
+      "tier_promote": 0,
+      "tier_flush": 0,
+      "tier_flush_fail": 0,
+      "tier_try_flush": 0,
+      "tier_try_flush_fail": 0,
+      "tier_evict": 0,
+      "tier_whiteout": 0,
+      "tier_dirty": 230,
+      "tier_clean": 0,
+      "tier_delay": 0,
+      "agent_wake": 0,
+      "agent_skip": 0,
+      "agent_flush": 0,
+      "agent_evict": 0},
+  "recoverystate_perf": { "initial_latency": { "avgcount": 473,
+          "sum": 0.027207000},
+      "started_latency": { "avgcount": 1480,
+          "sum": 9854902.397648000},
+      "reset_latency": { "avgcount": 1953,
+          "sum": 0.096206000},
+      "start_latency": { "avgcount": 1953,
+          "sum": 0.059947000},
+      "primary_latency": { "avgcount": 765,
+          "sum": 4688922.186935000},
+      "peering_latency": { "avgcount": 704,
+          "sum": 1668.652135000},
+      "backfilling_latency": { "avgcount": 0,
+          "sum": 0.000000000},
+      "waitremotebackfillreserved_latency": { "avgcount": 0,
+          "sum": 0.000000000},
+      "waitlocalbackfillreserved_latency": { "avgcount": 0,
+          "sum": 0.000000000},
+      "notbackfilling_latency": { "avgcount": 0,
+          "sum": 0.000000000},
+      "repnotrecovering_latency": { "avgcount": 462,
+          "sum": 5158922.114600000},
+      "repwaitrecoveryreserved_latency": { "avgcount": 15,
+          "sum": 0.008275000},
+      "repwaitbackfillreserved_latency": { "avgcount": 1,
+          "sum": 0.000095000},
+      "RepRecovering_latency": { "avgcount": 16,
+          "sum": 2274.944727000},
+      "activating_latency": { "avgcount": 514,
+          "sum": 261.008520000},
+      "waitlocalrecoveryreserved_latency": { "avgcount": 20,
+          "sum": 0.175422000},
+      "waitremoterecoveryreserved_latency": { "avgcount": 20,
+          "sum": 0.682778000},
+      "recovering_latency": { "avgcount": 20,
+          "sum": 0.697551000},
+      "recovered_latency": { "avgcount": 511,
+          "sum": 0.011038000},
+      "clean_latency": { "avgcount": 503,
+          "sum": 4686961.154278000},
+      "active_latency": { "avgcount": 506,
+          "sum": 4687223.640464000},
+      "replicaactive_latency": { "avgcount": 446,
+          "sum": 5161197.078966000},
+      "stray_latency": { "avgcount": 794,
+          "sum": 4805.105128000},
+      "getinfo_latency": { "avgcount": 704,
+          "sum": 1138.477937000},
+      "getlog_latency": { "avgcount": 678,
+          "sum": 0.036393000},
+      "waitactingchange_latency": { "avgcount": 69,
+          "sum": 59.172893000},
+      "incomplete_latency": { "avgcount": 0,
+          "sum": 0.000000000},
+      "getmissing_latency": { "avgcount": 609,
+          "sum": 0.012288000},
+      "waitupthru_latency": { "avgcount": 576,
+          "sum": 530.106999000}},
+  "throttle-filestore_bytes": { "val": 0,
+      "max": 0,
+      "get": 0,
+      "get_sum": 0,
+      "get_or_fail_fail": 0,
+      "get_or_fail_success": 0,
+      "take": 0,
+      "take_sum": 0,
+      "put": 0,
+      "put_sum": 0,
+      "wait": { "avgcount": 0,
+          "sum": 0.000000000}},
+  "throttle-filestore_ops": { "val": 0,
+      "max": 0,
+      "get": 0,
+      "get_sum": 0,
+      "get_or_fail_fail": 0,
+      "get_or_fail_success": 0,
+      "take": 0,
+      "take_sum": 0,
+      "put": 0,
+      "put_sum": 0,
+      "wait": { "avgcount": 0,
+          "sum": 0.000000000}},
+  "throttle-msgr_dispatch_throttler-client": { "val": 0,
+      "max": 104857600,
+      "get": 130730,
+      "get_sum": 1246039872,
+      "get_or_fail_fail": 0,
+      "get_or_fail_success": 0,
+      "take": 0,
+      "take_sum": 0,
+      "put": 130730,
+      "put_sum": 1246039872,
+      "wait": { "avgcount": 0,
+          "sum": 0.000000000}},
+  "throttle-msgr_dispatch_throttler-cluster": { "val": 0,
+      "max": 104857600,
+      "get": 1108033,
+      "get_sum": 71277949992,
+      "get_or_fail_fail": 0,
+      "get_or_fail_success": 0,
+      "take": 0,
+      "take_sum": 0,
+      "put": 1108033,
+      "put_sum": 71277949992,
+      "wait": { "avgcount": 0,
+          "sum": 0.000000000}},
+  "throttle-msgr_dispatch_throttler-hb_back_server": { "val": 0,
+      "max": 104857600,
+      "get": 18320575,
+      "get_sum": 861067025,
+      "get_or_fail_fail": 0,
+      "get_or_fail_success": 0,
+      "take": 0,
+      "take_sum": 0,
+      "put": 18320575,
+      "put_sum": 861067025,
+      "wait": { "avgcount": 0,
+          "sum": 0.000000000}},
+  "throttle-msgr_dispatch_throttler-hb_front_server": { "val": 0,
+      "max": 104857600,
+      "get": 18320575,
+      "get_sum": 861067025,
+      "get_or_fail_fail": 0,
+      "get_or_fail_success": 0,
+      "take": 0,
+      "take_sum": 0,
+      "put": 18320575,
+      "put_sum": 861067025,
+      "wait": { "avgcount": 0,
+          "sum": 0.000000000}},
+  "throttle-msgr_dispatch_throttler-hbclient": { "val": 0,
+      "max": 104857600,
+      "get": 40479394,
+      "get_sum": 1902531518,
+      "get_or_fail_fail": 0,
+      "get_or_fail_success": 0,
+      "take": 0,
+      "take_sum": 0,
+      "put": 40479394,
+      "put_sum": 1902531518,
+      "wait": { "avgcount": 0,
+          "sum": 0.000000000}},
+  "throttle-msgr_dispatch_throttler-ms_objecter": { "val": 0,
+      "max": 104857600,
+      "get": 0,
+      "get_sum": 0,
+      "get_or_fail_fail": 0,
+      "get_or_fail_success": 0,
+      "take": 0,
+      "take_sum": 0,
+      "put": 0,
+      "put_sum": 0,
+      "wait": { "avgcount": 0,
+          "sum": 0.000000000}},
+  "throttle-objecter_bytes": { "val": 0,
+      "max": 104857600,
+      "get": 0,
+      "get_sum": 0,
+      "get_or_fail_fail": 0,
+      "get_or_fail_success": 0,
+      "take": 0,
+      "take_sum": 0,
+      "put": 0,
+      "put_sum": 0,
+      "wait": { "avgcount": 0,
+          "sum": 0.000000000}},
+  "throttle-objecter_ops": { "val": 0,
+      "max": 1024,
+      "get": 0,
+      "get_sum": 0,
+      "get_or_fail_fail": 0,
+      "get_or_fail_success": 0,
+      "take": 0,
+      "take_sum": 0,
+      "put": 0,
+      "put_sum": 0,
+      "wait": { "avgcount": 0,
+          "sum": 0.000000000}},
+  "throttle-osd_client_bytes": { "val": 0,
+      "max": 524288000,
+      "get": 24241,
+      "get_sum": 1241992581,
+      "get_or_fail_fail": 0,
+      "get_or_fail_success": 0,
+      "take": 0,
+      "take_sum": 0,
+      "put": 25958,
+      "put_sum": 1241992581,
+      "wait": { "avgcount": 0,
+          "sum": 0.000000000}},
+  "throttle-osd_client_messages": { "val": 0,
+      "max": 100,
+      "get": 49214,
+      "get_sum": 49214,
+      "get_or_fail_fail": 0,
+      "get_or_fail_success": 0,
+      "take": 0,
+      "take_sum": 0,
+      "put": 49214,
+      "put_sum": 49214,
+      "wait": { "avgcount": 0,
+          "sum": 0.000000000}}}
+`

plugins/inputs/chrony/README.md

@@ -0,0 +1,91 @@
+# chrony Input Plugin
+
+Get standard chrony metrics, requires chronyc executable.
+
+Below is the documentation of the various headers returned by `chronyc tracking`.
+
+- Reference ID - This is the refid and name (or IP address) if available, of the
+server to which the computer is currently synchronised. If this is 127.127.1.1
+it means the computer is not synchronised to any external source and that you
+have the ‘local’ mode operating (via the local command in chronyc (see section local),
+or the local directive in the ‘/etc/chrony.conf’ file (see section local)).
+- Stratum - The stratum indicates how many hops away from a computer with an attached
+reference clock we are. Such a computer is a stratum-1 computer, so the computer in the
+example is two hops away (i.e. a.b.c is a stratum-2 and is synchronised from a stratum-1).
+- Ref time - This is the time (UTC) at which the last measurement from the reference
+source was processed.
+- System time - In normal operation, chronyd never steps the system clock, because any
+jump in the timescale can have adverse consequences for certain application programs.
+Instead, any error in the system clock is corrected by slightly speeding up or slowing
+down the system clock until the error has been removed, and then returning to the system
+clock’s normal speed. A consequence of this is that there will be a period when the
+system clock (as read by other programs using the gettimeofday() system call, or by the
+date command in the shell) will be different from chronyd's estimate of the current true
+time (which it reports to NTP clients when it is operating in server mode). The value
+reported on this line is the difference due to this effect.
+- Last offset - This is the estimated local offset on the last clock update.
+- RMS offset - This is a long-term average of the offset value.
+- Frequency - The ‘frequency’ is the rate by which the system’s clock would be
+wrong if chronyd was not correcting it. It is expressed in ppm (parts per million).
+For example, a value of 1ppm would mean that when the system’s clock thinks it has
+advanced 1 second, it has actually advanced by 1.000001 seconds relative to true time.
+- Residual freq - This shows the ‘residual frequency’ for the currently selected
+reference source. This reflects any difference between what the measurements from the
+reference source indicate the frequency should be and the frequency currently being used.
+The reason this is not always zero is that a smoothing procedure is applied to the
+frequency. Each time a measurement from the reference source is obtained and a new
+residual frequency computed, the estimated accuracy of this residual is compared with the
+estimated accuracy (see ‘skew’ next) of the existing frequency value. A weighted average
+is computed for the new frequency, with weights depending on these accuracies. If the
+measurements from the reference source follow a consistent trend, the residual will be
+driven to zero over time.
+- Skew - This is the estimated error bound on the frequency.
+- Root delay -This is the total of the network path delays to the stratum-1 computer
+from which the computer is ultimately synchronised. In certain extreme situations, this
+value can be negative. (This can arise in a symmetric peer arrangement where the computers’
+frequencies are not tracking each other and the network delay is very short relative to the
+turn-around time at each computer.)
+- Root dispersion - This is the total dispersion accumulated through all the computers
+back to the stratum-1 computer from which the computer is ultimately synchronised.
+Dispersion is due to system clock resolution, statistical measurement variations etc.
+- Leap status - This is the leap status, which can be Normal, Insert second,
+Delete second or Not synchronised.
+
+### Configuration:
+
+```toml
+# Get standard chrony metrics, requires chronyc executable.
+[[inputs.chrony]]
+  # no configuration
+```
+
+### Measurements & Fields:
+
+- chrony
+    - last_offset (float, seconds)
+    - rms_offset (float, seconds)
+    - frequency (float, ppm)
+    - residual_freq (float, ppm)
+    - skew (float, ppm)
+    - root_delay (float, seconds)
+    - root_dispersion (float, seconds)
+    - update_interval (float, seconds)
+
+### Tags:
+
+- All measurements have the following tags:
+    - reference_id
+    - stratum
+    - leap_status
+
+### Example Output:
+
+```
+$ telegraf -config telegraf.conf -input-filter chrony -test
+* Plugin: chrony, Collection 1
+> chrony,leap_status=normal,reference_id=192.168.1.1,stratum=3 frequency=-35.657,last_offset=-0.000013616,residual_freq=-0,rms_offset=0.000027073,root_delay=0.000644,root_dispersion=0.003444,skew=0.001,update_interval=1031.2 1463750789687639161
+```
+
+
+
+

plugins/inputs/chrony/chrony.go

@@ -0,0 +1,118 @@
+// +build linux
+
+package chrony
+
+import (
+	"errors"
+	"fmt"
+	"os/exec"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/influxdata/telegraf"
+	"github.com/influxdata/telegraf/internal"
+	"github.com/influxdata/telegraf/plugins/inputs"
+)
+
+var (
+	execCommand = exec.Command // execCommand is used to mock commands in tests.
+)
+
+type Chrony struct {
+	path string
+}
+
+func (*Chrony) Description() string {
+	return "Get standard chrony metrics, requires chronyc executable."
+}
+
+func (*Chrony) SampleConfig() string {
+	return ""
+}
+
+func (c *Chrony) Gather(acc telegraf.Accumulator) error {
+	if len(c.path) == 0 {
+		return errors.New("chronyc not found: verify that chrony is installed and that chronyc is in your PATH")
+	}
+	cmd := execCommand(c.path, "tracking")
+	out, err := internal.CombinedOutputTimeout(cmd, time.Second*5)
+	if err != nil {
+		return fmt.Errorf("failed to run command %s: %s - %s", strings.Join(cmd.Args, " "), err, string(out))
+	}
+	fields, tags, err := processChronycOutput(string(out))
+	if err != nil {
+		return err
+	}
+	acc.AddFields("chrony", fields, tags)
+	return nil
+}
+
+// processChronycOutput takes in a string output from the chronyc command, like:
+//
+//     Reference ID    : 192.168.1.22 (ntp.example.com)
+//     Stratum         : 3
+//     Ref time (UTC)  : Thu May 12 14:27:07 2016
+//     System time     : 0.000020390 seconds fast of NTP time
+//     Last offset     : +0.000012651 seconds
+//     RMS offset      : 0.000025577 seconds
+//     Frequency       : 16.001 ppm slow
+//     Residual freq   : -0.000 ppm
+//     Skew            : 0.006 ppm
+//     Root delay      : 0.001655 seconds
+//     Root dispersion : 0.003307 seconds
+//     Update interval : 507.2 seconds
+//     Leap status     : Normal
+//
+// The value on the left side of the colon is used as field name, if the first field on
+// the right side is a float. If it cannot be parsed as float, it is a tag name.
+//
+// Ref time is ignored and all names are converted to snake case.
+//
+// It returns (<fields>, <tags>)
+func processChronycOutput(out string) (map[string]interface{}, map[string]string, error) {
+	tags := map[string]string{}
+	fields := map[string]interface{}{}
+	lines := strings.Split(strings.TrimSpace(out), "\n")
+	for _, line := range lines {
+		stats := strings.Split(line, ":")
+		if len(stats) < 2 {
+			return nil, nil, fmt.Errorf("unexpected output from chronyc, expected ':' in %s", out)
+		}
+		name := strings.ToLower(strings.Replace(strings.TrimSpace(stats[0]), " ", "_", -1))
+		// ignore reference time
+		if strings.Contains(name, "time") {
+			continue
+		}
+		valueFields := strings.Fields(stats[1])
+		if len(valueFields) == 0 {
+			return nil, nil, fmt.Errorf("unexpected output from chronyc: %s", out)
+		}
+		if strings.Contains(strings.ToLower(name), "stratum") {
+			tags["stratum"] = valueFields[0]
+			continue
+		}
+		value, err := strconv.ParseFloat(valueFields[0], 64)
+		if err != nil {
+			tags[name] = strings.ToLower(valueFields[0])
+			continue
+		}
+		if strings.Contains(stats[1], "slow") {
+			value = -value
+		}
+		fields[name] = value
+	}
+
+	return fields, tags, nil
+}
+
+func init() {
+	c := Chrony{}
+	path, _ := exec.LookPath("chronyc")
+	if len(path) > 0 {
+		c.path = path
+	}
+	inputs.Add("chrony", func() telegraf.Input {
+		return &c
+	})
+}

plugins/inputs/chrony/chrony_notlinux.go

@@ -0,0 +1,3 @@
+// +build !linux
+
+package chrony

plugins/inputs/chrony/chrony_test.go

@@ -0,0 +1,95 @@
+// +build linux
+
+package chrony
+
+import (
+	"fmt"
+	"os"
+	"os/exec"
+	"testing"
+
+	"github.com/influxdata/telegraf/testutil"
+)
+
+func TestGather(t *testing.T) {
+	c := Chrony{
+		path: "chronyc",
+	}
+	// overwriting exec commands with mock commands
+	execCommand = fakeExecCommand
+	defer func() { execCommand = exec.Command }()
+	var acc testutil.Accumulator
+
+	err := c.Gather(&acc)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	tags := map[string]string{
+		"reference_id": "192.168.1.22",
+		"leap_status":  "normal",
+		"stratum":      "3",
+	}
+	fields := map[string]interface{}{
+		"last_offset":     0.000012651,
+		"rms_offset":      0.000025577,
+		"frequency":       -16.001,
+		"residual_freq":   0.0,
+		"skew":            0.006,
+		"root_delay":      0.001655,
+		"root_dispersion": 0.003307,
+		"update_interval": 507.2,
+	}
+
+	acc.AssertContainsTaggedFields(t, "chrony", fields, tags)
+}
+
+// fackeExecCommand is a helper function that mock
+// the exec.Command call (and call the test binary)
+func fakeExecCommand(command string, args ...string) *exec.Cmd {
+	cs := []string{"-test.run=TestHelperProcess", "--", command}
+	cs = append(cs, args...)
+	cmd := exec.Command(os.Args[0], cs...)
+	cmd.Env = []string{"GO_WANT_HELPER_PROCESS=1"}
+	return cmd
+}
+
+// TestHelperProcess isn't a real test. It's used to mock exec.Command
+// For example, if you run:
+// GO_WANT_HELPER_PROCESS=1 go test -test.run=TestHelperProcess -- chrony tracking
+// it returns below mockData.
+func TestHelperProcess(t *testing.T) {
+	if os.Getenv("GO_WANT_HELPER_PROCESS") != "1" {
+		return
+	}
+
+	mockData := `Reference ID    : 192.168.1.22 (ntp.example.com)
+Stratum         : 3
+Ref time (UTC)  : Thu May 12 14:27:07 2016
+System time     : 0.000020390 seconds fast of NTP time
+Last offset     : +0.000012651 seconds
+RMS offset      : 0.000025577 seconds
+Frequency       : 16.001 ppm slow
+Residual freq   : -0.000 ppm
+Skew            : 0.006 ppm
+Root delay      : 0.001655 seconds
+Root dispersion : 0.003307 seconds
+Update interval : 507.2 seconds
+Leap status     : Normal
+`
+
+	args := os.Args
+
+	// Previous arguments are tests stuff, that looks like :
+	// /tmp/go-build970079519/…/_test/integration.test -test.run=TestHelperProcess --
+	cmd, args := args[3], args[4:]
+
+	if cmd == "chronyc" && args[0] == "tracking" {
+		fmt.Fprint(os.Stdout, mockData)
+	} else {
+		fmt.Fprint(os.Stdout, "command not found")
+		os.Exit(1)
+
+	}
+	os.Exit(0)
+}

plugins/inputs/cloudwatch/README.md

@@ -0,0 +1,86 @@
+# Amazon CloudWatch Statistics Input
+
+This plugin will pull Metric Statistics from Amazon CloudWatch.
+
+### Amazon Authentication
+
+This plugin uses a credential chain for Authentication with the CloudWatch
+API endpoint. In the following order the plugin will attempt to authenticate.
+1. [IAMS Role](http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html)
+2. [Environment Variables](https://github.com/aws/aws-sdk-go/wiki/configuring-sdk#environment-variables)
+3. [Shared Credentials](https://github.com/aws/aws-sdk-go/wiki/configuring-sdk#shared-credentials-file)
+
+### Configuration:
+
+```toml
+[[inputs.cloudwatch]]
+  ## Amazon Region (required)
+  region = 'us-east-1'
+
+  ## Requested CloudWatch aggregation Period (required - must be a multiple of 60s)
+  period = '1m'
+
+  ## Collection Delay (required - must account for metrics availability via CloudWatch API)
+  delay = '1m'
+
+  ## Override global run interval (optional - defaults to global interval)
+  ## Recomended: use metric 'interval' that is a multiple of 'period' to avoid 
+  ## gaps or overlap in pulled data
+  interval = '1m'
+
+  ## Metric Statistic Namespace (required)
+  namespace = 'AWS/ELB'
+
+  ## Metrics to Pull (optional)
+  ## Defaults to all Metrics in Namespace if nothing is provided
+  ## Refreshes Namespace available metrics every 1h
+  [[inputs.cloudwatch.metrics]]
+    names = ['Latency', 'RequestCount']
+	
+    ## Dimension filters for Metric (optional)
+    [[inputs.cloudwatch.metrics.dimensions]]
+      name = 'LoadBalancerName'
+      value = 'p-example'
+```
+#### Requirements and Terminology
+
+Plugin Configuration utilizes [CloudWatch concepts](http://docs.aws.amazon.com/AmazonCloudWatch/latest/DeveloperGuide/cloudwatch_concepts.html) and access pattern to allow monitoring of any CloudWatch Metric.
+
+- `region` must be a valid AWS [Region](http://docs.aws.amazon.com/AmazonCloudWatch/latest/DeveloperGuide/cloudwatch_concepts.html#CloudWatchRegions) value
+- `period` must be a valid CloudWatch [Period](http://docs.aws.amazon.com/AmazonCloudWatch/latest/DeveloperGuide/cloudwatch_concepts.html#CloudWatchPeriods) value
+- `namespace` must be a valid CloudWatch [Namespace](http://docs.aws.amazon.com/AmazonCloudWatch/latest/DeveloperGuide/cloudwatch_concepts.html#Namespace) value
+- `names` must be valid CloudWatch [Metric](http://docs.aws.amazon.com/AmazonCloudWatch/latest/DeveloperGuide/cloudwatch_concepts.html#Metric) names
+- `dimensions` must be valid CloudWatch [Dimension](http://docs.aws.amazon.com/AmazonCloudWatch/latest/DeveloperGuide/cloudwatch_concepts.html#Dimension) name/value pairs
+
+#### Restrictions and Limitations
+- CloudWatch metrics are not available instantly via the CloudWatch API. You should adjust your collection `delay` to account for this lag in metrics availability based on your [monitoring subscription level](http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-cloudwatch-new.html)
+- CloudWatch API usage incurs cost - see [GetMetricStatistics Pricing](https://aws.amazon.com/cloudwatch/pricing/)
+
+### Measurements & Fields:
+
+Each CloudWatch Namespace monitored records a measurement with fields for each available Metric Statistic
+Namespace and Metrics are represented in [snake case](https://en.wikipedia.org/wiki/Snake_case)
+
+- cloudwatch_{namespace}
+  - {metric}_sum         (metric Sum value)
+  - {metric}_average     (metric Average value)
+  - {metric}_minimum     (metric Minimum value)
+  - {metric}_maximum     (metric Maximum value)
+  - {metric}_sample_count (metric SampleCount value)
+
+
+### Tags:
+Each measurement is tagged with the following identifiers to uniquely identify the associated metric
+Tag Dimension names are represented in [snake case](https://en.wikipedia.org/wiki/Snake_case)
+
+- All measurements have the following tags:
+  - region           (CloudWatch Region)
+  - unit             (CloudWatch Metric Unit)
+  - {dimension-name} (Cloudwatch Dimension value - one for each metric dimension)
+
+### Example Output:
+
+```
+$ ./telegraf -config telegraf.conf -input-filter cloudwatch -test
+> cloudwatch_aws_elb,load_balancer_name=p-example,region=us-east-1,unit=seconds latency_average=0.004810798017284538,latency_maximum=0.1100282669067383,latency_minimum=0.0006084442138671875,latency_sample_count=4029,latency_sum=19.382705211639404 1459542420000000000
+```

plugins/inputs/cloudwatch/cloudwatch.go

@@ -0,0 +1,311 @@
+package cloudwatch
+
+import (
+	"fmt"
+	"strings"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/credentials"
+	"github.com/aws/aws-sdk-go/aws/session"
+
+	"github.com/aws/aws-sdk-go/service/cloudwatch"
+
+	"github.com/influxdata/telegraf"
+	"github.com/influxdata/telegraf/internal"
+	"github.com/influxdata/telegraf/plugins/inputs"
+)
+
+type (
+	CloudWatch struct {
+		Region      string            `toml:"region"`
+		AccessKey   string            `toml:"access_key"`
+		SecretKey   string            `toml:"secret_key"`
+		Period      internal.Duration `toml:"period"`
+		Delay       internal.Duration `toml:"delay"`
+		Namespace   string            `toml:"namespace"`
+		Metrics     []*Metric         `toml:"metrics"`
+		client      cloudwatchClient
+		metricCache *MetricCache
+	}
+
+	Metric struct {
+		MetricNames []string     `toml:"names"`
+		Dimensions  []*Dimension `toml:"dimensions"`
+	}
+
+	Dimension struct {
+		Name  string `toml:"name"`
+		Value string `toml:"value"`
+	}
+
+	MetricCache struct {
+		TTL     time.Duration
+		Fetched time.Time
+		Metrics []*cloudwatch.Metric
+	}
+
+	cloudwatchClient interface {
+		ListMetrics(*cloudwatch.ListMetricsInput) (*cloudwatch.ListMetricsOutput, error)
+		GetMetricStatistics(*cloudwatch.GetMetricStatisticsInput) (*cloudwatch.GetMetricStatisticsOutput, error)
+	}
+)
+
+func (c *CloudWatch) SampleConfig() string {
+	return `
+  ## Amazon Region
+  region = 'us-east-1'
+
+  ## Amazon Credentials
+  ## Credentials are loaded in the following order
+  ## 1) explicit credentials from 'access_key' and 'secret_key'
+  ## 2) environment variables
+  ## 3) shared credentials file
+  ## 4) EC2 Instance Profile
+  #access_key = ""
+  #secret_key = ""
+
+  ## Requested CloudWatch aggregation Period (required - must be a multiple of 60s)
+  period = '1m'
+
+  ## Collection Delay (required - must account for metrics availability via CloudWatch API)
+  delay = '1m'
+
+  ## Recomended: use metric 'interval' that is a multiple of 'period' to avoid
+  ## gaps or overlap in pulled data
+  interval = '1m'
+
+  ## Metric Statistic Namespace (required)
+  namespace = 'AWS/ELB'
+
+  ## Metrics to Pull (optional)
+  ## Defaults to all Metrics in Namespace if nothing is provided
+  ## Refreshes Namespace available metrics every 1h
+  #[[inputs.cloudwatch.metrics]]
+  #  names = ['Latency', 'RequestCount']
+  #
+  #  ## Dimension filters for Metric (optional)
+  #  [[inputs.cloudwatch.metrics.dimensions]]
+  #    name = 'LoadBalancerName'
+  #    value = 'p-example'
+`
+}
+
+func (c *CloudWatch) Description() string {
+	return "Pull Metric Statistics from Amazon CloudWatch"
+}
+
+func (c *CloudWatch) Gather(acc telegraf.Accumulator) error {
+	if c.client == nil {
+		c.initializeCloudWatch()
+	}
+
+	var metrics []*cloudwatch.Metric
+
+	// check for provided metric filter
+	if c.Metrics != nil {
+		metrics = []*cloudwatch.Metric{}
+		for _, m := range c.Metrics {
+			dimensions := make([]*cloudwatch.Dimension, len(m.Dimensions))
+			for k, d := range m.Dimensions {
+				dimensions[k] = &cloudwatch.Dimension{
+					Name:  aws.String(d.Name),
+					Value: aws.String(d.Value),
+				}
+			}
+			for _, name := range m.MetricNames {
+				metrics = append(metrics, &cloudwatch.Metric{
+					Namespace:  aws.String(c.Namespace),
+					MetricName: aws.String(name),
+					Dimensions: dimensions,
+				})
+			}
+		}
+	} else {
+		var err error
+		metrics, err = c.fetchNamespaceMetrics()
+		if err != nil {
+			return err
+		}
+	}
+
+	metricCount := len(metrics)
+	var errChan = make(chan error, metricCount)
+
+	now := time.Now()
+
+	// limit concurrency or we can easily exhaust user connection limit
+	semaphore := make(chan byte, 64)
+
+	for _, m := range metrics {
+		semaphore <- 0x1
+		go c.gatherMetric(acc, m, now, semaphore, errChan)
+	}
+
+	for i := 1; i <= metricCount; i++ {
+		err := <-errChan
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func init() {
+	inputs.Add("cloudwatch", func() telegraf.Input {
+		return &CloudWatch{}
+	})
+}
+
+/*
+ * Initialize CloudWatch client
+ */
+func (c *CloudWatch) initializeCloudWatch() error {
+	config := &aws.Config{
+		Region: aws.String(c.Region),
+	}
+	if c.AccessKey != "" || c.SecretKey != "" {
+		config.Credentials = credentials.NewStaticCredentials(c.AccessKey, c.SecretKey, "")
+	}
+
+	c.client = cloudwatch.New(session.New(config))
+	return nil
+}
+
+/*
+ * Fetch available metrics for given CloudWatch Namespace
+ */
+func (c *CloudWatch) fetchNamespaceMetrics() (metrics []*cloudwatch.Metric, err error) {
+	if c.metricCache != nil && c.metricCache.IsValid() {
+		metrics = c.metricCache.Metrics
+		return
+	}
+
+	metrics = []*cloudwatch.Metric{}
+
+	var token *string
+	for more := true; more; {
+		params := &cloudwatch.ListMetricsInput{
+			Namespace:  aws.String(c.Namespace),
+			Dimensions: []*cloudwatch.DimensionFilter{},
+			NextToken:  token,
+			MetricName: nil,
+		}
+
+		resp, err := c.client.ListMetrics(params)
+		if err != nil {
+			return nil, err
+		}
+
+		metrics = append(metrics, resp.Metrics...)
+
+		token = resp.NextToken
+		more = token != nil
+	}
+
+	cacheTTL, _ := time.ParseDuration("1hr")
+	c.metricCache = &MetricCache{
+		Metrics: metrics,
+		Fetched: time.Now(),
+		TTL:     cacheTTL,
+	}
+
+	return
+}
+
+/*
+ * Gather given Metric and emit any error
+ */
+func (c *CloudWatch) gatherMetric(acc telegraf.Accumulator, metric *cloudwatch.Metric, now time.Time, semaphore chan byte, errChan chan error) {
+	params := c.getStatisticsInput(metric, now)
+	resp, err := c.client.GetMetricStatistics(params)
+	if err != nil {
+		errChan <- err
+		<-semaphore
+		return
+	}
+
+	for _, point := range resp.Datapoints {
+		tags := map[string]string{
+			"region": c.Region,
+			"unit":   snakeCase(*point.Unit),
+		}
+
+		for _, d := range metric.Dimensions {
+			tags[snakeCase(*d.Name)] = *d.Value
+		}
+
+		// record field for each statistic
+		fields := map[string]interface{}{}
+
+		if point.Average != nil {
+			fields[formatField(*metric.MetricName, cloudwatch.StatisticAverage)] = *point.Average
+		}
+		if point.Maximum != nil {
+			fields[formatField(*metric.MetricName, cloudwatch.StatisticMaximum)] = *point.Maximum
+		}
+		if point.Minimum != nil {
+			fields[formatField(*metric.MetricName, cloudwatch.StatisticMinimum)] = *point.Minimum
+		}
+		if point.SampleCount != nil {
+			fields[formatField(*metric.MetricName, cloudwatch.StatisticSampleCount)] = *point.SampleCount
+		}
+		if point.Sum != nil {
+			fields[formatField(*metric.MetricName, cloudwatch.StatisticSum)] = *point.Sum
+		}
+
+		acc.AddFields(formatMeasurement(c.Namespace), fields, tags, *point.Timestamp)
+	}
+
+	errChan <- nil
+	<-semaphore
+}
+
+/*
+ * Formatting helpers
+ */
+func formatField(metricName string, statistic string) string {
+	return fmt.Sprintf("%s_%s", snakeCase(metricName), snakeCase(statistic))
+}
+
+func formatMeasurement(namespace string) string {
+	namespace = strings.Replace(namespace, "/", "_", -1)
+	namespace = snakeCase(namespace)
+	return fmt.Sprintf("cloudwatch_%s", namespace)
+}
+
+func snakeCase(s string) string {
+	s = internal.SnakeCase(s)
+	s = strings.Replace(s, "__", "_", -1)
+	return s
+}
+
+/*
+ * Map Metric to *cloudwatch.GetMetricStatisticsInput for given timeframe
+ */
+func (c *CloudWatch) getStatisticsInput(metric *cloudwatch.Metric, now time.Time) *cloudwatch.GetMetricStatisticsInput {
+	end := now.Add(-c.Delay.Duration)
+
+	input := &cloudwatch.GetMetricStatisticsInput{
+		StartTime:  aws.Time(end.Add(-c.Period.Duration)),
+		EndTime:    aws.Time(end),
+		MetricName: metric.MetricName,
+		Namespace:  metric.Namespace,
+		Period:     aws.Int64(int64(c.Period.Duration.Seconds())),
+		Dimensions: metric.Dimensions,
+		Statistics: []*string{
+			aws.String(cloudwatch.StatisticAverage),
+			aws.String(cloudwatch.StatisticMaximum),
+			aws.String(cloudwatch.StatisticMinimum),
+			aws.String(cloudwatch.StatisticSum),
+			aws.String(cloudwatch.StatisticSampleCount)},
+	}
+	return input
+}
+
+/*
+ * Check Metric Cache validity
+ */
+func (c *MetricCache) IsValid() bool {
+	return c.Metrics != nil && time.Since(c.Fetched) < c.TTL
+}

plugins/inputs/cloudwatch/cloudwatch_test.go

@@ -0,0 +1,131 @@
+package cloudwatch
+
+import (
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/cloudwatch"
+	"github.com/influxdata/telegraf/internal"
+	"github.com/influxdata/telegraf/testutil"
+	"github.com/stretchr/testify/assert"
+)
+
+type mockCloudWatchClient struct{}
+
+func (m *mockCloudWatchClient) ListMetrics(params *cloudwatch.ListMetricsInput) (*cloudwatch.ListMetricsOutput, error) {
+	metric := &cloudwatch.Metric{
+		Namespace:  params.Namespace,
+		MetricName: aws.String("Latency"),
+		Dimensions: []*cloudwatch.Dimension{
+			&cloudwatch.Dimension{
+				Name:  aws.String("LoadBalancerName"),
+				Value: aws.String("p-example"),
+			},
+		},
+	}
+
+	result := &cloudwatch.ListMetricsOutput{
+		Metrics: []*cloudwatch.Metric{metric},
+	}
+	return result, nil
+}
+
+func (m *mockCloudWatchClient) GetMetricStatistics(params *cloudwatch.GetMetricStatisticsInput) (*cloudwatch.GetMetricStatisticsOutput, error) {
+	dataPoint := &cloudwatch.Datapoint{
+		Timestamp:   params.EndTime,
+		Minimum:     aws.Float64(0.1),
+		Maximum:     aws.Float64(0.3),
+		Average:     aws.Float64(0.2),
+		Sum:         aws.Float64(123),
+		SampleCount: aws.Float64(100),
+		Unit:        aws.String("Seconds"),
+	}
+	result := &cloudwatch.GetMetricStatisticsOutput{
+		Label:      aws.String("Latency"),
+		Datapoints: []*cloudwatch.Datapoint{dataPoint},
+	}
+	return result, nil
+}
+
+func TestGather(t *testing.T) {
+	duration, _ := time.ParseDuration("1m")
+	internalDuration := internal.Duration{
+		Duration: duration,
+	}
+	c := &CloudWatch{
+		Region:    "us-east-1",
+		Namespace: "AWS/ELB",
+		Delay:     internalDuration,
+		Period:    internalDuration,
+	}
+
+	var acc testutil.Accumulator
+	c.client = &mockCloudWatchClient{}
+
+	c.Gather(&acc)
+
+	fields := map[string]interface{}{}
+	fields["latency_minimum"] = 0.1
+	fields["latency_maximum"] = 0.3
+	fields["latency_average"] = 0.2
+	fields["latency_sum"] = 123.0
+	fields["latency_sample_count"] = 100.0
+
+	tags := map[string]string{}
+	tags["unit"] = "seconds"
+	tags["region"] = "us-east-1"
+	tags["load_balancer_name"] = "p-example"
+
+	assert.True(t, acc.HasMeasurement("cloudwatch_aws_elb"))
+	acc.AssertContainsTaggedFields(t, "cloudwatch_aws_elb", fields, tags)
+
+}
+
+func TestGenerateStatisticsInputParams(t *testing.T) {
+	d := &cloudwatch.Dimension{
+		Name:  aws.String("LoadBalancerName"),
+		Value: aws.String("p-example"),
+	}
+
+	m := &cloudwatch.Metric{
+		MetricName: aws.String("Latency"),
+		Dimensions: []*cloudwatch.Dimension{d},
+	}
+
+	duration, _ := time.ParseDuration("1m")
+	internalDuration := internal.Duration{
+		Duration: duration,
+	}
+
+	c := &CloudWatch{
+		Namespace: "AWS/ELB",
+		Delay:     internalDuration,
+		Period:    internalDuration,
+	}
+
+	c.initializeCloudWatch()
+
+	now := time.Now()
+
+	params := c.getStatisticsInput(m, now)
+
+	assert.EqualValues(t, *params.EndTime, now.Add(-c.Delay.Duration))
+	assert.EqualValues(t, *params.StartTime, now.Add(-c.Period.Duration).Add(-c.Delay.Duration))
+	assert.Len(t, params.Dimensions, 1)
+	assert.Len(t, params.Statistics, 5)
+	assert.EqualValues(t, *params.Period, 60)
+}
+
+func TestMetricsCacheTimeout(t *testing.T) {
+	ttl, _ := time.ParseDuration("5ms")
+	cache := &MetricCache{
+		Metrics: []*cloudwatch.Metric{},
+		Fetched: time.Now(),
+		TTL:     ttl,
+	}
+
+	assert.True(t, cache.IsValid())
+	time.Sleep(ttl)
+	assert.False(t, cache.IsValid())
+}

plugins/inputs/couchbase/README.md

@@ -0,0 +1,63 @@
+# Telegraf Plugin: Couchbase
+
+## Configuration:
+
+```
+# Read per-node and per-bucket metrics from Couchbase
+[[inputs.couchbase]]
+  ## specify servers via a url matching:
+  ##  [protocol://][:password]@address[:port]
+  ##  e.g.
+  ##    http://couchbase-0.example.com/
+  ##    http://admin:secret@couchbase-0.example.com:8091/
+  ##
+  ## If no servers are specified, then localhost is used as the host.
+  ## If no protocol is specifed, HTTP is used.
+  ## If no port is specified, 8091 is used.
+  servers = ["http://localhost:8091"]
+```
+
+## Measurements:
+
+### couchbase_node
+
+Tags:
+- cluster: whatever you called it in `servers` in the configuration, e.g.: `http://couchbase-0.example.com/`
+- hostname: Couchbase's name for the node and port, e.g., `172.16.10.187:8091`
+
+Fields:
+- memory_free (unit: bytes, example: 23181365248.0)
+- memory_total (unit: bytes, example: 64424656896.0)
+
+### couchbase_bucket
+
+Tags:
+- cluster: whatever you called it in `servers` in the configuration, e.g.: `http://couchbase-0.example.com/`)
+- bucket: the name of the couchbase bucket, e.g., `blastro-df`
+
+Fields:
+- quota_percent_used (unit: percent, example: 68.85424936294555)
+- ops_per_sec (unit: count, example: 5686.789686789687)
+- disk_fetches (unit: count, example: 0.0)
+- item_count (unit: count, example: 943239752.0)
+- disk_used (unit: bytes, example: 409178772321.0)
+- data_used (unit: bytes, example: 212179309111.0)
+- mem_used (unit: bytes, example: 202156957464.0)
+
+
+## Example output
+
+```
+$ telegraf -config telegraf.conf -input-filter couchbase -test
+* Plugin: couchbase, Collection 1
+> couchbase_node,cluster=https://couchbase-0.example.com/,hostname=172.16.10.187:8091 memory_free=22927384576,memory_total=64424656896 1458381183695864929
+> couchbase_node,cluster=https://couchbase-0.example.com/,hostname=172.16.10.65:8091 memory_free=23520161792,memory_total=64424656896 1458381183695972112
+> couchbase_node,cluster=https://couchbase-0.example.com/,hostname=172.16.13.105:8091 memory_free=23531704320,memory_total=64424656896 1458381183695995259
+> couchbase_node,cluster=https://couchbase-0.example.com/,hostname=172.16.13.173:8091 memory_free=23628767232,memory_total=64424656896 1458381183696010870
+> couchbase_node,cluster=https://couchbase-0.example.com/,hostname=172.16.15.120:8091 memory_free=23616692224,memory_total=64424656896 1458381183696027406
+> couchbase_node,cluster=https://couchbase-0.example.com/,hostname=172.16.8.127:8091 memory_free=23431770112,memory_total=64424656896 1458381183696041040
+> couchbase_node,cluster=https://couchbase-0.example.com/,hostname=172.16.8.148:8091 memory_free=23811371008,memory_total=64424656896 1458381183696059060
+> couchbase_bucket,bucket=default,cluster=https://couchbase-0.example.com/ data_used=25743360,disk_fetches=0,disk_used=31744886,item_count=0,mem_used=77729224,ops_per_sec=0,quota_percent_used=10.58976636614118 1458381183696210074
+> couchbase_bucket,bucket=demoncat,cluster=https://couchbase-0.example.com/ data_used=38157584951,disk_fetches=0,disk_used=62730302441,item_count=14662532,mem_used=24015304256,ops_per_sec=1207.753207753208,quota_percent_used=79.87855353525707 1458381183696242695
+> couchbase_bucket,bucket=blastro-df,cluster=https://couchbase-0.example.com/ data_used=212552491622,disk_fetches=0,disk_used=413323157621,item_count=944655680,mem_used=202421103760,ops_per_sec=1692.176692176692,quota_percent_used=68.9442170551845 1458381183696272206
+```

plugins/inputs/couchbase/couchbase.go

@@ -0,0 +1,104 @@
+package couchbase
+
+import (
+	couchbase "github.com/couchbase/go-couchbase"
+	"github.com/influxdata/telegraf"
+	"github.com/influxdata/telegraf/plugins/inputs"
+	"sync"
+)
+
+type Couchbase struct {
+	Servers []string
+}
+
+var sampleConfig = `
+  ## specify servers via a url matching:
+  ##  [protocol://][:password]@address[:port]
+  ##  e.g.
+  ##    http://couchbase-0.example.com/
+  ##    http://admin:secret@couchbase-0.example.com:8091/
+  ##
+  ## If no servers are specified, then localhost is used as the host.
+  ## If no protocol is specifed, HTTP is used.
+  ## If no port is specified, 8091 is used.
+  servers = ["http://localhost:8091"]
+`
+
+func (r *Couchbase) SampleConfig() string {
+	return sampleConfig
+}
+
+func (r *Couchbase) Description() string {
+	return "Read metrics from one or many couchbase clusters"
+}
+
+// Reads stats from all configured clusters. Accumulates stats.
+// Returns one of the errors encountered while gathering stats (if any).
+func (r *Couchbase) Gather(acc telegraf.Accumulator) error {
+	if len(r.Servers) == 0 {
+		r.gatherServer("http://localhost:8091/", acc, nil)
+		return nil
+	}
+
+	var wg sync.WaitGroup
+
+	var outerr error
+
+	for _, serv := range r.Servers {
+		wg.Add(1)
+		go func(serv string) {
+			defer wg.Done()
+			outerr = r.gatherServer(serv, acc, nil)
+		}(serv)
+	}
+
+	wg.Wait()
+
+	return outerr
+}
+
+func (r *Couchbase) gatherServer(addr string, acc telegraf.Accumulator, pool *couchbase.Pool) error {
+	if pool == nil {
+		client, err := couchbase.Connect(addr)
+		if err != nil {
+			return err
+		}
+
+		// `default` is the only possible pool name. It's a
+		// placeholder for a possible future Couchbase feature. See
+		// http://stackoverflow.com/a/16990911/17498.
+		p, err := client.GetPool("default")
+		if err != nil {
+			return err
+		}
+		pool = &p
+	}
+	for i := 0; i < len(pool.Nodes); i++ {
+		node := pool.Nodes[i]
+		tags := map[string]string{"cluster": addr, "hostname": node.Hostname}
+		fields := make(map[string]interface{})
+		fields["memory_free"] = node.MemoryFree
+		fields["memory_total"] = node.MemoryTotal
+		acc.AddFields("couchbase_node", fields, tags)
+	}
+	for bucketName, _ := range pool.BucketMap {
+		tags := map[string]string{"cluster": addr, "bucket": bucketName}
+		bs := pool.BucketMap[bucketName].BasicStats
+		fields := make(map[string]interface{})
+		fields["quota_percent_used"] = bs["quotaPercentUsed"]
+		fields["ops_per_sec"] = bs["opsPerSec"]
+		fields["disk_fetches"] = bs["diskFetches"]
+		fields["item_count"] = bs["itemCount"]
+		fields["disk_used"] = bs["diskUsed"]
+		fields["data_used"] = bs["dataUsed"]
+		fields["mem_used"] = bs["memUsed"]
+		acc.AddFields("couchbase_bucket", fields, tags)
+	}
+	return nil
+}
+
+func init() {
+	inputs.Add("couchbase", func() telegraf.Input {
+		return &Couchbase{}
+	})
+}

plugins/inputs/couchdb/couchdb.go

@@ -10,6 +10,7 @@ import (
 	"reflect"
 	"strings"
 	"sync"
+	"time"
 )
 
 // Schema:
@@ -112,9 +113,18 @@ func (c *CouchDB) Gather(accumulator telegraf.Accumulator) error {
 
 }
 
+var tr = &http.Transport{
+	ResponseHeaderTimeout: time.Duration(3 * time.Second),
+}
+
+var client = &http.Client{
+	Transport: tr,
+	Timeout:   time.Duration(4 * time.Second),
+}
+
 func (c *CouchDB) fetchAndInsertData(accumulator telegraf.Accumulator, host string) error {
 
-	response, error := http.Get(host)
+	response, error := client.Get(host)
 	if error != nil {
 		return error
 	}

plugins/inputs/disque/disque.go

@@ -9,6 +9,7 @@ import (
 	"strconv"
 	"strings"
 	"sync"
+	"time"
 
 	"github.com/influxdata/telegraf"
 	"github.com/influxdata/telegraf/plugins/inputs"
@@ -23,13 +24,14 @@ type Disque struct {
 
 var sampleConfig = `
   ## An array of URI to gather stats about. Specify an ip or hostname
-  ## with optional port and password. ie disque://localhost, disque://10.10.3.33:18832,
-  ## 10.0.0.1:10000, etc.
-
+  ## with optional port and password.
+  ## ie disque://localhost, disque://10.10.3.33:18832, 10.0.0.1:10000, etc.
   ## If no servers are specified, then localhost is used as the host.
   servers = ["localhost"]
 `
 
+var defaultTimeout = 5 * time.Second
+
 func (r *Disque) SampleConfig() string {
 	return sampleConfig
 }
@@ -107,7 +109,7 @@ func (g *Disque) gatherServer(addr *url.URL, acc telegraf.Accumulator) error {
 			addr.Host = addr.Host + ":" + defaultPort
 		}
 
-		c, err := net.Dial("tcp", addr.Host)
+		c, err := net.DialTimeout("tcp", addr.Host, defaultTimeout)
 		if err != nil {
 			return fmt.Errorf("Unable to connect to disque server '%s': %s", addr.Host, err)
 		}
@@ -132,6 +134,9 @@ func (g *Disque) gatherServer(addr *url.URL, acc telegraf.Accumulator) error {
 		g.c = c
 	}
 
+	// Extend connection
+	g.c.SetDeadline(time.Now().Add(defaultTimeout))
+
 	g.c.Write([]byte("info\r\n"))
 
 	r := bufio.NewReader(g.c)
@@ -157,7 +162,7 @@ func (g *Disque) gatherServer(addr *url.URL, acc telegraf.Accumulator) error {
 	var read int
 
 	fields := make(map[string]interface{})
-	tags := map[string]string{"host": addr.String()}
+	tags := map[string]string{"disque_host": addr.String()}
 	for read < sz {
 		line, err := r.ReadString('\n')
 		if err != nil {

plugins/inputs/dns_query/dns_query.go

@@ -35,7 +35,8 @@ var sampleConfig = `
   ## Domains or subdomains to query. "."(root) is default
   domains = ["."] # optional
 
-  ## Query record type. Posible values: A, AAAA, CNAME, MX,  NS, PTR, TXT, SOA, SPF, SRV. Default is "NS"
+  ## Query record type. Default is "A"
+  ## Posible values: A, AAAA, CNAME, MX, NS, PTR, TXT, SOA, SPF, SRV.
   record_type = "A" # optional
 
   ## Dns server port. 53 is default

plugins/inputs/dns_query/dns_query_test.go

@@ -1,31 +1,42 @@
 package dns_query
 
 import (
-	"github.com/influxdata/telegraf/testutil"
-	"github.com/miekg/dns"
-	"github.com/stretchr/testify/assert"
 	"testing"
 	"time"
+
+	"github.com/influxdata/telegraf/testutil"
+
+	"github.com/miekg/dns"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
 var servers = []string{"8.8.8.8"}
-var domains = []string{"mjasion.pl"}
+var domains = []string{"google.com"}
 
 func TestGathering(t *testing.T) {
+	if testing.Short() {
+		t.Skip("Skipping network-dependent test in short mode.")
+	}
 	var dnsConfig = DnsQuery{
 		Servers: servers,
 		Domains: domains,
 	}
 	var acc testutil.Accumulator
 
-	dnsConfig.Gather(&acc)
-	metric, _ := acc.Get("dns_query")
+	err := dnsConfig.Gather(&acc)
+	assert.NoError(t, err)
+	metric, ok := acc.Get("dns_query")
+	require.True(t, ok)
 	queryTime, _ := metric.Fields["query_time_ms"].(float64)
 
 	assert.NotEqual(t, 0, queryTime)
 }
 
 func TestGatheringMxRecord(t *testing.T) {
+	if testing.Short() {
+		t.Skip("Skipping network-dependent test in short mode.")
+	}
 	var dnsConfig = DnsQuery{
 		Servers: servers,
 		Domains: domains,
@@ -33,14 +44,19 @@ func TestGatheringMxRecord(t *testing.T) {
 	var acc testutil.Accumulator
 	dnsConfig.RecordType = "MX"
 
-	dnsConfig.Gather(&acc)
-	metric, _ := acc.Get("dns_query")
+	err := dnsConfig.Gather(&acc)
+	assert.NoError(t, err)
+	metric, ok := acc.Get("dns_query")
+	require.True(t, ok)
 	queryTime, _ := metric.Fields["query_time_ms"].(float64)
 
 	assert.NotEqual(t, 0, queryTime)
 }
 
 func TestGatheringRootDomain(t *testing.T) {
+	if testing.Short() {
+		t.Skip("Skipping network-dependent test in short mode.")
+	}
 	var dnsConfig = DnsQuery{
 		Servers:    servers,
 		Domains:    []string{"."},
@@ -54,8 +70,10 @@ func TestGatheringRootDomain(t *testing.T) {
 	}
 	fields := map[string]interface{}{}
 
-	dnsConfig.Gather(&acc)
-	metric, _ := acc.Get("dns_query")
+	err := dnsConfig.Gather(&acc)
+	assert.NoError(t, err)
+	metric, ok := acc.Get("dns_query")
+	require.True(t, ok)
 	queryTime, _ := metric.Fields["query_time_ms"].(float64)
 
 	fields["query_time_ms"] = queryTime
@@ -63,6 +81,9 @@ func TestGatheringRootDomain(t *testing.T) {
 }
 
 func TestMetricContainsServerAndDomainAndRecordTypeTags(t *testing.T) {
+	if testing.Short() {
+		t.Skip("Skipping network-dependent test in short mode.")
+	}
 	var dnsConfig = DnsQuery{
 		Servers: servers,
 		Domains: domains,
@@ -70,13 +91,15 @@ func TestMetricContainsServerAndDomainAndRecordTypeTags(t *testing.T) {
 	var acc testutil.Accumulator
 	tags := map[string]string{
 		"server":      "8.8.8.8",
-		"domain":      "mjasion.pl",
+		"domain":      "google.com",
 		"record_type": "NS",
 	}
 	fields := map[string]interface{}{}
 
-	dnsConfig.Gather(&acc)
-	metric, _ := acc.Get("dns_query")
+	err := dnsConfig.Gather(&acc)
+	assert.NoError(t, err)
+	metric, ok := acc.Get("dns_query")
+	require.True(t, ok)
 	queryTime, _ := metric.Fields["query_time_ms"].(float64)
 
 	fields["query_time_ms"] = queryTime
@@ -84,6 +107,9 @@ func TestMetricContainsServerAndDomainAndRecordTypeTags(t *testing.T) {
 }
 
 func TestGatheringTimeout(t *testing.T) {
+	if testing.Short() {
+		t.Skip("Skipping network-dependent test in short mode.")
+	}
 	var dnsConfig = DnsQuery{
 		Servers: servers,
 		Domains: domains,

plugins/inputs/docker/README.md

@@ -5,11 +5,11 @@ docker containers. You can read Docker's documentation for their remote API
 [here](https://docs.docker.com/engine/reference/api/docker_remote_api_v1.20/#get-container-stats-based-on-resource-usage)
 
 The docker plugin uses the excellent
-[fsouza go-dockerclient](https://github.com/fsouza/go-dockerclient) library to
+[docker engine-api](https://github.com/docker/engine-api) library to
 gather stats. Documentation for the library can be found
-[here](https://godoc.org/github.com/fsouza/go-dockerclient) and documentation
+[here](https://godoc.org/github.com/docker/engine-api) and documentation
 for the stat structure can be found
-[here](https://godoc.org/github.com/fsouza/go-dockerclient#Stats)
+[here](https://godoc.org/github.com/docker/engine-api/types#Stats)
 
 ### Configuration:
 
@@ -29,10 +29,10 @@ for the stat structure can be found
 Every effort was made to preserve the names based on the JSON response from the
 docker API.
 
-Note that the docker_cpu metric may appear multiple times per collection, based
-on the availability of per-cpu stats on your system.
+Note that the docker_container_cpu metric may appear multiple times per collection,
+based on the availability of per-cpu stats on your system.
 
-- docker_mem
+- docker_container_mem
     - total_pgmafault
     - cache
     - mapped_file
@@ -66,7 +66,8 @@ on the availability of per-cpu stats on your system.
     - usage
     - failcnt
     - limit
-- docker_cpu
+    - container_id
+- docker_container_cpu
     - throttling_periods
     - throttling_throttled_periods
     - throttling_throttled_time
@@ -74,7 +75,9 @@ on the availability of per-cpu stats on your system.
     - usage_in_usermode
     - usage_system
     - usage_total
-- docker_net
+    - usage_percent
+    - container_id
+- docker_container_net
     - rx_dropped
     - rx_bytes
     - rx_errors
@@ -83,7 +86,8 @@ on the availability of per-cpu stats on your system.
     - rx_packets
     - tx_errors
     - tx_bytes
-- docker_blkio
+    - container_id
+- docker_container_blkio
     - io_service_bytes_recursive_async
     - io_service_bytes_recursive_read
     - io_service_bytes_recursive_sync
@@ -94,18 +98,51 @@ on the availability of per-cpu stats on your system.
     - io_serviced_recursive_sync
     - io_serviced_recursive_total
     - io_serviced_recursive_write
+    - container_id
+- docker_
+    - n_used_file_descriptors
+    - n_cpus
+    - n_containers
+    - n_images
+    - n_goroutines
+    - n_listener_events
+    - memory_total
+    - pool_blocksize
+- docker_data
+    - available
+    - total
+    - used
+- docker_metadata
+    - available
+    - total
+    - used
+
 
 ### Tags:
 
-- All stats have the following tags:
-    - cont_id (container ID)
-    - cont_image (container image)
-    - cont_name (container name)
-- docker_cpu specific:
+- docker (memory_total)
+    - unit=bytes
+- docker (pool_blocksize)
+    - unit=bytes
+- docker_data
+    - unit=bytes
+- docker_metadata
+    - unit=bytes
+
+- docker_container_mem specific:
+    - container_image
+    - container_name
+- docker_container_cpu specific:
+    - container_image
+    - container_name
     - cpu
-- docker_net specific:
+- docker_container_net specific:
+    - container_image
+    - container_name
     - network
-- docker_blkio specific:
+- docker_container_blkio specific:
+    - container_image
+    - container_name
     - device
 
 ### Example Output:
@@ -113,8 +150,18 @@ on the availability of per-cpu stats on your system.
 ```
 % ./telegraf -config ~/ws/telegraf.conf -input-filter docker -test
 * Plugin: docker, Collection 1
-> docker_mem,cont_id=5705ba8ed8fb47527410653d60a8bb2f3af5e62372297c419022a3cc6d45d848,\
-cont_image=spotify/kafka,cont_name=kafka \
+> docker n_cpus=8i 1456926671065383978
+> docker n_used_file_descriptors=15i 1456926671065383978
+> docker n_containers=7i 1456926671065383978
+> docker n_images=152i 1456926671065383978
+> docker n_goroutines=36i 1456926671065383978
+> docker n_listener_events=0i 1456926671065383978
+> docker,unit=bytes memory_total=18935443456i 1456926671065383978
+> docker,unit=bytes pool_blocksize=65540i 1456926671065383978
+> docker_data,unit=bytes available=24340000000i,total=107400000000i,used=14820000000i 1456926671065383978
+> docker_metadata,unit=bytes available=2126999999i,total=2146999999i,used=20420000i 145692667106538
+> docker_container_mem,
+container_image=spotify/kafka,container_name=kafka \
 active_anon=52568064i,active_file=6926336i,cache=12038144i,fail_count=0i,\
 hierarchical_memory_limit=9223372036854771712i,inactive_anon=52707328i,\
 inactive_file=5111808i,limit=1044578304i,mapped_file=10301440i,\
@@ -125,21 +172,21 @@ total_inactive_file=5111808i,total_mapped_file=10301440i,total_pgfault=63762i,\
 total_pgmafault=0i,total_pgpgin=73355i,total_pgpgout=45736i,\
 total_rss=105275392i,total_rss_huge=4194304i,total_unevictable=0i,\
 total_writeback=0i,unevictable=0i,usage=117440512i,writeback=0i 1453409536840126713
-> docker_cpu,cont_id=5705ba8ed8fb47527410653d60a8bb2f3af5e62372297c419022a3cc6d45d848,\
-cont_image=spotify/kafka,cont_name=kafka,cpu=cpu-total \
+> docker_container_cpu,
+container_image=spotify/kafka,container_name=kafka,cpu=cpu-total \
 throttling_periods=0i,throttling_throttled_periods=0i,\
 throttling_throttled_time=0i,usage_in_kernelmode=440000000i,\
 usage_in_usermode=2290000000i,usage_system=84795360000000i,\
 usage_total=6628208865i 1453409536840126713
-> docker_cpu,cont_id=5705ba8ed8fb47527410653d60a8bb2f3af5e62372297c419022a3cc6d45d848,\
-cont_image=spotify/kafka,cont_name=kafka,cpu=cpu0 \
+> docker_container_cpu,
+container_image=spotify/kafka,container_name=kafka,cpu=cpu0 \
 usage_total=6628208865i 1453409536840126713
-> docker_net,cont_id=5705ba8ed8fb47527410653d60a8bb2f3af5e62372297c419022a3cc6d45d848,\
-cont_image=spotify/kafka,cont_name=kafka,network=eth0 \
+> docker_container_net,\
+container_image=spotify/kafka,container_name=kafka,network=eth0 \
 rx_bytes=7468i,rx_dropped=0i,rx_errors=0i,rx_packets=94i,tx_bytes=946i,\
 tx_dropped=0i,tx_errors=0i,tx_packets=13i 1453409536840126713
-> docker_blkio,cont_id=5705ba8ed8fb47527410653d60a8bb2f3af5e62372297c419022a3cc6d45d848,\
-cont_image=spotify/kafka,cont_name=kafka,device=8:0 \
+> docker_container_blkio,
+container_image=spotify/kafka,container_name=kafka,device=8:0 \
 io_service_bytes_recursive_async=80216064i,io_service_bytes_recursive_read=79925248i,\
 io_service_bytes_recursive_sync=77824i,io_service_bytes_recursive_total=80293888i,\
 io_service_bytes_recursive_write=368640i,io_serviced_recursive_async=6562i,\

plugins/inputs/docker/docker.go

@@ -1,25 +1,54 @@
 package system
 
 import (
+	"encoding/json"
 	"fmt"
+	"io"
 	"log"
+	"regexp"
+	"strconv"
 	"strings"
 	"sync"
 	"time"
 
-	"github.com/influxdata/telegraf"
-	"github.com/influxdata/telegraf/plugins/inputs"
+	"golang.org/x/net/context"
 
-	"github.com/fsouza/go-dockerclient"
+	"github.com/docker/engine-api/client"
+	"github.com/docker/engine-api/types"
+	"github.com/influxdata/telegraf"
+	"github.com/influxdata/telegraf/internal"
+	"github.com/influxdata/telegraf/plugins/inputs"
 )
 
+// Docker object
 type Docker struct {
 	Endpoint       string
 	ContainerNames []string
+	Timeout        internal.Duration
 
-	client *docker.Client
+	client DockerClient
 }
 
+// DockerClient interface, useful for testing
+type DockerClient interface {
+	Info(ctx context.Context) (types.Info, error)
+	ContainerList(ctx context.Context, options types.ContainerListOptions) ([]types.Container, error)
+	ContainerStats(ctx context.Context, containerID string, stream bool) (io.ReadCloser, error)
+}
+
+// KB, MB, GB, TB, PB...human friendly
+const (
+	KB = 1000
+	MB = 1000 * KB
+	GB = 1000 * MB
+	TB = 1000 * GB
+	PB = 1000 * TB
+)
+
+var (
+	sizeRegex = regexp.MustCompile(`^(\d+(\.\d+)*) ?([kKmMgGtTpP])?[bB]?$`)
+)
+
 var sampleConfig = `
   ## Docker Endpoint
   ##   To use TCP, set endpoint = "tcp://[ip]:[port]"
@@ -27,30 +56,36 @@ var sampleConfig = `
   endpoint = "unix:///var/run/docker.sock"
   ## Only collect metrics for these containers, collect all if empty
   container_names = []
+  ## Timeout for docker list, info, and stats commands
+  timeout = "5s"
 `
 
+// Description returns input description
 func (d *Docker) Description() string {
 	return "Read metrics about docker containers"
 }
 
+// SampleConfig prints sampleConfig
 func (d *Docker) SampleConfig() string { return sampleConfig }
 
+// Gather starts stats collection
 func (d *Docker) Gather(acc telegraf.Accumulator) error {
 	if d.client == nil {
-		var c *docker.Client
+		var c *client.Client
 		var err error
+		defaultHeaders := map[string]string{"User-Agent": "engine-api-cli-1.0"}
 		if d.Endpoint == "ENV" {
-			c, err = docker.NewClientFromEnv()
+			c, err = client.NewEnvClient()
 			if err != nil {
 				return err
 			}
 		} else if d.Endpoint == "" {
-			c, err = docker.NewClient("unix:///var/run/docker.sock")
+			c, err = client.NewClient("unix:///var/run/docker.sock", "", nil, defaultHeaders)
 			if err != nil {
 				return err
 			}
 		} else {
-			c, err = docker.NewClient(d.Endpoint)
+			c, err = client.NewClient(d.Endpoint, "", nil, defaultHeaders)
 			if err != nil {
 				return err
 			}
@@ -58,21 +93,31 @@ func (d *Docker) Gather(acc telegraf.Accumulator) error {
 		d.client = c
 	}
 
-	opts := docker.ListContainersOptions{}
-	containers, err := d.client.ListContainers(opts)
+	// Get daemon info
+	err := d.gatherInfo(acc)
+	if err != nil {
+		fmt.Println(err.Error())
+	}
+
+	// List containers
+	opts := types.ContainerListOptions{}
+	ctx, cancel := context.WithTimeout(context.Background(), d.Timeout.Duration)
+	defer cancel()
+	containers, err := d.client.ContainerList(ctx, opts)
 	if err != nil {
 		return err
 	}
 
+	// Get container data
 	var wg sync.WaitGroup
 	wg.Add(len(containers))
 	for _, container := range containers {
-
-		go func(c docker.APIContainers) {
+		go func(c types.Container) {
 			defer wg.Done()
 			err := d.gatherContainer(c, acc)
 			if err != nil {
-				fmt.Println(err.Error())
+				log.Printf("Error gathering container %s stats: %s\n",
+					c.Names, err.Error())
 			}
 		}(container)
 	}
@@ -81,10 +126,80 @@ func (d *Docker) Gather(acc telegraf.Accumulator) error {
 	return nil
 }
 
+func (d *Docker) gatherInfo(acc telegraf.Accumulator) error {
+	// Init vars
+	dataFields := make(map[string]interface{})
+	metadataFields := make(map[string]interface{})
+	now := time.Now()
+	// Get info from docker daemon
+	ctx, cancel := context.WithTimeout(context.Background(), d.Timeout.Duration)
+	defer cancel()
+	info, err := d.client.Info(ctx)
+	if err != nil {
+		return err
+	}
+
+	fields := map[string]interface{}{
+		"n_cpus":                  info.NCPU,
+		"n_used_file_descriptors": info.NFd,
+		"n_containers":            info.Containers,
+		"n_images":                info.Images,
+		"n_goroutines":            info.NGoroutines,
+		"n_listener_events":       info.NEventsListener,
+	}
+	// Add metrics
+	acc.AddFields("docker",
+		fields,
+		nil,
+		now)
+	acc.AddFields("docker",
+		map[string]interface{}{"memory_total": info.MemTotal},
+		map[string]string{"unit": "bytes"},
+		now)
+	// Get storage metrics
+	for _, rawData := range info.DriverStatus {
+		// Try to convert string to int (bytes)
+		value, err := parseSize(rawData[1])
+		if err != nil {
+			continue
+		}
+		name := strings.ToLower(strings.Replace(rawData[0], " ", "_", -1))
+		if name == "pool_blocksize" {
+			// pool blocksize
+			acc.AddFields("docker",
+				map[string]interface{}{"pool_blocksize": value},
+				map[string]string{"unit": "bytes"},
+				now)
+		} else if strings.HasPrefix(name, "data_space_") {
+			// data space
+			fieldName := strings.TrimPrefix(name, "data_space_")
+			dataFields[fieldName] = value
+		} else if strings.HasPrefix(name, "metadata_space_") {
+			// metadata space
+			fieldName := strings.TrimPrefix(name, "metadata_space_")
+			metadataFields[fieldName] = value
+		}
+	}
+	if len(dataFields) > 0 {
+		acc.AddFields("docker_data",
+			dataFields,
+			map[string]string{"unit": "bytes"},
+			now)
+	}
+	if len(metadataFields) > 0 {
+		acc.AddFields("docker_metadata",
+			metadataFields,
+			map[string]string{"unit": "bytes"},
+			now)
+	}
+	return nil
+}
+
 func (d *Docker) gatherContainer(
-	container docker.APIContainers,
+	container types.Container,
 	acc telegraf.Accumulator,
 ) error {
+	var v *types.StatsJSON
 	// Parse container name
 	cname := "unknown"
 	if len(container.Names) > 0 {
@@ -93,9 +208,8 @@ func (d *Docker) gatherContainer(
 	}
 
 	tags := map[string]string{
-		"cont_id":    container.ID,
-		"cont_name":  cname,
-		"cont_image": container.Image,
+		"container_name":  cname,
+		"container_image": container.Image,
 	}
 	if len(d.ContainerNames) > 0 {
 		if !sliceContains(cname, d.ContainerNames) {
@@ -103,44 +217,36 @@ func (d *Docker) gatherContainer(
 		}
 	}
 
-	statChan := make(chan *docker.Stats)
-	done := make(chan bool)
-	statOpts := docker.StatsOptions{
-		Stream:  false,
-		ID:      container.ID,
-		Stats:   statChan,
-		Done:    done,
-		Timeout: time.Duration(time.Second * 5),
+	ctx, cancel := context.WithTimeout(context.Background(), d.Timeout.Duration)
+	defer cancel()
+	r, err := d.client.ContainerStats(ctx, container.ID, false)
+	if err != nil {
+		return fmt.Errorf("Error getting docker stats: %s", err.Error())
 	}
-
-	go func() {
-		err := d.client.Stats(statOpts)
-		if err != nil {
-			log.Printf("Error getting docker stats: %s\n", err.Error())
+	defer r.Close()
+	dec := json.NewDecoder(r)
+	if err = dec.Decode(&v); err != nil {
+		if err == io.EOF {
+			return nil
 		}
-	}()
-
-	stat := <-statChan
-	close(done)
-
-	if stat == nil {
-		return nil
+		return fmt.Errorf("Error decoding: %s", err.Error())
 	}
 
 	// Add labels to tags
-	for k, v := range container.Labels {
-		tags[k] = v
+	for k, label := range container.Labels {
+		tags[k] = label
 	}
 
-	gatherContainerStats(stat, acc, tags)
+	gatherContainerStats(v, acc, tags, container.ID)
 
 	return nil
 }
 
 func gatherContainerStats(
-	stat *docker.Stats,
+	stat *types.StatsJSON,
 	acc telegraf.Accumulator,
 	tags map[string]string,
+	id string,
 ) {
 	now := stat.Read
 
@@ -149,80 +255,87 @@ func gatherContainerStats(
 		"usage":                     stat.MemoryStats.Usage,
 		"fail_count":                stat.MemoryStats.Failcnt,
 		"limit":                     stat.MemoryStats.Limit,
-		"total_pgmafault":           stat.MemoryStats.Stats.TotalPgmafault,
-		"cache":                     stat.MemoryStats.Stats.Cache,
-		"mapped_file":               stat.MemoryStats.Stats.MappedFile,
-		"total_inactive_file":       stat.MemoryStats.Stats.TotalInactiveFile,
-		"pgpgout":                   stat.MemoryStats.Stats.Pgpgout,
-		"rss":                       stat.MemoryStats.Stats.Rss,
-		"total_mapped_file":         stat.MemoryStats.Stats.TotalMappedFile,
-		"writeback":                 stat.MemoryStats.Stats.Writeback,
-		"unevictable":               stat.MemoryStats.Stats.Unevictable,
-		"pgpgin":                    stat.MemoryStats.Stats.Pgpgin,
-		"total_unevictable":         stat.MemoryStats.Stats.TotalUnevictable,
-		"pgmajfault":                stat.MemoryStats.Stats.Pgmajfault,
-		"total_rss":                 stat.MemoryStats.Stats.TotalRss,
-		"total_rss_huge":            stat.MemoryStats.Stats.TotalRssHuge,
-		"total_writeback":           stat.MemoryStats.Stats.TotalWriteback,
-		"total_inactive_anon":       stat.MemoryStats.Stats.TotalInactiveAnon,
-		"rss_huge":                  stat.MemoryStats.Stats.RssHuge,
-		"hierarchical_memory_limit": stat.MemoryStats.Stats.HierarchicalMemoryLimit,
-		"total_pgfault":             stat.MemoryStats.Stats.TotalPgfault,
-		"total_active_file":         stat.MemoryStats.Stats.TotalActiveFile,
-		"active_anon":               stat.MemoryStats.Stats.ActiveAnon,
-		"total_active_anon":         stat.MemoryStats.Stats.TotalActiveAnon,
-		"total_pgpgout":             stat.MemoryStats.Stats.TotalPgpgout,
-		"total_cache":               stat.MemoryStats.Stats.TotalCache,
-		"inactive_anon":             stat.MemoryStats.Stats.InactiveAnon,
-		"active_file":               stat.MemoryStats.Stats.ActiveFile,
-		"pgfault":                   stat.MemoryStats.Stats.Pgfault,
-		"inactive_file":             stat.MemoryStats.Stats.InactiveFile,
-		"total_pgpgin":              stat.MemoryStats.Stats.TotalPgpgin,
+		"total_pgmafault":           stat.MemoryStats.Stats["total_pgmajfault"],
+		"cache":                     stat.MemoryStats.Stats["cache"],
+		"mapped_file":               stat.MemoryStats.Stats["mapped_file"],
+		"total_inactive_file":       stat.MemoryStats.Stats["total_inactive_file"],
+		"pgpgout":                   stat.MemoryStats.Stats["pagpgout"],
+		"rss":                       stat.MemoryStats.Stats["rss"],
+		"total_mapped_file":         stat.MemoryStats.Stats["total_mapped_file"],
+		"writeback":                 stat.MemoryStats.Stats["writeback"],
+		"unevictable":               stat.MemoryStats.Stats["unevictable"],
+		"pgpgin":                    stat.MemoryStats.Stats["pgpgin"],
+		"total_unevictable":         stat.MemoryStats.Stats["total_unevictable"],
+		"pgmajfault":                stat.MemoryStats.Stats["pgmajfault"],
+		"total_rss":                 stat.MemoryStats.Stats["total_rss"],
+		"total_rss_huge":            stat.MemoryStats.Stats["total_rss_huge"],
+		"total_writeback":           stat.MemoryStats.Stats["total_write_back"],
+		"total_inactive_anon":       stat.MemoryStats.Stats["total_inactive_anon"],
+		"rss_huge":                  stat.MemoryStats.Stats["rss_huge"],
+		"hierarchical_memory_limit": stat.MemoryStats.Stats["hierarchical_memory_limit"],
+		"total_pgfault":             stat.MemoryStats.Stats["total_pgfault"],
+		"total_active_file":         stat.MemoryStats.Stats["total_active_file"],
+		"active_anon":               stat.MemoryStats.Stats["active_anon"],
+		"total_active_anon":         stat.MemoryStats.Stats["total_active_anon"],
+		"total_pgpgout":             stat.MemoryStats.Stats["total_pgpgout"],
+		"total_cache":               stat.MemoryStats.Stats["total_cache"],
+		"inactive_anon":             stat.MemoryStats.Stats["inactive_anon"],
+		"active_file":               stat.MemoryStats.Stats["active_file"],
+		"pgfault":                   stat.MemoryStats.Stats["pgfault"],
+		"inactive_file":             stat.MemoryStats.Stats["inactive_file"],
+		"total_pgpgin":              stat.MemoryStats.Stats["total_pgpgin"],
 		"usage_percent":             calculateMemPercent(stat),
+		"container_id":              id,
 	}
-	acc.AddFields("docker_mem", memfields, tags, now)
+	acc.AddFields("docker_container_mem", memfields, tags, now)
 
 	cpufields := map[string]interface{}{
 		"usage_total":                  stat.CPUStats.CPUUsage.TotalUsage,
 		"usage_in_usermode":            stat.CPUStats.CPUUsage.UsageInUsermode,
 		"usage_in_kernelmode":          stat.CPUStats.CPUUsage.UsageInKernelmode,
-		"usage_system":                 stat.CPUStats.SystemCPUUsage,
+		"usage_system":                 stat.CPUStats.SystemUsage,
 		"throttling_periods":           stat.CPUStats.ThrottlingData.Periods,
 		"throttling_throttled_periods": stat.CPUStats.ThrottlingData.ThrottledPeriods,
 		"throttling_throttled_time":    stat.CPUStats.ThrottlingData.ThrottledTime,
 		"usage_percent":                calculateCPUPercent(stat),
+		"container_id":                 id,
 	}
 	cputags := copyTags(tags)
 	cputags["cpu"] = "cpu-total"
-	acc.AddFields("docker_cpu", cpufields, cputags, now)
+	acc.AddFields("docker_container_cpu", cpufields, cputags, now)
 
 	for i, percpu := range stat.CPUStats.CPUUsage.PercpuUsage {
 		percputags := copyTags(tags)
 		percputags["cpu"] = fmt.Sprintf("cpu%d", i)
-		acc.AddFields("docker_cpu", map[string]interface{}{"usage_total": percpu}, percputags, now)
+		fields := map[string]interface{}{
+			"usage_total":  percpu,
+			"container_id": id,
+		}
+		acc.AddFields("docker_container_cpu", fields, percputags, now)
 	}
 
 	for network, netstats := range stat.Networks {
 		netfields := map[string]interface{}{
-			"rx_dropped": netstats.RxDropped,
-			"rx_bytes":   netstats.RxBytes,
-			"rx_errors":  netstats.RxErrors,
-			"tx_packets": netstats.TxPackets,
-			"tx_dropped": netstats.TxDropped,
-			"rx_packets": netstats.RxPackets,
-			"tx_errors":  netstats.TxErrors,
-			"tx_bytes":   netstats.TxBytes,
+			"rx_dropped":   netstats.RxDropped,
+			"rx_bytes":     netstats.RxBytes,
+			"rx_errors":    netstats.RxErrors,
+			"tx_packets":   netstats.TxPackets,
+			"tx_dropped":   netstats.TxDropped,
+			"rx_packets":   netstats.RxPackets,
+			"tx_errors":    netstats.TxErrors,
+			"tx_bytes":     netstats.TxBytes,
+			"container_id": id,
 		}
 		// Create a new network tag dictionary for the "network" tag
 		nettags := copyTags(tags)
 		nettags["network"] = network
-		acc.AddFields("docker_net", netfields, nettags, now)
+		acc.AddFields("docker_container_net", netfields, nettags, now)
 	}
 
-	gatherBlockIOMetrics(stat, acc, tags, now)
+	gatherBlockIOMetrics(stat, acc, tags, now, id)
 }
 
-func calculateMemPercent(stat *docker.Stats) float64 {
+func calculateMemPercent(stat *types.StatsJSON) float64 {
 	var memPercent = 0.0
 	if stat.MemoryStats.Limit > 0 {
 		memPercent = float64(stat.MemoryStats.Usage) / float64(stat.MemoryStats.Limit) * 100.0
@@ -230,11 +343,11 @@ func calculateMemPercent(stat *docker.Stats) float64 {
 	return memPercent
 }
 
-func calculateCPUPercent(stat *docker.Stats) float64 {
+func calculateCPUPercent(stat *types.StatsJSON) float64 {
 	var cpuPercent = 0.0
 	// calculate the change for the cpu and system usage of the container in between readings
 	cpuDelta := float64(stat.CPUStats.CPUUsage.TotalUsage) - float64(stat.PreCPUStats.CPUUsage.TotalUsage)
-	systemDelta := float64(stat.CPUStats.SystemCPUUsage) - float64(stat.PreCPUStats.SystemCPUUsage)
+	systemDelta := float64(stat.CPUStats.SystemUsage) - float64(stat.PreCPUStats.SystemUsage)
 
 	if systemDelta > 0.0 && cpuDelta > 0.0 {
 		cpuPercent = (cpuDelta / systemDelta) * float64(len(stat.CPUStats.CPUUsage.PercpuUsage)) * 100.0
@@ -243,16 +356,17 @@ func calculateCPUPercent(stat *docker.Stats) float64 {
 }
 
 func gatherBlockIOMetrics(
-	stat *docker.Stats,
+	stat *types.StatsJSON,
 	acc telegraf.Accumulator,
 	tags map[string]string,
 	now time.Time,
+	id string,
 ) {
 	blkioStats := stat.BlkioStats
 	// Make a map of devices to their block io stats
 	deviceStatMap := make(map[string]map[string]interface{})
 
-	for _, metric := range blkioStats.IOServiceBytesRecursive {
+	for _, metric := range blkioStats.IoServiceBytesRecursive {
 		device := fmt.Sprintf("%d:%d", metric.Major, metric.Minor)
 		_, ok := deviceStatMap[device]
 		if !ok {
@@ -263,7 +377,7 @@ func gatherBlockIOMetrics(
 		deviceStatMap[device][field] = metric.Value
 	}
 
-	for _, metric := range blkioStats.IOServicedRecursive {
+	for _, metric := range blkioStats.IoServicedRecursive {
 		device := fmt.Sprintf("%d:%d", metric.Major, metric.Minor)
 		_, ok := deviceStatMap[device]
 		if !ok {
@@ -274,46 +388,45 @@ func gatherBlockIOMetrics(
 		deviceStatMap[device][field] = metric.Value
 	}
 
-	for _, metric := range blkioStats.IOQueueRecursive {
+	for _, metric := range blkioStats.IoQueuedRecursive {
 		device := fmt.Sprintf("%d:%d", metric.Major, metric.Minor)
 		field := fmt.Sprintf("io_queue_recursive_%s", strings.ToLower(metric.Op))
 		deviceStatMap[device][field] = metric.Value
 	}
 
-	for _, metric := range blkioStats.IOServiceTimeRecursive {
+	for _, metric := range blkioStats.IoServiceTimeRecursive {
 		device := fmt.Sprintf("%d:%d", metric.Major, metric.Minor)
 		field := fmt.Sprintf("io_service_time_recursive_%s", strings.ToLower(metric.Op))
 		deviceStatMap[device][field] = metric.Value
 	}
 
-	for _, metric := range blkioStats.IOWaitTimeRecursive {
+	for _, metric := range blkioStats.IoWaitTimeRecursive {
 		device := fmt.Sprintf("%d:%d", metric.Major, metric.Minor)
 		field := fmt.Sprintf("io_wait_time_%s", strings.ToLower(metric.Op))
 		deviceStatMap[device][field] = metric.Value
 	}
 
-	for _, metric := range blkioStats.IOMergedRecursive {
+	for _, metric := range blkioStats.IoMergedRecursive {
 		device := fmt.Sprintf("%d:%d", metric.Major, metric.Minor)
 		field := fmt.Sprintf("io_merged_recursive_%s", strings.ToLower(metric.Op))
 		deviceStatMap[device][field] = metric.Value
 	}
 
-	for _, metric := range blkioStats.IOTimeRecursive {
+	for _, metric := range blkioStats.IoTimeRecursive {
 		device := fmt.Sprintf("%d:%d", metric.Major, metric.Minor)
-		field := fmt.Sprintf("io_time_recursive_%s", strings.ToLower(metric.Op))
-		deviceStatMap[device][field] = metric.Value
+		deviceStatMap[device]["io_time_recursive"] = metric.Value
 	}
 
 	for _, metric := range blkioStats.SectorsRecursive {
 		device := fmt.Sprintf("%d:%d", metric.Major, metric.Minor)
-		field := fmt.Sprintf("sectors_recursive_%s", strings.ToLower(metric.Op))
-		deviceStatMap[device][field] = metric.Value
+		deviceStatMap[device]["sectors_recursive"] = metric.Value
 	}
 
 	for device, fields := range deviceStatMap {
 		iotags := copyTags(tags)
 		iotags["device"] = device
-		acc.AddFields("docker_blkio", fields, iotags, now)
+		fields["container_id"] = id
+		acc.AddFields("docker_container_blkio", fields, iotags, now)
 	}
 }
 
@@ -334,8 +447,31 @@ func sliceContains(in string, sl []string) bool {
 	return false
 }
 
+// Parses the human-readable size string into the amount it represents.
+func parseSize(sizeStr string) (int64, error) {
+	matches := sizeRegex.FindStringSubmatch(sizeStr)
+	if len(matches) != 4 {
+		return -1, fmt.Errorf("invalid size: '%s'", sizeStr)
+	}
+
+	size, err := strconv.ParseFloat(matches[1], 64)
+	if err != nil {
+		return -1, err
+	}
+
+	uMap := map[string]int64{"k": KB, "m": MB, "g": GB, "t": TB, "p": PB}
+	unitPrefix := strings.ToLower(matches[3])
+	if mul, ok := uMap[unitPrefix]; ok {
+		size *= float64(mul)
+	}
+
+	return int64(size), nil
+}
+
 func init() {
 	inputs.Add("docker", func() telegraf.Input {
-		return &Docker{}
+		return &Docker{
+			Timeout: internal.Duration{Duration: time.Second * 5},
+		}
 	})
 }

plugins/inputs/docker/docker_test.go

@@ -1,12 +1,19 @@
 package system
 
 import (
+	"io"
+	"io/ioutil"
+	"strings"
 	"testing"
 	"time"
 
+	"golang.org/x/net/context"
+
+	"github.com/docker/engine-api/types"
+	"github.com/docker/engine-api/types/registry"
 	"github.com/influxdata/telegraf/testutil"
 
-	"github.com/fsouza/go-dockerclient"
+	"github.com/stretchr/testify/require"
 )
 
 func TestDockerGatherContainerStats(t *testing.T) {
@@ -14,26 +21,26 @@ func TestDockerGatherContainerStats(t *testing.T) {
 	stats := testStats()
 
 	tags := map[string]string{
-		"cont_id":    "foobarbaz",
-		"cont_name":  "redis",
-		"cont_image": "redis/image",
+		"container_name":  "redis",
+		"container_image": "redis/image",
 	}
-	gatherContainerStats(stats, &acc, tags)
+	gatherContainerStats(stats, &acc, tags, "123456789")
 
-	// test docker_net measurement
+	// test docker_container_net measurement
 	netfields := map[string]interface{}{
-		"rx_dropped": uint64(1),
-		"rx_bytes":   uint64(2),
-		"rx_errors":  uint64(3),
-		"tx_packets": uint64(4),
-		"tx_dropped": uint64(1),
-		"rx_packets": uint64(2),
-		"tx_errors":  uint64(3),
-		"tx_bytes":   uint64(4),
+		"rx_dropped":   uint64(1),
+		"rx_bytes":     uint64(2),
+		"rx_errors":    uint64(3),
+		"tx_packets":   uint64(4),
+		"tx_dropped":   uint64(1),
+		"rx_packets":   uint64(2),
+		"tx_errors":    uint64(3),
+		"tx_bytes":     uint64(4),
+		"container_id": "123456789",
 	}
 	nettags := copyTags(tags)
 	nettags["network"] = "eth0"
-	acc.AssertContainsTaggedFields(t, "docker_net", netfields, nettags)
+	acc.AssertContainsTaggedFields(t, "docker_container_net", netfields, nettags)
 
 	// test docker_blkio measurement
 	blkiotags := copyTags(tags)
@@ -41,10 +48,11 @@ func TestDockerGatherContainerStats(t *testing.T) {
 	blkiofields := map[string]interface{}{
 		"io_service_bytes_recursive_read": uint64(100),
 		"io_serviced_recursive_write":     uint64(101),
+		"container_id":                    "123456789",
 	}
-	acc.AssertContainsTaggedFields(t, "docker_blkio", blkiofields, blkiotags)
+	acc.AssertContainsTaggedFields(t, "docker_container_blkio", blkiofields, blkiotags)
 
-	// test docker_mem measurement
+	// test docker_container_mem measurement
 	memfields := map[string]interface{}{
 		"max_usage":                 uint64(1001),
 		"usage":                     uint64(1111),
@@ -80,11 +88,12 @@ func TestDockerGatherContainerStats(t *testing.T) {
 		"inactive_file":             uint64(3),
 		"total_pgpgin":              uint64(4),
 		"usage_percent":             float64(55.55),
+		"container_id":              "123456789",
 	}
 
-	acc.AssertContainsTaggedFields(t, "docker_mem", memfields, tags)
+	acc.AssertContainsTaggedFields(t, "docker_container_mem", memfields, tags)
 
-	// test docker_cpu measurement
+	// test docker_container_cpu measurement
 	cputags := copyTags(tags)
 	cputags["cpu"] = "cpu-total"
 	cpufields := map[string]interface{}{
@@ -96,74 +105,77 @@ func TestDockerGatherContainerStats(t *testing.T) {
 		"throttling_throttled_periods": uint64(0),
 		"throttling_throttled_time":    uint64(0),
 		"usage_percent":                float64(400.0),
+		"container_id":                 "123456789",
 	}
-	acc.AssertContainsTaggedFields(t, "docker_cpu", cpufields, cputags)
+	acc.AssertContainsTaggedFields(t, "docker_container_cpu", cpufields, cputags)
 
 	cputags["cpu"] = "cpu0"
 	cpu0fields := map[string]interface{}{
-		"usage_total": uint64(1),
+		"usage_total":  uint64(1),
+		"container_id": "123456789",
 	}
-	acc.AssertContainsTaggedFields(t, "docker_cpu", cpu0fields, cputags)
+	acc.AssertContainsTaggedFields(t, "docker_container_cpu", cpu0fields, cputags)
 
 	cputags["cpu"] = "cpu1"
 	cpu1fields := map[string]interface{}{
-		"usage_total": uint64(1002),
+		"usage_total":  uint64(1002),
+		"container_id": "123456789",
 	}
-	acc.AssertContainsTaggedFields(t, "docker_cpu", cpu1fields, cputags)
+	acc.AssertContainsTaggedFields(t, "docker_container_cpu", cpu1fields, cputags)
 }
 
-func testStats() *docker.Stats {
-	stats := &docker.Stats{
-		Read:     time.Now(),
-		Networks: make(map[string]docker.NetworkStats),
-	}
+func testStats() *types.StatsJSON {
+	stats := &types.StatsJSON{}
+	stats.Read = time.Now()
+	stats.Networks = make(map[string]types.NetworkStats)
 
 	stats.CPUStats.CPUUsage.PercpuUsage = []uint64{1, 1002}
 	stats.CPUStats.CPUUsage.UsageInUsermode = 100
 	stats.CPUStats.CPUUsage.TotalUsage = 500
 	stats.CPUStats.CPUUsage.UsageInKernelmode = 200
-	stats.CPUStats.SystemCPUUsage = 100
+	stats.CPUStats.SystemUsage = 100
 	stats.CPUStats.ThrottlingData.Periods = 1
 
 	stats.PreCPUStats.CPUUsage.TotalUsage = 400
-	stats.PreCPUStats.SystemCPUUsage = 50
+	stats.PreCPUStats.SystemUsage = 50
 
-	stats.MemoryStats.Stats.TotalPgmafault = 0
-	stats.MemoryStats.Stats.Cache = 0
-	stats.MemoryStats.Stats.MappedFile = 0
-	stats.MemoryStats.Stats.TotalInactiveFile = 0
-	stats.MemoryStats.Stats.Pgpgout = 0
-	stats.MemoryStats.Stats.Rss = 0
-	stats.MemoryStats.Stats.TotalMappedFile = 0
-	stats.MemoryStats.Stats.Writeback = 0
-	stats.MemoryStats.Stats.Unevictable = 0
-	stats.MemoryStats.Stats.Pgpgin = 0
-	stats.MemoryStats.Stats.TotalUnevictable = 0
-	stats.MemoryStats.Stats.Pgmajfault = 0
-	stats.MemoryStats.Stats.TotalRss = 44
-	stats.MemoryStats.Stats.TotalRssHuge = 444
-	stats.MemoryStats.Stats.TotalWriteback = 55
-	stats.MemoryStats.Stats.TotalInactiveAnon = 0
-	stats.MemoryStats.Stats.RssHuge = 0
-	stats.MemoryStats.Stats.HierarchicalMemoryLimit = 0
-	stats.MemoryStats.Stats.TotalPgfault = 0
-	stats.MemoryStats.Stats.TotalActiveFile = 0
-	stats.MemoryStats.Stats.ActiveAnon = 0
-	stats.MemoryStats.Stats.TotalActiveAnon = 0
-	stats.MemoryStats.Stats.TotalPgpgout = 0
-	stats.MemoryStats.Stats.TotalCache = 0
-	stats.MemoryStats.Stats.InactiveAnon = 0
-	stats.MemoryStats.Stats.ActiveFile = 1
-	stats.MemoryStats.Stats.Pgfault = 2
-	stats.MemoryStats.Stats.InactiveFile = 3
-	stats.MemoryStats.Stats.TotalPgpgin = 4
+	stats.MemoryStats.Stats = make(map[string]uint64)
+	stats.MemoryStats.Stats["total_pgmajfault"] = 0
+	stats.MemoryStats.Stats["cache"] = 0
+	stats.MemoryStats.Stats["mapped_file"] = 0
+	stats.MemoryStats.Stats["total_inactive_file"] = 0
+	stats.MemoryStats.Stats["pagpgout"] = 0
+	stats.MemoryStats.Stats["rss"] = 0
+	stats.MemoryStats.Stats["total_mapped_file"] = 0
+	stats.MemoryStats.Stats["writeback"] = 0
+	stats.MemoryStats.Stats["unevictable"] = 0
+	stats.MemoryStats.Stats["pgpgin"] = 0
+	stats.MemoryStats.Stats["total_unevictable"] = 0
+	stats.MemoryStats.Stats["pgmajfault"] = 0
+	stats.MemoryStats.Stats["total_rss"] = 44
+	stats.MemoryStats.Stats["total_rss_huge"] = 444
+	stats.MemoryStats.Stats["total_write_back"] = 55
+	stats.MemoryStats.Stats["total_inactive_anon"] = 0
+	stats.MemoryStats.Stats["rss_huge"] = 0
+	stats.MemoryStats.Stats["hierarchical_memory_limit"] = 0
+	stats.MemoryStats.Stats["total_pgfault"] = 0
+	stats.MemoryStats.Stats["total_active_file"] = 0
+	stats.MemoryStats.Stats["active_anon"] = 0
+	stats.MemoryStats.Stats["total_active_anon"] = 0
+	stats.MemoryStats.Stats["total_pgpgout"] = 0
+	stats.MemoryStats.Stats["total_cache"] = 0
+	stats.MemoryStats.Stats["inactive_anon"] = 0
+	stats.MemoryStats.Stats["active_file"] = 1
+	stats.MemoryStats.Stats["pgfault"] = 2
+	stats.MemoryStats.Stats["inactive_file"] = 3
+	stats.MemoryStats.Stats["total_pgpgin"] = 4
 
 	stats.MemoryStats.MaxUsage = 1001
 	stats.MemoryStats.Usage = 1111
 	stats.MemoryStats.Failcnt = 1
 	stats.MemoryStats.Limit = 2000
 
-	stats.Networks["eth0"] = docker.NetworkStats{
+	stats.Networks["eth0"] = types.NetworkStats{
 		RxDropped: 1,
 		RxBytes:   2,
 		RxErrors:  3,
@@ -174,23 +186,247 @@ func testStats() *docker.Stats {
 		TxBytes:   4,
 	}
 
-	sbr := docker.BlkioStatsEntry{
+	sbr := types.BlkioStatEntry{
 		Major: 6,
 		Minor: 0,
 		Op:    "read",
 		Value: 100,
 	}
-	sr := docker.BlkioStatsEntry{
+	sr := types.BlkioStatEntry{
 		Major: 6,
 		Minor: 0,
 		Op:    "write",
 		Value: 101,
 	}
 
-	stats.BlkioStats.IOServiceBytesRecursive = append(
-		stats.BlkioStats.IOServiceBytesRecursive, sbr)
-	stats.BlkioStats.IOServicedRecursive = append(
-		stats.BlkioStats.IOServicedRecursive, sr)
+	stats.BlkioStats.IoServiceBytesRecursive = append(
+		stats.BlkioStats.IoServiceBytesRecursive, sbr)
+	stats.BlkioStats.IoServicedRecursive = append(
+		stats.BlkioStats.IoServicedRecursive, sr)
 
 	return stats
 }
+
+type FakeDockerClient struct {
+}
+
+func (d FakeDockerClient) Info(ctx context.Context) (types.Info, error) {
+	env := types.Info{
+		Containers:         108,
+		OomKillDisable:     false,
+		SystemTime:         "2016-02-24T00:55:09.15073105-05:00",
+		NEventsListener:    0,
+		ID:                 "5WQQ:TFWR:FDNG:OKQ3:37Y4:FJWG:QIKK:623T:R3ME:QTKB:A7F7:OLHD",
+		Debug:              false,
+		LoggingDriver:      "json-file",
+		KernelVersion:      "4.3.0-1-amd64",
+		IndexServerAddress: "https://index.docker.io/v1/",
+		MemTotal:           3840757760,
+		Images:             199,
+		CPUCfsQuota:        true,
+		Name:               "absol",
+		SwapLimit:          false,
+		IPv4Forwarding:     true,
+		ExecutionDriver:    "native-0.2",
+		ExperimentalBuild:  false,
+		CPUCfsPeriod:       true,
+		RegistryConfig: &registry.ServiceConfig{
+			IndexConfigs: map[string]*registry.IndexInfo{
+				"docker.io": {
+					Name:     "docker.io",
+					Mirrors:  []string{},
+					Official: true,
+					Secure:   true,
+				},
+			}, InsecureRegistryCIDRs: []*registry.NetIPNet{{IP: []byte{127, 0, 0, 0}, Mask: []byte{255, 0, 0, 0}}}, Mirrors: []string{}},
+		OperatingSystem:   "Linux Mint LMDE (containerized)",
+		BridgeNfIptables:  true,
+		HTTPSProxy:        "",
+		Labels:            []string{},
+		MemoryLimit:       false,
+		DriverStatus:      [][2]string{{"Pool Name", "docker-8:1-1182287-pool"}, {"Pool Blocksize", "65.54 kB"}, {"Backing Filesystem", "extfs"}, {"Data file", "/dev/loop0"}, {"Metadata file", "/dev/loop1"}, {"Data Space Used", "17.3 GB"}, {"Data Space Total", "107.4 GB"}, {"Data Space Available", "36.53 GB"}, {"Metadata Space Used", "20.97 MB"}, {"Metadata Space Total", "2.147 GB"}, {"Metadata Space Available", "2.127 GB"}, {"Udev Sync Supported", "true"}, {"Deferred Removal Enabled", "false"}, {"Data loop file", "/var/lib/docker/devicemapper/devicemapper/data"}, {"Metadata loop file", "/var/lib/docker/devicemapper/devicemapper/metadata"}, {"Library Version", "1.02.115 (2016-01-25)"}},
+		NFd:               19,
+		HTTPProxy:         "",
+		Driver:            "devicemapper",
+		NGoroutines:       39,
+		NCPU:              4,
+		DockerRootDir:     "/var/lib/docker",
+		NoProxy:           "",
+		BridgeNfIP6tables: true,
+	}
+	return env, nil
+}
+
+func (d FakeDockerClient) ContainerList(octx context.Context, options types.ContainerListOptions) ([]types.Container, error) {
+	container1 := types.Container{
+		ID:      "e2173b9478a6ae55e237d4d74f8bbb753f0817192b5081334dc78476296b7dfb",
+		Names:   []string{"/etcd"},
+		Image:   "quay.io/coreos/etcd:v2.2.2",
+		Command: "/etcd -name etcd0 -advertise-client-urls http://localhost:2379 -listen-client-urls http://0.0.0.0:2379",
+		Created: 1455941930,
+		Status:  "Up 4 hours",
+		Ports: []types.Port{
+			types.Port{
+				PrivatePort: 7001,
+				PublicPort:  0,
+				Type:        "tcp",
+			},
+			types.Port{
+				PrivatePort: 4001,
+				PublicPort:  0,
+				Type:        "tcp",
+			},
+			types.Port{
+				PrivatePort: 2380,
+				PublicPort:  0,
+				Type:        "tcp",
+			},
+			types.Port{
+				PrivatePort: 2379,
+				PublicPort:  2379,
+				Type:        "tcp",
+				IP:          "0.0.0.0",
+			},
+		},
+		SizeRw:     0,
+		SizeRootFs: 0,
+	}
+	container2 := types.Container{
+		ID:      "b7dfbb9478a6ae55e237d4d74f8bbb753f0817192b5081334dc78476296e2173",
+		Names:   []string{"/etcd2"},
+		Image:   "quay.io/coreos/etcd:v2.2.2",
+		Command: "/etcd -name etcd2 -advertise-client-urls http://localhost:2379 -listen-client-urls http://0.0.0.0:2379",
+		Created: 1455941933,
+		Status:  "Up 4 hours",
+		Ports: []types.Port{
+			types.Port{
+				PrivatePort: 7002,
+				PublicPort:  0,
+				Type:        "tcp",
+			},
+			types.Port{
+				PrivatePort: 4002,
+				PublicPort:  0,
+				Type:        "tcp",
+			},
+			types.Port{
+				PrivatePort: 2381,
+				PublicPort:  0,
+				Type:        "tcp",
+			},
+			types.Port{
+				PrivatePort: 2382,
+				PublicPort:  2382,
+				Type:        "tcp",
+				IP:          "0.0.0.0",
+			},
+		},
+		SizeRw:     0,
+		SizeRootFs: 0,
+	}
+
+	containers := []types.Container{container1, container2}
+	return containers, nil
+
+	//#{e6a96c84ca91a5258b7cb752579fb68826b68b49ff957487695cd4d13c343b44 titilambert/snmpsim /bin/sh -c 'snmpsimd --agent-udpv4-endpoint=0.0.0.0:31161 --process-user=root --process-group=user' 1455724831 Up 4 hours [{31161 31161 udp 0.0.0.0}] 0 0 [/snmp] map[]}]2016/02/24 01:05:01 Gathered metrics, (3s interval), from 1 inputs in 1.233836656s
+}
+
+func (d FakeDockerClient) ContainerStats(ctx context.Context, containerID string, stream bool) (io.ReadCloser, error) {
+	var stat io.ReadCloser
+	jsonStat := `{"read":"2016-02-24T11:42:27.472459608-05:00","memory_stats":{"stats":{},"limit":18935443456},"blkio_stats":{"io_service_bytes_recursive":[{"major":252,"minor":1,"op":"Read","value":753664},{"major":252,"minor":1,"op":"Write"},{"major":252,"minor":1,"op":"Sync"},{"major":252,"minor":1,"op":"Async","value":753664},{"major":252,"minor":1,"op":"Total","value":753664}],"io_serviced_recursive":[{"major":252,"minor":1,"op":"Read","value":26},{"major":252,"minor":1,"op":"Write"},{"major":252,"minor":1,"op":"Sync"},{"major":252,"minor":1,"op":"Async","value":26},{"major":252,"minor":1,"op":"Total","value":26}]},"cpu_stats":{"cpu_usage":{"percpu_usage":[17871,4959158,1646137,1231652,11829401,244656,369972,0],"usage_in_usermode":10000000,"total_usage":20298847},"system_cpu_usage":24052607520000000,"throttling_data":{}},"precpu_stats":{"cpu_usage":{"percpu_usage":[17871,4959158,1646137,1231652,11829401,244656,369972,0],"usage_in_usermode":10000000,"total_usage":20298847},"system_cpu_usage":24052599550000000,"throttling_data":{}}}`
+	stat = ioutil.NopCloser(strings.NewReader(jsonStat))
+	return stat, nil
+}
+
+func TestDockerGatherInfo(t *testing.T) {
+	var acc testutil.Accumulator
+	client := FakeDockerClient{}
+	d := Docker{client: client}
+
+	err := d.Gather(&acc)
+
+	require.NoError(t, err)
+
+	acc.AssertContainsTaggedFields(t,
+		"docker",
+		map[string]interface{}{
+			"n_listener_events":       int(0),
+			"n_cpus":                  int(4),
+			"n_used_file_descriptors": int(19),
+			"n_containers":            int(108),
+			"n_images":                int(199),
+			"n_goroutines":            int(39),
+		},
+		map[string]string{},
+	)
+
+	acc.AssertContainsTaggedFields(t,
+		"docker_data",
+		map[string]interface{}{
+			"used":      int64(17300000000),
+			"total":     int64(107400000000),
+			"available": int64(36530000000),
+		},
+		map[string]string{
+			"unit": "bytes",
+		},
+	)
+	acc.AssertContainsTaggedFields(t,
+		"docker_container_cpu",
+		map[string]interface{}{
+			"usage_total":  uint64(1231652),
+			"container_id": "b7dfbb9478a6ae55e237d4d74f8bbb753f0817192b5081334dc78476296e2173",
+		},
+		map[string]string{
+			"container_name":  "etcd2",
+			"container_image": "quay.io/coreos/etcd:v2.2.2",
+			"cpu":             "cpu3",
+		},
+	)
+	acc.AssertContainsTaggedFields(t,
+		"docker_container_mem",
+		map[string]interface{}{
+			"total_pgpgout":             uint64(0),
+			"usage_percent":             float64(0),
+			"rss":                       uint64(0),
+			"total_writeback":           uint64(0),
+			"active_anon":               uint64(0),
+			"total_pgmafault":           uint64(0),
+			"total_rss":                 uint64(0),
+			"total_unevictable":         uint64(0),
+			"active_file":               uint64(0),
+			"total_mapped_file":         uint64(0),
+			"pgpgin":                    uint64(0),
+			"total_active_file":         uint64(0),
+			"total_active_anon":         uint64(0),
+			"total_cache":               uint64(0),
+			"inactive_anon":             uint64(0),
+			"pgmajfault":                uint64(0),
+			"total_inactive_anon":       uint64(0),
+			"total_rss_huge":            uint64(0),
+			"rss_huge":                  uint64(0),
+			"hierarchical_memory_limit": uint64(0),
+			"pgpgout":                   uint64(0),
+			"unevictable":               uint64(0),
+			"total_inactive_file":       uint64(0),
+			"writeback":                 uint64(0),
+			"total_pgfault":             uint64(0),
+			"total_pgpgin":              uint64(0),
+			"cache":                     uint64(0),
+			"mapped_file":               uint64(0),
+			"inactive_file":             uint64(0),
+			"max_usage":                 uint64(0),
+			"fail_count":                uint64(0),
+			"pgfault":                   uint64(0),
+			"usage":                     uint64(0),
+			"limit":                     uint64(18935443456),
+			"container_id":              "b7dfbb9478a6ae55e237d4d74f8bbb753f0817192b5081334dc78476296e2173",
+		},
+		map[string]string{
+			"container_name":  "etcd2",
+			"container_image": "quay.io/coreos/etcd:v2.2.2",
+		},
+	)
+
+	//fmt.Print(info)
+}

plugins/inputs/dovecot/README.md

@@ -10,20 +10,25 @@ domains. You can read Dovecot's documentation
 ```
 # Read metrics about dovecot servers
 [[inputs.dovecot]]
-  # Dovecot servers
-  #  specify dovecot servers via an address:port list
-  #  e.g.
-  #    localhost:24242
-  #
-  # If no servers are specified, then localhost is used as the host.
+  ## specify dovecot servers via an address:port list
+  ##  e.g.
+  ##    localhost:24242
+  ##
+  ## If no servers are specified, then localhost is used as the host.
   servers = ["localhost:24242"]
-  # Only collect metrics for these domains, collect all if empty
-  domains = []
+  ## Type is one of "user", "domain", "ip", or "global"
+  type = "global"
+  ## Wildcard matches like "*.com". An empty string "" is same as "*"
+  ## If type = "ip" filters should be <IP/network>
+  filters = [""]
 ```
 
 
 ### Tags:
 	server: hostname
+	type: query type
+	ip: ip addr
+	user: username
 	domain: domain name
 
 
@@ -33,7 +38,7 @@ domains. You can read Dovecot's documentation
 	last_update            time.Time
 	num_logins             int64
 	num_cmds               int64
-	num_connected_sessions int64
+	num_connected_sessions int64				## not in <user> type
 	user_cpu               float32
 	sys_cpu                float32
 	clock_time             float64
@@ -57,11 +62,13 @@ domains. You can read Dovecot's documentation
 ### Example Output:
 
 ```
-telegraf -config telegraf.cfg -input-filter dovecot -test
+telegraf -config t.cfg -input-filter dovecot -test
 * Plugin: dovecot, Collection 1
-> dovecot,domain=xxxxx.it,server=dovecot--1.mail.sys clock_time=12105746411632.5,disk_input=115285225472i,disk_output=4885067755520i,invol_cs=169701886i,last_update="2016-02-09 08:49:47.000014113 +0100 CET",mail_cache_hits=441828i,mail_lookup_attr=0i,mail_lookup_path=25323i,mail_read_bytes=241188145i,mail_read_count=11719i,maj_faults=3168i,min_faults=321438988i,num_cmds=51635i,num_connected_sessions=2i,num_logins=17149i,read_bytes=7939026951110i,read_count=3716991752i,reset_timestamp="2016-01-28 09:34:36 +0100 CET",sys_cpu=222595.288,user_cpu=267468.08,vol_cs=3288715920i,write_bytes=4483648967059i,write_count=1640646952i 1455004219924838345
-> dovecot,domain=yyyyy.com,server=dovecot-1.mail.sys clock_time=6650794455331782,disk_input=61957695569920i,disk_output=2638244004487168i,invol_cs=2004805041i,last_update="2016-02-09 08:49:49.000251296 +0100 CET",mail_cache_hits=2499112513i,mail_lookup_attr=506730i,mail_lookup_path=39128227i,mail_read_bytes=1076496874501i,mail_read_count=32615262i,maj_faults=1643304i,min_faults=4216116325i,num_cmds=85785559i,num_connected_sessions=1177i,num_logins=11658255i,read_bytes=4289150974554145i,read_count=1112000703i,reset_timestamp="2016-01-28 09:31:26 +0100 CET",sys_cpu=121125923.032,user_cpu=145561336.428,vol_cs=205451885i,write_bytes=2420130526835796i,write_count=2991367252i 1455004219925152529
-> dovecot,domain=xxxxx.it,server=dovecot-2.mail.sys clock_time=10710826586999.143,disk_input=79792410624i,disk_output=4496066158592i,invol_cs=150426876i,last_update="2016-02-09 08:48:19.000209134 +0100 CET",mail_cache_hits=5480869i,mail_lookup_attr=0i,mail_lookup_path=122563i,mail_read_bytes=340746273i,mail_read_count=44275i,maj_faults=1722i,min_faults=288071875i,num_cmds=50098i,num_connected_sessions=0i,num_logins=16389i,read_bytes=7259551999517i,read_count=3396625369i,reset_timestamp="2016-01-28 09:31:29 +0100 CET",sys_cpu=200762.792,user_cpu=242477.664,vol_cs=2996657358i,write_bytes=4133381575263i,write_count=1497242759i 1455004219924888283
-> dovecot,domain=yyyyy.com,server=dovecot-2.mail.sys clock_time=6522131245483702,disk_input=48259150004224i,disk_output=2754333359087616i,invol_cs=2294595260i,last_update="2016-02-09 08:49:49.000251919 +0100 CET",mail_cache_hits=2139113611i,mail_lookup_attr=520276i,mail_lookup_path=37940318i,mail_read_bytes=1088002215022i,mail_read_count=31350271i,maj_faults=994420i,min_faults=1486260543i,num_cmds=40414997i,num_connected_sessions=978i,num_logins=11259672i,read_bytes=4445546612487315i,read_count=1763534543i,reset_timestamp="2016-01-28 09:31:24 +0100 CET",sys_cpu=123655962.668,user_cpu=149259327.032,vol_cs=4215130546i,write_bytes=2531186030222761i,write_count=2186579650i 1455004219925398372
-```
-
+> dovecot,ip=192.168.0.1,server=dovecot-1.domain.test,type=ip clock_time=0,disk_input=0i,disk_output=0i,invol_cs=0i,last_update="2016-04-08 10:59:47.000208479 +0200 CEST",mail_cache_hits=0i,mail_lookup_attr=0i,mail_lookup_path=0i,mail_read_bytes=0i,mail_read_count=0i,maj_faults=0i,min_faults=0i,num_cmds=12i,num_connected_sessions=0i,num_logins=6i,read_bytes=0i,read_count=0i,reset_timestamp="2016-04-08 10:33:34 +0200 CEST",sys_cpu=0,user_cpu=0,vol_cs=0i,write_bytes=0i,write_count=0i 1460106251633824223
+* Plugin: dovecot, Collection 1
+> dovecot,server=dovecot-1.domain.test,type=user,user=user-1@domain.test clock_time=0.00006,disk_input=405504i,disk_output=77824i,invol_cs=67i,last_update="2016-04-08 11:02:55.000111634 +0200 CEST",mail_cache_hits=26i,mail_lookup_attr=0i,mail_lookup_path=6i,mail_read_bytes=86233i,mail_read_count=5i,maj_faults=0i,min_faults=975i,num_cmds=41i,num_logins=3i,read_bytes=368833i,read_count=394i,reset_timestamp="2016-04-08 11:01:32 +0200 CEST",sys_cpu=0.008,user_cpu=0.004,vol_cs=323i,write_bytes=105086i,write_count=176i 1460106256637049167
+* Plugin: dovecot, Collection 1
+> dovecot,domain=domain.test,server=dovecot-1.domain.test,type=domain clock_time=100896189179847.7,disk_input=6467588263936i,disk_output=17933680439296i,invol_cs=1194808498i,last_update="2016-04-08 11:04:08.000377367 +0200 CEST",mail_cache_hits=46455781i,mail_lookup_attr=0i,mail_lookup_path=571490i,mail_read_bytes=79287033067i,mail_read_count=491243i,maj_faults=16992i,min_faults=1278442541i,num_cmds=606005i,num_connected_sessions=6597i,num_logins=166381i,read_bytes=30231409780721i,read_count=1624912080i,reset_timestamp="2016-04-08 10:28:45 +0200 CEST",sys_cpu=156440.372,user_cpu=216676.476,vol_cs=2749291157i,write_bytes=17097106707594i,write_count=944448998i 1460106261639672622
+* Plugin: dovecot, Collection 1
+> dovecot,server=dovecot-1.domain.test,type=global clock_time=101196971074203.94,disk_input=6493168218112i,disk_output=17978638815232i,invol_cs=1198855447i,last_update="2016-04-08 11:04:13.000379245 +0200 CEST",mail_cache_hits=68192209i,mail_lookup_attr=0i,mail_lookup_path=653861i,mail_read_bytes=86705151847i,mail_read_count=566125i,maj_faults=17208i,min_faults=1286179702i,num_cmds=917469i,num_connected_sessions=8896i,num_logins=174827i,read_bytes=30327690466186i,read_count=1772396430i,reset_timestamp="2016-04-08 10:28:45 +0200 CEST",sys_cpu=157965.692,user_cpu=219337.48,vol_cs=2827615787i,write_bytes=17150837661940i,write_count=992653220i 1460106266642153907
+```

plugins/inputs/dovecot/dovecot.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"fmt"
 	"io"
+	//	"log"
 	"net"
 	"strconv"
 	"strings"
@@ -15,8 +16,9 @@ import (
 )
 
 type Dovecot struct {
+	Type    string
+	Filters []string
 	Servers []string
-	Domains []string
 }
 
 func (d *Dovecot) Description() string {
@@ -30,10 +32,19 @@ var sampleConfig = `
   ##
   ## If no servers are specified, then localhost is used as the host.
   servers = ["localhost:24242"]
-  ## Only collect metrics for these domains, collect all if empty
-  domains = []
+  ## Type is one of "user", "domain", "ip", or "global"
+  type = "global"
+  ## Wildcard matches like "*.com". An empty string "" is same as "*"
+  ## If type = "ip" filters should be <IP/network>
+  filters = [""]
 `
 
+var defaultTimeout = time.Second * time.Duration(5)
+
+var validQuery = map[string]bool{
+	"user": true, "domain": true, "global": true, "ip": true,
+}
+
 func (d *Dovecot) SampleConfig() string { return sampleConfig }
 
 const defaultPort = "24242"
@@ -41,6 +52,11 @@ const defaultPort = "24242"
 // Reads stats from all configured servers.
 func (d *Dovecot) Gather(acc telegraf.Accumulator) error {
 
+	if !validQuery[d.Type] {
+		return fmt.Errorf("Error: %s is not a valid query type\n",
+			d.Type)
+	}
+
 	if len(d.Servers) == 0 {
 		d.Servers = append(d.Servers, "127.0.0.1:24242")
 	}
@@ -49,18 +65,18 @@ func (d *Dovecot) Gather(acc telegraf.Accumulator) error {
 
 	var outerr error
 
-	var domains = make(map[string]bool)
-
-	for _, dom := range d.Domains {
-		domains[dom] = true
+	if len(d.Filters) <= 0 {
+		d.Filters = append(d.Filters, "")
 	}
 
 	for _, serv := range d.Servers {
-		wg.Add(1)
-		go func(serv string) {
-			defer wg.Done()
-			outerr = d.gatherServer(serv, acc, domains)
-		}(serv)
+		for _, filter := range d.Filters {
+			wg.Add(1)
+			go func(serv string, filter string) {
+				defer wg.Done()
+				outerr = d.gatherServer(serv, acc, d.Type, filter)
+			}(serv, filter)
+		}
 	}
 
 	wg.Wait()
@@ -68,29 +84,38 @@ func (d *Dovecot) Gather(acc telegraf.Accumulator) error {
 	return outerr
 }
 
-func (d *Dovecot) gatherServer(addr string, acc telegraf.Accumulator, doms map[string]bool) error {
+func (d *Dovecot) gatherServer(addr string, acc telegraf.Accumulator, qtype string, filter string) error {
+
 	_, _, err := net.SplitHostPort(addr)
 	if err != nil {
 		return fmt.Errorf("Error: %s on url %s\n", err, addr)
 	}
 
-	c, err := net.Dial("tcp", addr)
+	c, err := net.DialTimeout("tcp", addr, defaultTimeout)
 	if err != nil {
 		return fmt.Errorf("Unable to connect to dovecot server '%s': %s", addr, err)
 	}
 	defer c.Close()
 
-	c.Write([]byte("EXPORT\tdomain\n\n"))
+	// Extend connection
+	c.SetDeadline(time.Now().Add(defaultTimeout))
+
+	msg := fmt.Sprintf("EXPORT\t%s", qtype)
+	if len(filter) > 0 {
+		msg += fmt.Sprintf("\t%s=%s", qtype, filter)
+	}
+	msg += "\n"
+
+	c.Write([]byte(msg))
 	var buf bytes.Buffer
 	io.Copy(&buf, c)
-	//	buf := bufio.NewReader(c)
 
 	host, _, _ := net.SplitHostPort(addr)
 
-	return gatherStats(&buf, acc, doms, host)
+	return gatherStats(&buf, acc, host, qtype)
 }
 
-func gatherStats(buf *bytes.Buffer, acc telegraf.Accumulator, doms map[string]bool, host string) error {
+func gatherStats(buf *bytes.Buffer, acc telegraf.Accumulator, host string, qtype string) error {
 
 	lines := strings.Split(buf.String(), "\n")
 	head := strings.Split(lines[0], "\t")
@@ -101,16 +126,18 @@ func gatherStats(buf *bytes.Buffer, acc telegraf.Accumulator, doms map[string]bo
 			continue
 		}
 		val := strings.Split(vals[i], "\t")
+
 		fields := make(map[string]interface{})
-		if len(doms) > 0 && !doms[val[0]] {
-			continue
+		tags := map[string]string{"server": host, "type": qtype}
+
+		if qtype != "global" {
+			tags[qtype] = val[0]
 		}
-		tags := map[string]string{"server": host, "domain": val[0]}
+
 		for n := range val {
 			switch head[n] {
-			case "domain":
+			case qtype:
 				continue
-				//				fields[head[n]] = val[n]
 			case "user_cpu", "sys_cpu", "clock_time":
 				fields[head[n]] = secParser(val[n])
 			case "reset_timestamp", "last_update":

plugins/inputs/dovecot/dovecot_test.go

@@ -15,17 +15,6 @@ func TestDovecot(t *testing.T) {
 		t.Skip("Skipping integration test in short mode")
 	}
 
-	var acc testutil.Accumulator
-	tags := map[string]string{"server": "dovecot.test", "domain": "domain.test"}
-	buf := bytes.NewBufferString(sampleStats)
-
-	var doms = map[string]bool{
-		"domain.test": true,
-	}
-
-	err := gatherStats(buf, &acc, doms, "dovecot.test")
-	require.NoError(t, err)
-
 	fields := map[string]interface{}{
 		"reset_timestamp":        time.Unix(1453969886, 0),
 		"last_update":            time.Unix(1454603963, 39864),
@@ -52,10 +41,79 @@ func TestDovecot(t *testing.T) {
 		"mail_cache_hits":        int64(1557255080),
 	}
 
+	var acc testutil.Accumulator
+
+	// Test type=global
+	tags := map[string]string{"server": "dovecot.test", "type": "global"}
+	buf := bytes.NewBufferString(sampleGlobal)
+
+	err := gatherStats(buf, &acc, "dovecot.test", "global")
+	require.NoError(t, err)
+
+	acc.AssertContainsTaggedFields(t, "dovecot", fields, tags)
+
+	// Test type=domain
+	tags = map[string]string{"server": "dovecot.test", "type": "domain", "domain": "domain.test"}
+	buf = bytes.NewBufferString(sampleDomain)
+
+	err = gatherStats(buf, &acc, "dovecot.test", "domain")
+	require.NoError(t, err)
+
+	acc.AssertContainsTaggedFields(t, "dovecot", fields, tags)
+
+	// Test type=ip
+	tags = map[string]string{"server": "dovecot.test", "type": "ip", "ip": "192.168.0.100"}
+	buf = bytes.NewBufferString(sampleIp)
+
+	err = gatherStats(buf, &acc, "dovecot.test", "ip")
+	require.NoError(t, err)
+
+	acc.AssertContainsTaggedFields(t, "dovecot", fields, tags)
+
+	// Test type=user
+	fields = map[string]interface{}{
+		"reset_timestamp":  time.Unix(1453969886, 0),
+		"last_update":      time.Unix(1454603963, 39864),
+		"num_logins":       int64(7503897),
+		"num_cmds":         int64(52595715),
+		"user_cpu":         1.00831175372e+08,
+		"sys_cpu":          8.3849071112e+07,
+		"clock_time":       4.3260019315281835e+15,
+		"min_faults":       int64(763950011),
+		"maj_faults":       int64(1112443),
+		"vol_cs":           int64(4120386897),
+		"invol_cs":         int64(3685239306),
+		"disk_input":       int64(41679480946688),
+		"disk_output":      int64(1819070669176832),
+		"read_count":       int64(2368906465),
+		"read_bytes":       int64(2957928122981169),
+		"write_count":      int64(3545389615),
+		"write_bytes":      int64(1666822498251286),
+		"mail_lookup_path": int64(24396105),
+		"mail_lookup_attr": int64(302845),
+		"mail_read_count":  int64(20155768),
+		"mail_read_bytes":  int64(669946617705),
+		"mail_cache_hits":  int64(1557255080),
+	}
+
+	tags = map[string]string{"server": "dovecot.test", "type": "user", "user": "user.1@domain.test"}
+	buf = bytes.NewBufferString(sampleUser)
+
+	err = gatherStats(buf, &acc, "dovecot.test", "user")
+	require.NoError(t, err)
+
 	acc.AssertContainsTaggedFields(t, "dovecot", fields, tags)
 
 }
 
-const sampleStats = `domain	reset_timestamp	last_update	num_logins	num_cmds	num_connected_sessions	user_cpu	sys_cpu	clock_time	min_faults	maj_faults	vol_cs	invol_cs	disk_input	disk_output	read_count	read_bytes	write_count	write_bytes	mail_lookup_path	mail_lookup_attr	mail_read_count	mail_read_bytes	mail_cache_hits
-domain.bad	1453970076	1454603947.383029	10749	33828	0	177988.524000	148071.772000	7531838964717.193706	212491179	2125	2190386067	112779200	74487934976	3221808119808	2469948401	5237602841760	1091171292	2951966459802	15363	0	2922	136403379	334372
+const sampleGlobal = `reset_timestamp	last_update	num_logins	num_cmds	num_connected_sessions	user_cpu	sys_cpu	clock_time	min_faults	maj_faults	vol_cs	invol_cs	disk_input	disk_output	read_count	read_bytes	write_count	write_bytes	mail_lookup_path	mail_lookup_attr	mail_read_count	mail_read_bytes	mail_cache_hits
+1453969886	1454603963.039864	7503897	52595715	1204	100831175.372000	83849071.112000	4326001931528183.495762	763950011	1112443	4120386897	3685239306	41679480946688	1819070669176832	2368906465	2957928122981169	3545389615	1666822498251286	24396105	302845	20155768	669946617705	1557255080`
+
+const sampleDomain = `domain	reset_timestamp	last_update	num_logins	num_cmds	num_connected_sessions	user_cpu	sys_cpu	clock_time	min_faults	maj_faults	vol_cs	invol_cs	disk_input	disk_output	read_count	read_bytes	write_count	write_bytes	mail_lookup_path	mail_lookup_attr	mail_read_count	mail_read_bytes	mail_cache_hits
 domain.test	1453969886	1454603963.039864	7503897	52595715	1204	100831175.372000	83849071.112000	4326001931528183.495762	763950011	1112443	4120386897	3685239306	41679480946688	1819070669176832	2368906465	2957928122981169	3545389615	1666822498251286	24396105	302845	20155768	669946617705	1557255080`
+
+const sampleIp = `ip	reset_timestamp	last_update	num_logins	num_cmds	num_connected_sessions	user_cpu	sys_cpu	clock_time	min_faults	maj_faults	vol_cs	invol_cs	disk_input	disk_output	read_count	read_bytes	write_count	write_bytes	mail_lookup_path	mail_lookup_attr	mail_read_count	mail_read_bytes	mail_cache_hits
+192.168.0.100	1453969886	1454603963.039864	7503897	52595715	1204	100831175.372000	83849071.112000	4326001931528183.495762	763950011	1112443	4120386897	3685239306	41679480946688	1819070669176832	2368906465	2957928122981169	3545389615	1666822498251286	24396105	302845	20155768	669946617705	1557255080`
+
+const sampleUser = `user	reset_timestamp	last_update	num_logins	num_cmds	user_cpu	sys_cpu	clock_time	min_faults	maj_faults	vol_cs	invol_cs	disk_input	disk_output	read_count	read_bytes	write_count	write_bytes	mail_lookup_path	mail_lookup_attr	mail_read_count	mail_read_bytes	mail_cache_hits
+user.1@domain.test	1453969886	1454603963.039864	7503897	52595715	100831175.372000	83849071.112000	4326001931528183.495762	763950011	1112443	4120386897	3685239306	41679480946688	1819070669176832	2368906465	2957928122981169	3545389615	1666822498251286	24396105	302845	20155768	669946617705	1557255080`

plugins/inputs/elasticsearch/README.md

@@ -1,320 +1,307 @@
-# Elasticsearch plugin
-
-#### Plugin arguments:
-- **servers** []string: list of one or more Elasticsearch servers
-- **local** boolean: If false, it will read the indices stats from all nodes
-- **cluster_health** boolean: If true, it will also obtain cluster level stats
-
-#### Description
+# Elasticsearch input plugin
 
 The [elasticsearch](https://www.elastic.co/) plugin queries endpoints to obtain
 [node](https://www.elastic.co/guide/en/elasticsearch/reference/current/cluster-nodes-stats.html)
 and optionally [cluster](https://www.elastic.co/guide/en/elasticsearch/reference/current/cluster-health.html) stats.
 
-Example:
+### Configuration:
 
 ```
-    [elasticsearch]
-
-    servers = ["http://localhost:9200"]
-
-    local = true
-
-    cluster_health = true
+[[inputs.elasticsearch]]
+  servers = ["http://localhost:9200"]
+  local = true
+  cluster_health = true
 ```
 
-# Measurements
-#### cluster measurements (utilizes fields instead of single values):
-
-contains `status`, `timed_out`, `number_of_nodes`, `number_of_data_nodes`,
-`active_primary_shards`, `active_shards`, `relocating_shards`,
-`initializing_shards`, `unassigned_shards` fields
-- elasticsearch_cluster_health
-
-contains `status`, `number_of_shards`, `number_of_replicas`,
-`active_primary_shards`, `active_shards`, `relocating_shards`,
-`initializing_shards`, `unassigned_shards` fields
-- elasticsearch_indices
-
-#### node measurements:
+### Measurements & Fields:
 
 field data circuit breaker measurement names:
-- elasticsearch_breakers_fielddata_estimated_size_in_bytes value=0
-- elasticsearch_breakers_fielddata_overhead value=1.03
-- elasticsearch_breakers_fielddata_tripped value=0
-- elasticsearch_breakers_fielddata_limit_size_in_bytes value=623326003
-- elasticsearch_breakers_request_estimated_size_in_bytes value=0
-- elasticsearch_breakers_request_overhead value=1.0
-- elasticsearch_breakers_request_tripped value=0
-- elasticsearch_breakers_request_limit_size_in_bytes value=415550668
-- elasticsearch_breakers_parent_overhead value=1.0
-- elasticsearch_breakers_parent_tripped value=0
-- elasticsearch_breakers_parent_limit_size_in_bytes value=727213670
-- elasticsearch_breakers_parent_estimated_size_in_bytes value=0
+- elasticsearch_breakers
+  - fielddata_estimated_size_in_bytes value=0
+  - fielddata_overhead value=1.03
+  - fielddata_tripped value=0
+  - fielddata_limit_size_in_bytes value=623326003
+  - request_estimated_size_in_bytes value=0
+  - request_overhead value=1.0
+  - request_tripped value=0
+  - request_limit_size_in_bytes value=415550668
+  - parent_overhead value=1.0
+  - parent_tripped value=0
+  - parent_limit_size_in_bytes value=727213670
+  - parent_estimated_size_in_bytes value=0
 
 File system information, data path, free disk space, read/write measurement names:
-- elasticsearch_fs_timestamp value=1436460392946
-- elasticsearch_fs_total_free_in_bytes value=16909316096
-- elasticsearch_fs_total_available_in_bytes value=15894814720
-- elasticsearch_fs_total_total_in_bytes value=19507089408
+- elasticsearch_fs
+  - timestamp value=1436460392946
+  - total_free_in_bytes value=16909316096
+  - total_available_in_bytes value=15894814720
+  - total_total_in_bytes value=19507089408
 
 indices size, document count, indexing and deletion times, search times,
 field cache size, merges and flushes measurement names:
-- elasticsearch_indices_id_cache_memory_size_in_bytes value=0
-- elasticsearch_indices_completion_size_in_bytes value=0
-- elasticsearch_indices_suggest_total value=0
-- elasticsearch_indices_suggest_time_in_millis value=0
-- elasticsearch_indices_suggest_current value=0
-- elasticsearch_indices_query_cache_memory_size_in_bytes value=0
-- elasticsearch_indices_query_cache_evictions value=0
-- elasticsearch_indices_query_cache_hit_count value=0
-- elasticsearch_indices_query_cache_miss_count value=0
-- elasticsearch_indices_store_size_in_bytes value=37715234
-- elasticsearch_indices_store_throttle_time_in_millis value=215
-- elasticsearch_indices_merges_current_docs value=0
-- elasticsearch_indices_merges_current_size_in_bytes value=0
-- elasticsearch_indices_merges_total value=133
-- elasticsearch_indices_merges_total_time_in_millis value=21060
-- elasticsearch_indices_merges_total_docs value=203672
-- elasticsearch_indices_merges_total_size_in_bytes value=142900226
-- elasticsearch_indices_merges_current value=0
-- elasticsearch_indices_filter_cache_memory_size_in_bytes value=7384
-- elasticsearch_indices_filter_cache_evictions value=0
-- elasticsearch_indices_indexing_index_total value=84790
-- elasticsearch_indices_indexing_index_time_in_millis value=29680
-- elasticsearch_indices_indexing_index_current value=0
-- elasticsearch_indices_indexing_noop_update_total value=0
-- elasticsearch_indices_indexing_throttle_time_in_millis value=0
-- elasticsearch_indices_indexing_delete_tota value=13879
-- elasticsearch_indices_indexing_delete_time_in_millis value=1139
-- elasticsearch_indices_indexing_delete_current value=0
-- elasticsearch_indices_get_exists_time_in_millis value=0
-- elasticsearch_indices_get_missing_total value=1
-- elasticsearch_indices_get_missing_time_in_millis value=2
-- elasticsearch_indices_get_current value=0
-- elasticsearch_indices_get_total value=1
-- elasticsearch_indices_get_time_in_millis value=2
-- elasticsearch_indices_get_exists_total value=0
-- elasticsearch_indices_refresh_total value=1076
-- elasticsearch_indices_refresh_total_time_in_millis value=20078
-- elasticsearch_indices_percolate_current value=0
-- elasticsearch_indices_percolate_memory_size_in_bytes value=-1
-- elasticsearch_indices_percolate_queries value=0
-- elasticsearch_indices_percolate_total value=0
-- elasticsearch_indices_percolate_time_in_millis value=0
-- elasticsearch_indices_translog_operations value=17702
-- elasticsearch_indices_translog_size_in_bytes value=17
-- elasticsearch_indices_recovery_current_as_source value=0
-- elasticsearch_indices_recovery_current_as_target value=0
-- elasticsearch_indices_recovery_throttle_time_in_millis value=0
-- elasticsearch_indices_docs_count value=29652
-- elasticsearch_indices_docs_deleted value=5229
-- elasticsearch_indices_flush_total_time_in_millis value=2401
-- elasticsearch_indices_flush_total value=115
-- elasticsearch_indices_fielddata_memory_size_in_bytes value=12996
-- elasticsearch_indices_fielddata_evictions value=0
-- elasticsearch_indices_search_fetch_current value=0
-- elasticsearch_indices_search_open_contexts value=0
-- elasticsearch_indices_search_query_total value=1452
-- elasticsearch_indices_search_query_time_in_millis value=5695
-- elasticsearch_indices_search_query_current value=0
-- elasticsearch_indices_search_fetch_total value=414
-- elasticsearch_indices_search_fetch_time_in_millis value=146
-- elasticsearch_indices_warmer_current value=0
-- elasticsearch_indices_warmer_total value=2319
-- elasticsearch_indices_warmer_total_time_in_millis value=448
-- elasticsearch_indices_segments_count value=134
-- elasticsearch_indices_segments_memory_in_bytes value=1285212
-- elasticsearch_indices_segments_index_writer_memory_in_bytes value=0
-- elasticsearch_indices_segments_index_writer_max_memory_in_bytes value=172368955
-- elasticsearch_indices_segments_version_map_memory_in_bytes value=611844
-- elasticsearch_indices_segments_fixed_bit_set_memory_in_bytes value=0
+- elasticsearch_indices
+  - id_cache_memory_size_in_bytes value=0
+  - completion_size_in_bytes value=0
+  - suggest_total value=0
+  - suggest_time_in_millis value=0
+  - suggest_current value=0
+  - query_cache_memory_size_in_bytes value=0
+  - query_cache_evictions value=0
+  - query_cache_hit_count value=0
+  - query_cache_miss_count value=0
+  - store_size_in_bytes value=37715234
+  - store_throttle_time_in_millis value=215
+  - merges_current_docs value=0
+  - merges_current_size_in_bytes value=0
+  - merges_total value=133
+  - merges_total_time_in_millis value=21060
+  - merges_total_docs value=203672
+  - merges_total_size_in_bytes value=142900226
+  - merges_current value=0
+  - filter_cache_memory_size_in_bytes value=7384
+  - filter_cache_evictions value=0
+  - indexing_index_total value=84790
+  - indexing_index_time_in_millis value=29680
+  - indexing_index_current value=0
+  - indexing_noop_update_total value=0
+  - indexing_throttle_time_in_millis value=0
+  - indexing_delete_tota value=13879
+  - indexing_delete_time_in_millis value=1139
+  - indexing_delete_current value=0
+  - get_exists_time_in_millis value=0
+  - get_missing_total value=1
+  - get_missing_time_in_millis value=2
+  - get_current value=0
+  - get_total value=1
+  - get_time_in_millis value=2
+  - get_exists_total value=0
+  - refresh_total value=1076
+  - refresh_total_time_in_millis value=20078
+  - percolate_current value=0
+  - percolate_memory_size_in_bytes value=-1
+  - percolate_queries value=0
+  - percolate_total value=0
+  - percolate_time_in_millis value=0
+  - translog_operations value=17702
+  - translog_size_in_bytes value=17
+  - recovery_current_as_source value=0
+  - recovery_current_as_target value=0
+  - recovery_throttle_time_in_millis value=0
+  - docs_count value=29652
+  - docs_deleted value=5229
+  - flush_total_time_in_millis value=2401
+  - flush_total value=115
+  - fielddata_memory_size_in_bytes value=12996
+  - fielddata_evictions value=0
+  - search_fetch_current value=0
+  - search_open_contexts value=0
+  - search_query_total value=1452
+  - search_query_time_in_millis value=5695
+  - search_query_current value=0
+  - search_fetch_total value=414
+  - search_fetch_time_in_millis value=146
+  - warmer_current value=0
+  - warmer_total value=2319
+  - warmer_total_time_in_millis value=448
+  - segments_count value=134
+  - segments_memory_in_bytes value=1285212
+  - segments_index_writer_memory_in_bytes value=0
+  - segments_index_writer_max_memory_in_bytes value=172368955
+  - segments_version_map_memory_in_bytes value=611844
+  - segments_fixed_bit_set_memory_in_bytes value=0
 
 HTTP connection measurement names:
-- elasticsearch_http_current_open value=3
-- elasticsearch_http_total_opened value=3
+- elasticsearch_http
+  - current_open value=3
+  - total_opened value=3
 
 JVM stats, memory pool information, garbage collection, buffer pools measurement names:
-- elasticsearch_jvm_timestamp value=1436460392945
-- elasticsearch_jvm_uptime_in_millis value=202245
-- elasticsearch_jvm_mem_non_heap_used_in_bytes value=39634576
-- elasticsearch_jvm_mem_non_heap_committed_in_bytes value=40841216
-- elasticsearch_jvm_mem_pools_young_max_in_bytes value=279183360
-- elasticsearch_jvm_mem_pools_young_peak_used_in_bytes value=71630848
-- elasticsearch_jvm_mem_pools_young_peak_max_in_bytes value=279183360
-- elasticsearch_jvm_mem_pools_young_used_in_bytes value=32685760
-- elasticsearch_jvm_mem_pools_survivor_peak_used_in_bytes value=8912888
-- elasticsearch_jvm_mem_pools_survivor_peak_max_in_bytes value=34865152
-- elasticsearch_jvm_mem_pools_survivor_used_in_bytes value=8912880
-- elasticsearch_jvm_mem_pools_survivor_max_in_bytes value=34865152
-- elasticsearch_jvm_mem_pools_old_peak_max_in_bytes value=724828160
-- elasticsearch_jvm_mem_pools_old_used_in_bytes value=11110928
-- elasticsearch_jvm_mem_pools_old_max_in_bytes value=724828160
-- elasticsearch_jvm_mem_pools_old_peak_used_in_bytes value=14354608
-- elasticsearch_jvm_mem_heap_used_in_bytes value=52709568
-- elasticsearch_jvm_mem_heap_used_percent value=5
-- elasticsearch_jvm_mem_heap_committed_in_bytes value=259522560
-- elasticsearch_jvm_mem_heap_max_in_bytes value=1038876672
-- elasticsearch_jvm_threads_peak_count value=45
-- elasticsearch_jvm_threads_count value=44
-- elasticsearch_jvm_gc_collectors_young_collection_count value=2
-- elasticsearch_jvm_gc_collectors_young_collection_time_in_millis value=98
-- elasticsearch_jvm_gc_collectors_old_collection_count value=1
-- elasticsearch_jvm_gc_collectors_old_collection_time_in_millis value=24
-- elasticsearch_jvm_buffer_pools_direct_count value=40
-- elasticsearch_jvm_buffer_pools_direct_used_in_bytes value=6304239
-- elasticsearch_jvm_buffer_pools_direct_total_capacity_in_bytes value=6304239
-- elasticsearch_jvm_buffer_pools_mapped_count value=0
-- elasticsearch_jvm_buffer_pools_mapped_used_in_bytes value=0
-- elasticsearch_jvm_buffer_pools_mapped_total_capacity_in_bytes value=0
+- elasticsearch_jvm
+  - timestamp value=1436460392945
+  - uptime_in_millis value=202245
+  - mem_non_heap_used_in_bytes value=39634576
+  - mem_non_heap_committed_in_bytes value=40841216
+  - mem_pools_young_max_in_bytes value=279183360
+  - mem_pools_young_peak_used_in_bytes value=71630848
+  - mem_pools_young_peak_max_in_bytes value=279183360
+  - mem_pools_young_used_in_bytes value=32685760
+  - mem_pools_survivor_peak_used_in_bytes value=8912888
+  - mem_pools_survivor_peak_max_in_bytes value=34865152
+  - mem_pools_survivor_used_in_bytes value=8912880
+  - mem_pools_survivor_max_in_bytes value=34865152
+  - mem_pools_old_peak_max_in_bytes value=724828160
+  - mem_pools_old_used_in_bytes value=11110928
+  - mem_pools_old_max_in_bytes value=724828160
+  - mem_pools_old_peak_used_in_bytes value=14354608
+  - mem_heap_used_in_bytes value=52709568
+  - mem_heap_used_percent value=5
+  - mem_heap_committed_in_bytes value=259522560
+  - mem_heap_max_in_bytes value=1038876672
+  - threads_peak_count value=45
+  - threads_count value=44
+  - gc_collectors_young_collection_count value=2
+  - gc_collectors_young_collection_time_in_millis value=98
+  - gc_collectors_old_collection_count value=1
+  - gc_collectors_old_collection_time_in_millis value=24
+  - buffer_pools_direct_count value=40
+  - buffer_pools_direct_used_in_bytes value=6304239
+  - buffer_pools_direct_total_capacity_in_bytes value=6304239
+  - buffer_pools_mapped_count value=0
+  - buffer_pools_mapped_used_in_bytes value=0
+  - buffer_pools_mapped_total_capacity_in_bytes value=0
 
 TCP information measurement names:
-- elasticsearch_network_tcp_in_errs value=0
-- elasticsearch_network_tcp_passive_opens value=16
-- elasticsearch_network_tcp_curr_estab value=29
-- elasticsearch_network_tcp_in_segs value=113
-- elasticsearch_network_tcp_out_segs value=97
-- elasticsearch_network_tcp_retrans_segs value=0
-- elasticsearch_network_tcp_attempt_fails value=0
-- elasticsearch_network_tcp_active_opens value=13
-- elasticsearch_network_tcp_estab_resets value=0
-- elasticsearch_network_tcp_out_rsts value=0
+- elasticsearch_network
+  - tcp_in_errs value=0
+  - tcp_passive_opens value=16
+  - tcp_curr_estab value=29
+  - tcp_in_segs value=113
+  - tcp_out_segs value=97
+  - tcp_retrans_segs value=0
+  - tcp_attempt_fails value=0
+  - tcp_active_opens value=13
+  - tcp_estab_resets value=0
+  - tcp_out_rsts value=0
 
 Operating system stats, load average, cpu, mem, swap measurement names:
-- elasticsearch_os_swap_used_in_bytes value=0
-- elasticsearch_os_swap_free_in_bytes value=487997440
-- elasticsearch_os_timestamp value=1436460392944
-- elasticsearch_os_uptime_in_millis value=25092
-- elasticsearch_os_cpu_sys value=0
-- elasticsearch_os_cpu_user value=0
-- elasticsearch_os_cpu_idle value=99
-- elasticsearch_os_cpu_usage value=0
-- elasticsearch_os_cpu_stolen value=0
-- elasticsearch_os_mem_free_percent value=74
-- elasticsearch_os_mem_used_percent value=25
-- elasticsearch_os_mem_actual_free_in_bytes value=1565470720
-- elasticsearch_os_mem_actual_used_in_bytes value=534159360
-- elasticsearch_os_mem_free_in_bytes value=477761536
-- elasticsearch_os_mem_used_in_bytes value=1621868544
+- elasticsearch_os
+  - swap_used_in_bytes value=0
+  - swap_free_in_bytes value=487997440
+  - timestamp value=1436460392944
+  - uptime_in_millis value=25092
+  - cpu_sys value=0
+  - cpu_user value=0
+  - cpu_idle value=99
+  - cpu_usage value=0
+  - cpu_stolen value=0
+  - mem_free_percent value=74
+  - mem_used_percent value=25
+  - mem_actual_free_in_bytes value=1565470720
+  - mem_actual_used_in_bytes value=534159360
+  - mem_free_in_bytes value=477761536
+  - mem_used_in_bytes value=1621868544
 
 Process statistics, memory consumption, cpu usage, open file descriptors measurement names:
-- elasticsearch_process_mem_resident_in_bytes value=246382592
-- elasticsearch_process_mem_share_in_bytes value=18747392
-- elasticsearch_process_mem_total_virtual_in_bytes value=4747890688
-- elasticsearch_process_timestamp value=1436460392945
-- elasticsearch_process_open_file_descriptors value=160
-- elasticsearch_process_cpu_total_in_millis value=15480
-- elasticsearch_process_cpu_percent value=2
-- elasticsearch_process_cpu_sys_in_millis value=1870
-- elasticsearch_process_cpu_user_in_millis value=13610
+- elasticsearch_process
+  - mem_resident_in_bytes value=246382592
+  - mem_share_in_bytes value=18747392
+  - mem_total_virtual_in_bytes value=4747890688
+  - timestamp value=1436460392945
+  - open_file_descriptors value=160
+  - cpu_total_in_millis value=15480
+  - cpu_percent value=2
+  - cpu_sys_in_millis value=1870
+  - cpu_user_in_millis value=13610
 
 Statistics about each thread pool, including current size, queue and rejected tasks measurement names:
-- elasticsearch_thread_pool_merge_threads value=6
-- elasticsearch_thread_pool_merge_queue value=4
-- elasticsearch_thread_pool_merge_active value=5
-- elasticsearch_thread_pool_merge_rejected value=2
-- elasticsearch_thread_pool_merge_largest value=5
-- elasticsearch_thread_pool_merge_completed value=1
-- elasticsearch_thread_pool_bulk_threads value=4
-- elasticsearch_thread_pool_bulk_queue value=5
-- elasticsearch_thread_pool_bulk_active value=7
-- elasticsearch_thread_pool_bulk_rejected value=3
-- elasticsearch_thread_pool_bulk_largest value=1
-- elasticsearch_thread_pool_bulk_completed value=4
-- elasticsearch_thread_pool_warmer_threads value=2
-- elasticsearch_thread_pool_warmer_queue value=7
-- elasticsearch_thread_pool_warmer_active value=3
-- elasticsearch_thread_pool_warmer_rejected value=2
-- elasticsearch_thread_pool_warmer_largest value=3
-- elasticsearch_thread_pool_warmer_completed value=1
-- elasticsearch_thread_pool_get_largest value=2
-- elasticsearch_thread_pool_get_completed value=1
-- elasticsearch_thread_pool_get_threads value=1
-- elasticsearch_thread_pool_get_queue value=8
-- elasticsearch_thread_pool_get_active value=4
-- elasticsearch_thread_pool_get_rejected value=3
-- elasticsearch_thread_pool_index_threads value=6
-- elasticsearch_thread_pool_index_queue value=8
-- elasticsearch_thread_pool_index_active value=4
-- elasticsearch_thread_pool_index_rejected value=2
-- elasticsearch_thread_pool_index_largest value=3
-- elasticsearch_thread_pool_index_completed value=6
-- elasticsearch_thread_pool_suggest_threads value=2
-- elasticsearch_thread_pool_suggest_queue value=7
-- elasticsearch_thread_pool_suggest_active value=2
-- elasticsearch_thread_pool_suggest_rejected value=1
-- elasticsearch_thread_pool_suggest_largest value=8
-- elasticsearch_thread_pool_suggest_completed value=3
-- elasticsearch_thread_pool_fetch_shard_store_queue value=7
-- elasticsearch_thread_pool_fetch_shard_store_active value=4
-- elasticsearch_thread_pool_fetch_shard_store_rejected value=2
-- elasticsearch_thread_pool_fetch_shard_store_largest value=4
-- elasticsearch_thread_pool_fetch_shard_store_completed value=1
-- elasticsearch_thread_pool_fetch_shard_store_threads value=1
-- elasticsearch_thread_pool_management_threads value=2
-- elasticsearch_thread_pool_management_queue value=3
-- elasticsearch_thread_pool_management_active value=1
-- elasticsearch_thread_pool_management_rejected value=6
-- elasticsearch_thread_pool_management_largest value=2
-- elasticsearch_thread_pool_management_completed value=22
-- elasticsearch_thread_pool_percolate_queue value=23
-- elasticsearch_thread_pool_percolate_active value=13
-- elasticsearch_thread_pool_percolate_rejected value=235
-- elasticsearch_thread_pool_percolate_largest value=23
-- elasticsearch_thread_pool_percolate_completed value=33
-- elasticsearch_thread_pool_percolate_threads value=123
-- elasticsearch_thread_pool_listener_active value=4
-- elasticsearch_thread_pool_listener_rejected value=8
-- elasticsearch_thread_pool_listener_largest value=1
-- elasticsearch_thread_pool_listener_completed value=1
-- elasticsearch_thread_pool_listener_threads value=1
-- elasticsearch_thread_pool_listener_queue value=2
-- elasticsearch_thread_pool_search_rejected value=7
-- elasticsearch_thread_pool_search_largest value=2
-- elasticsearch_thread_pool_search_completed value=4
-- elasticsearch_thread_pool_search_threads value=5
-- elasticsearch_thread_pool_search_queue value=7
-- elasticsearch_thread_pool_search_active value=2
-- elasticsearch_thread_pool_fetch_shard_started_threads value=3
-- elasticsearch_thread_pool_fetch_shard_started_queue value=1
-- elasticsearch_thread_pool_fetch_shard_started_active value=5
-- elasticsearch_thread_pool_fetch_shard_started_rejected value=6
-- elasticsearch_thread_pool_fetch_shard_started_largest value=4
-- elasticsearch_thread_pool_fetch_shard_started_completed value=54
-- elasticsearch_thread_pool_refresh_rejected value=4
-- elasticsearch_thread_pool_refresh_largest value=8
-- elasticsearch_thread_pool_refresh_completed value=3
-- elasticsearch_thread_pool_refresh_threads value=23
-- elasticsearch_thread_pool_refresh_queue value=7
-- elasticsearch_thread_pool_refresh_active value=3
-- elasticsearch_thread_pool_optimize_threads value=3
-- elasticsearch_thread_pool_optimize_queue value=4
-- elasticsearch_thread_pool_optimize_active value=1
-- elasticsearch_thread_pool_optimize_rejected value=2
-- elasticsearch_thread_pool_optimize_largest value=7
-- elasticsearch_thread_pool_optimize_completed value=3
-- elasticsearch_thread_pool_snapshot_largest value=1
-- elasticsearch_thread_pool_snapshot_completed value=0
-- elasticsearch_thread_pool_snapshot_threads value=8
-- elasticsearch_thread_pool_snapshot_queue value=5
-- elasticsearch_thread_pool_snapshot_active value=6
-- elasticsearch_thread_pool_snapshot_rejected value=2
-- elasticsearch_thread_pool_generic_threads value=1
-- elasticsearch_thread_pool_generic_queue value=4
-- elasticsearch_thread_pool_generic_active value=6
-- elasticsearch_thread_pool_generic_rejected value=3
-- elasticsearch_thread_pool_generic_largest value=2
-- elasticsearch_thread_pool_generic_completed value=27
-- elasticsearch_thread_pool_flush_threads value=3
-- elasticsearch_thread_pool_flush_queue value=8
-- elasticsearch_thread_pool_flush_active value=0
-- elasticsearch_thread_pool_flush_rejected value=1
-- elasticsearch_thread_pool_flush_largest value=5
-- elasticsearch_thread_pool_flush_completed value=3
+- elasticsearch_thread_pool
+  - merge_threads value=6
+  - merge_queue value=4
+  - merge_active value=5
+  - merge_rejected value=2
+  - merge_largest value=5
+  - merge_completed value=1
+  - bulk_threads value=4
+  - bulk_queue value=5
+  - bulk_active value=7
+  - bulk_rejected value=3
+  - bulk_largest value=1
+  - bulk_completed value=4
+  - warmer_threads value=2
+  - warmer_queue value=7
+  - warmer_active value=3
+  - warmer_rejected value=2
+  - warmer_largest value=3
+  - warmer_completed value=1
+  - get_largest value=2
+  - get_completed value=1
+  - get_threads value=1
+  - get_queue value=8
+  - get_active value=4
+  - get_rejected value=3
+  - index_threads value=6
+  - index_queue value=8
+  - index_active value=4
+  - index_rejected value=2
+  - index_largest value=3
+  - index_completed value=6
+  - suggest_threads value=2
+  - suggest_queue value=7
+  - suggest_active value=2
+  - suggest_rejected value=1
+  - suggest_largest value=8
+  - suggest_completed value=3
+  - fetch_shard_store_queue value=7
+  - fetch_shard_store_active value=4
+  - fetch_shard_store_rejected value=2
+  - fetch_shard_store_largest value=4
+  - fetch_shard_store_completed value=1
+  - fetch_shard_store_threads value=1
+  - management_threads value=2
+  - management_queue value=3
+  - management_active value=1
+  - management_rejected value=6
+  - management_largest value=2
+  - management_completed value=22
+  - percolate_queue value=23
+  - percolate_active value=13
+  - percolate_rejected value=235
+  - percolate_largest value=23
+  - percolate_completed value=33
+  - percolate_threads value=123
+  - listener_active value=4
+  - listener_rejected value=8
+  - listener_largest value=1
+  - listener_completed value=1
+  - listener_threads value=1
+  - listener_queue value=2
+  - search_rejected value=7
+  - search_largest value=2
+  - search_completed value=4
+  - search_threads value=5
+  - search_queue value=7
+  - search_active value=2
+  - fetch_shard_started_threads value=3
+  - fetch_shard_started_queue value=1
+  - fetch_shard_started_active value=5
+  - fetch_shard_started_rejected value=6
+  - fetch_shard_started_largest value=4
+  - fetch_shard_started_completed value=54
+  - refresh_rejected value=4
+  - refresh_largest value=8
+  - refresh_completed value=3
+  - refresh_threads value=23
+  - refresh_queue value=7
+  - refresh_active value=3
+  - optimize_threads value=3
+  - optimize_queue value=4
+  - optimize_active value=1
+  - optimize_rejected value=2
+  - optimize_largest value=7
+  - optimize_completed value=3
+  - snapshot_largest value=1
+  - snapshot_completed value=0
+  - snapshot_threads value=8
+  - snapshot_queue value=5
+  - snapshot_active value=6
+  - snapshot_rejected value=2
+  - generic_threads value=1
+  - generic_queue value=4
+  - generic_active value=6
+  - generic_rejected value=3
+  - generic_largest value=2
+  - generic_completed value=27
+  - flush_threads value=3
+  - flush_queue value=8
+  - flush_active value=0
+  - flush_rejected value=1
+  - flush_largest value=5
+  - flush_completed value=3
 
 Transport statistics about sent and received bytes in cluster communication measurement names:
-- elasticsearch_transport_server_open value=13
-- elasticsearch_transport_rx_count value=6
-- elasticsearch_transport_rx_size_in_bytes value=1380
-- elasticsearch_transport_tx_count value=6
-- elasticsearch_transport_tx_size_in_bytes value=1380
+- elasticsearch_transport
+  - server_open value=13
+  - rx_count value=6
+  - rx_size_in_bytes value=1380
+  - tx_count value=6
+  - tx_size_in_bytes value=1380

plugins/inputs/elasticsearch/elasticsearch.go

@@ -81,7 +81,12 @@ type Elasticsearch struct {
 
 // NewElasticsearch return a new instance of Elasticsearch
 func NewElasticsearch() *Elasticsearch {
-	return &Elasticsearch{client: http.DefaultClient}
+	tr := &http.Transport{ResponseHeaderTimeout: time.Duration(3 * time.Second)}
+	client := &http.Client{
+		Transport: tr,
+		Timeout:   time.Duration(4 * time.Second),
+	}
+	return &Elasticsearch{client: client}
 }
 
 // SampleConfig returns sample configuration for this plugin.

plugins/inputs/elasticsearch/elasticsearch_test.go

@@ -34,6 +34,9 @@ func (t *transportMock) RoundTrip(r *http.Request) (*http.Response, error) {
 	return res, nil
 }
 
+func (t *transportMock) CancelRequest(_ *http.Request) {
+}
+
 func TestElasticsearch(t *testing.T) {
 	es := NewElasticsearch()
 	es.Servers = []string{"http://example.com:9200"}

plugins/inputs/exec/README.md

@@ -1,28 +1,13 @@
 # Exec Input Plugin
 
-The exec plugin can execute arbitrary commands which output:
+Please also see: [Telegraf Input Data Formats](https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md)
 
-* JSON
-* InfluxDB [line-protocol](https://docs.influxdata.com/influxdb/v0.9/write_protocols/line/)
-* Graphite [graphite-protocol](http://graphite.readthedocs.org/en/latest/feeding-carbon.html)
+### Example 1 - JSON
 
-> Graphite understands messages with this format:
+#### Configuration
 
-> ```
-metric_path value timestamp\n
-```
-
-> __metric_path__ is the metric namespace that you want to populate.
-
-> __value__ is the value that you want to assign to the metric at this time.
-
-> __timestamp__ is the unix epoch time.
-
-
-If using JSON, only numeric values are parsed and turned into floats. Booleans
-and strings will be ignored.
-
-### Configuration
+In this example a script called ```/tmp/test.sh``` and a script called ```/tmp/test2.sh```
+are configured for ```[[inputs.exec]]``` in JSON format.
 
 ```
 # Read flattened metrics from one or more commands that output JSON to stdout
@@ -30,7 +15,7 @@ and strings will be ignored.
   # Shell/commands array
   commands = ["/tmp/test.sh", "/tmp/test2.sh"]
 
-  # Data format to consume. This can be "json", "influx" or "graphite" (line-protocol)
+  # Data format to consume.
   # NOTE json only reads numerical measurements, strings and booleans are ignored.
   data_format = "json"
 
@@ -64,8 +49,6 @@ Other options for modifying the measurement names are:
 name_prefix = "prefix_"
 ```
 
-### Example 1
-
 Let's say that we have the above configuration, and mycollector outputs the
 following JSON:
 
@@ -85,10 +68,16 @@ The collected metrics will be stored as fields under the measurement
 ```
 exec_mycollector a=0.5,b_c=0.1,b_d=5 1452815002357578567
 ```
+If using JSON, only numeric values are parsed and turned into floats. Booleans
+and strings will be ignored.
 
-### Example 2
+### Example 2 - Influx Line-Protocol
 
-Now let's say we have the following configuration:
+In this example an application called ```/usr/bin/line_protocol_collector```
+and a script called ```/tmp/test2.sh``` are configured for ```[[inputs.exec]]```
+in influx line-protocol format.
+
+#### Configuration
 
 ```
 [[inputs.exec]]
@@ -98,12 +87,12 @@ Now let's say we have the following configuration:
   # command = "/usr/bin/line_protocol_collector"
   commands = ["/usr/bin/line_protocol_collector","/tmp/test2.sh"]
 
-  # Data format to consume. This can be "json" or "influx" (line-protocol)
+  # Data format to consume.
   # NOTE json only reads numerical measurements, strings and booleans are ignored.
   data_format = "influx"
 ```
 
-And line_protocol_collector outputs the following line protocol:
+The line_protocol_collector application outputs the following line protocol:
 
 ```
 cpu,cpu=cpu0,host=foo,datacenter=us-east usage_idle=99,usage_busy=1
@@ -118,14 +107,17 @@ cpu,cpu=cpu6,host=foo,datacenter=us-east usage_idle=99,usage_busy=1
 You will get data in InfluxDB exactly as it is defined above,
 tags are cpu=cpuN, host=foo, and datacenter=us-east with fields usage_idle
 and usage_busy. They will receive a timestamp at collection time.
+Each line must end in \n, just as the Influx line protocol does.
 
 
-### Example 3
+### Example 3 - Graphite
 
 We can also change the data_format to "graphite" to use the metrics collecting scripts such as (compatible with graphite):
 
-* Nagios [Mertics Plugins] (https://exchange.nagios.org/directory/Plugins)
-* Sensu [Mertics Plugins] (https://github.com/sensu-plugins)
+* Nagios [Metrics Plugins](https://exchange.nagios.org/directory/Plugins)
+* Sensu [Metrics Plugins](https://github.com/sensu-plugins)
+
+In this example a script called /tmp/test.sh and a script called /tmp/test2.sh are configured for [[inputs.exec]] in graphite format.
 
 #### Configuration
 ```
@@ -134,7 +126,7 @@ We can also change the data_format to "graphite" to use the metrics collecting s
   # Shell/commands array
   commands = ["/tmp/test.sh","/tmp/test2.sh"]
 
-  # Data format to consume. This can be "json", "influx" or "graphite" (line-protocol)
+  # Data format to consume.
   # NOTE json only reads numerical measurements, strings and booleans are ignored.
   data_format = "graphite"
 
@@ -161,6 +153,17 @@ We can also change the data_format to "graphite" to use the metrics collecting s
     "measurement*"
   ]
 ```
+Graphite messages are in this format:
+
+```
+metric_path value timestamp\n
+```
+
+__metric_path__ is the metric namespace that you want to populate.
+
+__value__ is the value that you want to assign to the metric at this time.
+
+__timestamp__ is the unix epoch time.
 
 And test.sh/test2.sh will output:
 
@@ -176,5 +179,5 @@ sensu.metric.net.server0.eth0.rx_dropped 0 1444234982
 
 The templates configuration will be used to parse the graphite metrics to support influxdb/opentsdb tagging store engines.
 
-More detail information about templates, please refer to [The graphite Input] (https://github.com/influxdata/influxdb/blob/master/services/graphite/README.md)
+More detail information about templates, please refer to [The graphite Input](https://github.com/influxdata/influxdb/blob/master/services/graphite/README.md)
 

plugins/inputs/exec/exec.go

@@ -5,22 +5,29 @@ import (
 	"fmt"
 	"os/exec"
 	"sync"
+	"syscall"
+	"time"
 
 	"github.com/gonuts/go-shellquote"
 
 	"github.com/influxdata/telegraf"
+	"github.com/influxdata/telegraf/internal"
 	"github.com/influxdata/telegraf/plugins/inputs"
 	"github.com/influxdata/telegraf/plugins/parsers"
+	"github.com/influxdata/telegraf/plugins/parsers/nagios"
 )
 
 const sampleConfig = `
   ## Commands array
   commands = ["/tmp/test.sh", "/usr/bin/mycollector --foo=bar"]
 
+  ## Timeout for each command to complete.
+  timeout = "5s"
+
   ## measurement name suffix (for separating different commands)
   name_suffix = "_mycollector"
 
-  ## Data format to consume. This can be "json", "influx" or "graphite"
+  ## Data format to consume.
   ## Each data format has it's own unique set of configuration options, read
   ## more about them here:
   ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md
@@ -30,6 +37,7 @@ const sampleConfig = `
 type Exec struct {
 	Commands []string
 	Command  string
+	Timeout  internal.Duration
 
 	parser parsers.Parser
 
@@ -41,17 +49,42 @@ type Exec struct {
 
 func NewExec() *Exec {
 	return &Exec{
-		runner: CommandRunner{},
+		runner:  CommandRunner{},
+		Timeout: internal.Duration{Duration: time.Second * 5},
 	}
 }
 
 type Runner interface {
-	Run(*Exec, string) ([]byte, error)
+	Run(*Exec, string, telegraf.Accumulator) ([]byte, error)
 }
 
 type CommandRunner struct{}
 
-func (c CommandRunner) Run(e *Exec, command string) ([]byte, error) {
+func AddNagiosState(exitCode error, acc telegraf.Accumulator) error {
+	nagiosState := 0
+	if exitCode != nil {
+		exiterr, ok := exitCode.(*exec.ExitError)
+		if ok {
+			status, ok := exiterr.Sys().(syscall.WaitStatus)
+			if ok {
+				nagiosState = status.ExitStatus()
+			} else {
+				return fmt.Errorf("exec: unable to get nagios plugin exit code")
+			}
+		} else {
+			return fmt.Errorf("exec: unable to get nagios plugin exit code")
+		}
+	}
+	fields := map[string]interface{}{"state": nagiosState}
+	acc.AddFields("nagios_state", fields, nil)
+	return nil
+}
+
+func (c CommandRunner) Run(
+	e *Exec,
+	command string,
+	acc telegraf.Accumulator,
+) ([]byte, error) {
 	split_cmd, err := shellquote.Split(command)
 	if err != nil || len(split_cmd) == 0 {
 		return nil, fmt.Errorf("exec: unable to parse command, %s", err)
@@ -62,8 +95,18 @@ func (c CommandRunner) Run(e *Exec, command string) ([]byte, error) {
 	var out bytes.Buffer
 	cmd.Stdout = &out
 
-	if err := cmd.Run(); err != nil {
-		return nil, fmt.Errorf("exec: %s for command '%s'", err, command)
+	if err := internal.RunTimeout(cmd, e.Timeout.Duration); err != nil {
+		switch e.parser.(type) {
+		case *nagios.NagiosParser:
+			AddNagiosState(err, acc)
+		default:
+			return nil, fmt.Errorf("exec: %s for command '%s'", err, command)
+		}
+	} else {
+		switch e.parser.(type) {
+		case *nagios.NagiosParser:
+			AddNagiosState(nil, acc)
+		}
 	}
 
 	return out.Bytes(), nil
@@ -72,7 +115,7 @@ func (c CommandRunner) Run(e *Exec, command string) ([]byte, error) {
 func (e *Exec) ProcessCommand(command string, acc telegraf.Accumulator) {
 	defer e.wg.Done()
 
-	out, err := e.runner.Run(e, command)
+	out, err := e.runner.Run(e, command, acc)
 	if err != nil {
 		e.errChan <- err
 		return

plugins/inputs/exec/exec_test.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"testing"
 
+	"github.com/influxdata/telegraf"
 	"github.com/influxdata/telegraf/plugins/parsers"
 
 	"github.com/influxdata/telegraf/testutil"
@@ -57,7 +58,7 @@ func newRunnerMock(out []byte, err error) Runner {
 	}
 }
 
-func (r runnerMock) Run(e *Exec, command string) ([]byte, error) {
+func (r runnerMock) Run(e *Exec, command string, acc telegraf.Accumulator) ([]byte, error) {
 	if r.err != nil {
 		return nil, r.err
 	}

plugins/inputs/filestat/README.md

@@ -0,0 +1,37 @@
+# filestat Input Plugin
+
+The filestat plugin gathers metrics about file existence, size, and other stats.
+
+### Configuration:
+
+```toml
+# Read stats about given file(s)
+[[inputs.filestat]]
+  ## Files to gather stats about.
+  ## These accept standard unix glob matching rules, but with the addition of
+  ## ** as a "super asterisk". See https://github.com/gobwas/glob.
+  files = ["/etc/telegraf/telegraf.conf", "/var/log/**.log"]
+  ## If true, read the entire file and calculate an md5 checksum.
+  md5 = false
+```
+
+### Measurements & Fields:
+
+- filestat
+    - exists (int, 0 | 1)
+    - size_bytes (int, bytes)
+    - md5 (optional, string)
+
+### Tags:
+
+- All measurements have the following tags:
+    - file (the path the to file, as specified in the config)
+
+### Example Output:
+
+```
+$ telegraf -config /etc/telegraf/telegraf.conf -input-filter filestat -test
+* Plugin: filestat, Collection 1
+> filestat,file=/tmp/foo/bar,host=tyrion exists=0i 1461203374493128216
+> filestat,file=/Users/sparrc/ws/telegraf.conf,host=tyrion exists=1i,size=47894i 1461203374493199335
+```

plugins/inputs/filestat/filestat.go

@@ -0,0 +1,125 @@
+package filestat
+
+import (
+	"crypto/md5"
+	"fmt"
+	"io"
+	"os"
+
+	"github.com/influxdata/telegraf"
+	"github.com/influxdata/telegraf/internal/globpath"
+	"github.com/influxdata/telegraf/plugins/inputs"
+)
+
+const sampleConfig = `
+  ## Files to gather stats about.
+  ## These accept standard unix glob matching rules, but with the addition of
+  ## ** as a "super asterisk". ie:
+  ##   "/var/log/**.log"  -> recursively find all .log files in /var/log
+  ##   "/var/log/*/*.log" -> find all .log files with a parent dir in /var/log
+  ##   "/var/log/apache.log" -> just tail the apache log file
+  ##
+  ## See https://github.com/gobwas/glob for more examples
+  ##
+  files = ["/var/log/**.log"]
+  ## If true, read the entire file and calculate an md5 checksum.
+  md5 = false
+`
+
+type FileStat struct {
+	Md5   bool
+	Files []string
+
+	// maps full file paths to globmatch obj
+	globs map[string]*globpath.GlobPath
+}
+
+func NewFileStat() *FileStat {
+	return &FileStat{
+		globs: make(map[string]*globpath.GlobPath),
+	}
+}
+
+func (_ *FileStat) Description() string {
+	return "Read stats about given file(s)"
+}
+
+func (_ *FileStat) SampleConfig() string { return sampleConfig }
+
+func (f *FileStat) Gather(acc telegraf.Accumulator) error {
+	var errS string
+	var err error
+
+	for _, filepath := range f.Files {
+		// Get the compiled glob object for this filepath
+		g, ok := f.globs[filepath]
+		if !ok {
+			if g, err = globpath.Compile(filepath); err != nil {
+				errS += err.Error() + " "
+				continue
+			}
+			f.globs[filepath] = g
+		}
+
+		files := g.Match()
+		if len(files) == 0 {
+			acc.AddFields("filestat",
+				map[string]interface{}{
+					"exists": int64(0),
+				},
+				map[string]string{
+					"file": filepath,
+				})
+			continue
+		}
+
+		for fileName, fileInfo := range files {
+			tags := map[string]string{
+				"file": fileName,
+			}
+			fields := map[string]interface{}{
+				"exists":     int64(1),
+				"size_bytes": fileInfo.Size(),
+			}
+
+			if f.Md5 {
+				md5, err := getMd5(fileName)
+				if err != nil {
+					errS += err.Error() + " "
+				} else {
+					fields["md5_sum"] = md5
+				}
+			}
+
+			acc.AddFields("filestat", fields, tags)
+		}
+	}
+
+	if errS != "" {
+		return fmt.Errorf(errS)
+	}
+	return nil
+}
+
+// Read given file and calculate an md5 hash.
+func getMd5(file string) (string, error) {
+	of, err := os.Open(file)
+	if err != nil {
+		return "", err
+	}
+	defer of.Close()
+
+	hash := md5.New()
+	_, err = io.Copy(hash, of)
+	if err != nil {
+		// fatal error
+		return "", err
+	}
+	return fmt.Sprintf("%x", hash.Sum(nil)), nil
+}
+
+func init() {
+	inputs.Add("filestat", func() telegraf.Input {
+		return NewFileStat()
+	})
+}

plugins/inputs/filestat/filestat_test.go

@@ -0,0 +1,180 @@
+package filestat
+
+import (
+	"runtime"
+	"strings"
+	"testing"
+
+	"github.com/influxdata/telegraf/testutil"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestGatherNoMd5(t *testing.T) {
+	dir := getTestdataDir()
+	fs := NewFileStat()
+	fs.Files = []string{
+		dir + "log1.log",
+		dir + "log2.log",
+		"/non/existant/file",
+	}
+
+	acc := testutil.Accumulator{}
+	fs.Gather(&acc)
+
+	tags1 := map[string]string{
+		"file": dir + "log1.log",
+	}
+	fields1 := map[string]interface{}{
+		"size_bytes": int64(0),
+		"exists":     int64(1),
+	}
+	acc.AssertContainsTaggedFields(t, "filestat", fields1, tags1)
+
+	tags2 := map[string]string{
+		"file": dir + "log2.log",
+	}
+	fields2 := map[string]interface{}{
+		"size_bytes": int64(0),
+		"exists":     int64(1),
+	}
+	acc.AssertContainsTaggedFields(t, "filestat", fields2, tags2)
+
+	tags3 := map[string]string{
+		"file": "/non/existant/file",
+	}
+	fields3 := map[string]interface{}{
+		"exists": int64(0),
+	}
+	acc.AssertContainsTaggedFields(t, "filestat", fields3, tags3)
+}
+
+func TestGatherExplicitFiles(t *testing.T) {
+	dir := getTestdataDir()
+	fs := NewFileStat()
+	fs.Md5 = true
+	fs.Files = []string{
+		dir + "log1.log",
+		dir + "log2.log",
+		"/non/existant/file",
+	}
+
+	acc := testutil.Accumulator{}
+	fs.Gather(&acc)
+
+	tags1 := map[string]string{
+		"file": dir + "log1.log",
+	}
+	fields1 := map[string]interface{}{
+		"size_bytes": int64(0),
+		"exists":     int64(1),
+		"md5_sum":    "d41d8cd98f00b204e9800998ecf8427e",
+	}
+	acc.AssertContainsTaggedFields(t, "filestat", fields1, tags1)
+
+	tags2 := map[string]string{
+		"file": dir + "log2.log",
+	}
+	fields2 := map[string]interface{}{
+		"size_bytes": int64(0),
+		"exists":     int64(1),
+		"md5_sum":    "d41d8cd98f00b204e9800998ecf8427e",
+	}
+	acc.AssertContainsTaggedFields(t, "filestat", fields2, tags2)
+
+	tags3 := map[string]string{
+		"file": "/non/existant/file",
+	}
+	fields3 := map[string]interface{}{
+		"exists": int64(0),
+	}
+	acc.AssertContainsTaggedFields(t, "filestat", fields3, tags3)
+}
+
+func TestGatherGlob(t *testing.T) {
+	dir := getTestdataDir()
+	fs := NewFileStat()
+	fs.Md5 = true
+	fs.Files = []string{
+		dir + "*.log",
+	}
+
+	acc := testutil.Accumulator{}
+	fs.Gather(&acc)
+
+	tags1 := map[string]string{
+		"file": dir + "log1.log",
+	}
+	fields1 := map[string]interface{}{
+		"size_bytes": int64(0),
+		"exists":     int64(1),
+		"md5_sum":    "d41d8cd98f00b204e9800998ecf8427e",
+	}
+	acc.AssertContainsTaggedFields(t, "filestat", fields1, tags1)
+
+	tags2 := map[string]string{
+		"file": dir + "log2.log",
+	}
+	fields2 := map[string]interface{}{
+		"size_bytes": int64(0),
+		"exists":     int64(1),
+		"md5_sum":    "d41d8cd98f00b204e9800998ecf8427e",
+	}
+	acc.AssertContainsTaggedFields(t, "filestat", fields2, tags2)
+}
+
+func TestGatherSuperAsterisk(t *testing.T) {
+	dir := getTestdataDir()
+	fs := NewFileStat()
+	fs.Md5 = true
+	fs.Files = []string{
+		dir + "**",
+	}
+
+	acc := testutil.Accumulator{}
+	fs.Gather(&acc)
+
+	tags1 := map[string]string{
+		"file": dir + "log1.log",
+	}
+	fields1 := map[string]interface{}{
+		"size_bytes": int64(0),
+		"exists":     int64(1),
+		"md5_sum":    "d41d8cd98f00b204e9800998ecf8427e",
+	}
+	acc.AssertContainsTaggedFields(t, "filestat", fields1, tags1)
+
+	tags2 := map[string]string{
+		"file": dir + "log2.log",
+	}
+	fields2 := map[string]interface{}{
+		"size_bytes": int64(0),
+		"exists":     int64(1),
+		"md5_sum":    "d41d8cd98f00b204e9800998ecf8427e",
+	}
+	acc.AssertContainsTaggedFields(t, "filestat", fields2, tags2)
+
+	tags3 := map[string]string{
+		"file": dir + "test.conf",
+	}
+	fields3 := map[string]interface{}{
+		"size_bytes": int64(104),
+		"exists":     int64(1),
+		"md5_sum":    "5a7e9b77fa25e7bb411dbd17cf403c1f",
+	}
+	acc.AssertContainsTaggedFields(t, "filestat", fields3, tags3)
+}
+
+func TestGetMd5(t *testing.T) {
+	dir := getTestdataDir()
+	md5, err := getMd5(dir + "test.conf")
+	assert.NoError(t, err)
+	assert.Equal(t, "5a7e9b77fa25e7bb411dbd17cf403c1f", md5)
+
+	md5, err = getMd5("/tmp/foo/bar/fooooo")
+	assert.Error(t, err)
+}
+
+func getTestdataDir() string {
+	_, filename, _, _ := runtime.Caller(1)
+	return strings.Replace(filename, "filestat_test.go", "testdata/", 1)
+}

plugins/inputs/filestat/testdata/test.conf

@@ -0,0 +1,5 @@
+# this is a fake testing config file
+# for testing the filestat plugin
+
+option1 = "foo"
+option2 = "bar"

plugins/inputs/github_webhooks/github_webhooks.go

@@ -73,14 +73,17 @@ func (gh *GithubWebhooks) Stop() {
 
 // Handles the / route
 func (gh *GithubWebhooks) eventHandler(w http.ResponseWriter, r *http.Request) {
+	defer r.Body.Close()
 	eventType := r.Header["X-Github-Event"][0]
 	data, err := ioutil.ReadAll(r.Body)
 	if err != nil {
 		w.WriteHeader(http.StatusBadRequest)
+		return
 	}
 	e, err := NewEvent(data, eventType)
 	if err != nil {
 		w.WriteHeader(http.StatusBadRequest)
+		return
 	}
 	gh.Lock()
 	gh.events = append(gh.events, e)
@@ -88,193 +91,12 @@ func (gh *GithubWebhooks) eventHandler(w http.ResponseWriter, r *http.Request) {
 	w.WriteHeader(http.StatusOK)
 }
 
-func newCommitComment(data []byte) (Event, error) {
-	commitCommentStruct := CommitCommentEvent{}
-	err := json.Unmarshal(data, &commitCommentStruct)
+func generateEvent(data []byte, event Event) (Event, error) {
+	err := json.Unmarshal(data, event)
 	if err != nil {
 		return nil, err
 	}
-	return commitCommentStruct, nil
-}
-
-func newCreate(data []byte) (Event, error) {
-	createStruct := CreateEvent{}
-	err := json.Unmarshal(data, &createStruct)
-	if err != nil {
-		return nil, err
-	}
-	return createStruct, nil
-}
-
-func newDelete(data []byte) (Event, error) {
-	deleteStruct := DeleteEvent{}
-	err := json.Unmarshal(data, &deleteStruct)
-	if err != nil {
-		return nil, err
-	}
-	return deleteStruct, nil
-}
-
-func newDeployment(data []byte) (Event, error) {
-	deploymentStruct := DeploymentEvent{}
-	err := json.Unmarshal(data, &deploymentStruct)
-	if err != nil {
-		return nil, err
-	}
-	return deploymentStruct, nil
-}
-
-func newDeploymentStatus(data []byte) (Event, error) {
-	deploymentStatusStruct := DeploymentStatusEvent{}
-	err := json.Unmarshal(data, &deploymentStatusStruct)
-	if err != nil {
-		return nil, err
-	}
-	return deploymentStatusStruct, nil
-}
-
-func newFork(data []byte) (Event, error) {
-	forkStruct := ForkEvent{}
-	err := json.Unmarshal(data, &forkStruct)
-	if err != nil {
-		return nil, err
-	}
-	return forkStruct, nil
-}
-
-func newGollum(data []byte) (Event, error) {
-	gollumStruct := GollumEvent{}
-	err := json.Unmarshal(data, &gollumStruct)
-	if err != nil {
-		return nil, err
-	}
-	return gollumStruct, nil
-}
-
-func newIssueComment(data []byte) (Event, error) {
-	issueCommentStruct := IssueCommentEvent{}
-	err := json.Unmarshal(data, &issueCommentStruct)
-	if err != nil {
-		return nil, err
-	}
-	return issueCommentStruct, nil
-}
-
-func newIssues(data []byte) (Event, error) {
-	issuesStruct := IssuesEvent{}
-	err := json.Unmarshal(data, &issuesStruct)
-	if err != nil {
-		return nil, err
-	}
-	return issuesStruct, nil
-}
-
-func newMember(data []byte) (Event, error) {
-	memberStruct := MemberEvent{}
-	err := json.Unmarshal(data, &memberStruct)
-	if err != nil {
-		return nil, err
-	}
-	return memberStruct, nil
-}
-
-func newMembership(data []byte) (Event, error) {
-	membershipStruct := MembershipEvent{}
-	err := json.Unmarshal(data, &membershipStruct)
-	if err != nil {
-		return nil, err
-	}
-	return membershipStruct, nil
-}
-
-func newPageBuild(data []byte) (Event, error) {
-	pageBuildEvent := PageBuildEvent{}
-	err := json.Unmarshal(data, &pageBuildEvent)
-	if err != nil {
-		return nil, err
-	}
-	return pageBuildEvent, nil
-}
-
-func newPublic(data []byte) (Event, error) {
-	publicEvent := PublicEvent{}
-	err := json.Unmarshal(data, &publicEvent)
-	if err != nil {
-		return nil, err
-	}
-	return publicEvent, nil
-}
-
-func newPullRequest(data []byte) (Event, error) {
-	pullRequestStruct := PullRequestEvent{}
-	err := json.Unmarshal(data, &pullRequestStruct)
-	if err != nil {
-		return nil, err
-	}
-	return pullRequestStruct, nil
-}
-
-func newPullRequestReviewComment(data []byte) (Event, error) {
-	pullRequestReviewCommentStruct := PullRequestReviewCommentEvent{}
-	err := json.Unmarshal(data, &pullRequestReviewCommentStruct)
-	if err != nil {
-		return nil, err
-	}
-	return pullRequestReviewCommentStruct, nil
-}
-
-func newPush(data []byte) (Event, error) {
-	pushStruct := PushEvent{}
-	err := json.Unmarshal(data, &pushStruct)
-	if err != nil {
-		return nil, err
-	}
-	return pushStruct, nil
-}
-
-func newRelease(data []byte) (Event, error) {
-	releaseStruct := ReleaseEvent{}
-	err := json.Unmarshal(data, &releaseStruct)
-	if err != nil {
-		return nil, err
-	}
-	return releaseStruct, nil
-}
-
-func newRepository(data []byte) (Event, error) {
-	repositoryStruct := RepositoryEvent{}
-	err := json.Unmarshal(data, &repositoryStruct)
-	if err != nil {
-		return nil, err
-	}
-	return repositoryStruct, nil
-}
-
-func newStatus(data []byte) (Event, error) {
-	statusStruct := StatusEvent{}
-	err := json.Unmarshal(data, &statusStruct)
-	if err != nil {
-		return nil, err
-	}
-	return statusStruct, nil
-}
-
-func newTeamAdd(data []byte) (Event, error) {
-	teamAddStruct := TeamAddEvent{}
-	err := json.Unmarshal(data, &teamAddStruct)
-	if err != nil {
-		return nil, err
-	}
-	return teamAddStruct, nil
-}
-
-func newWatch(data []byte) (Event, error) {
-	watchStruct := WatchEvent{}
-	err := json.Unmarshal(data, &watchStruct)
-	if err != nil {
-		return nil, err
-	}
-	return watchStruct, nil
+	return event, nil
 }
 
 type newEventError struct {
@@ -285,51 +107,51 @@ func (e *newEventError) Error() string {
 	return e.s
 }
 
-func NewEvent(r []byte, t string) (Event, error) {
-	log.Printf("New %v event recieved", t)
-	switch t {
+func NewEvent(data []byte, name string) (Event, error) {
+	log.Printf("New %v event received", name)
+	switch name {
 	case "commit_comment":
-		return newCommitComment(r)
+		return generateEvent(data, &CommitCommentEvent{})
 	case "create":
-		return newCreate(r)
+		return generateEvent(data, &CreateEvent{})
 	case "delete":
-		return newDelete(r)
+		return generateEvent(data, &DeleteEvent{})
 	case "deployment":
-		return newDeployment(r)
+		return generateEvent(data, &DeploymentEvent{})
 	case "deployment_status":
-		return newDeploymentStatus(r)
+		return generateEvent(data, &DeploymentStatusEvent{})
 	case "fork":
-		return newFork(r)
+		return generateEvent(data, &ForkEvent{})
 	case "gollum":
-		return newGollum(r)
+		return generateEvent(data, &GollumEvent{})
 	case "issue_comment":
-		return newIssueComment(r)
+		return generateEvent(data, &IssueCommentEvent{})
 	case "issues":
-		return newIssues(r)
+		return generateEvent(data, &IssuesEvent{})
 	case "member":
-		return newMember(r)
+		return generateEvent(data, &MemberEvent{})
 	case "membership":
-		return newMembership(r)
+		return generateEvent(data, &MembershipEvent{})
 	case "page_build":
-		return newPageBuild(r)
+		return generateEvent(data, &PageBuildEvent{})
 	case "public":
-		return newPublic(r)
+		return generateEvent(data, &PublicEvent{})
 	case "pull_request":
-		return newPullRequest(r)
+		return generateEvent(data, &PullRequestEvent{})
 	case "pull_request_review_comment":
-		return newPullRequestReviewComment(r)
+		return generateEvent(data, &PullRequestReviewCommentEvent{})
 	case "push":
-		return newPush(r)
+		return generateEvent(data, &PushEvent{})
 	case "release":
-		return newRelease(r)
+		return generateEvent(data, &ReleaseEvent{})
 	case "repository":
-		return newRepository(r)
+		return generateEvent(data, &RepositoryEvent{})
 	case "status":
-		return newStatus(r)
+		return generateEvent(data, &StatusEvent{})
 	case "team_add":
-		return newTeamAdd(r)
+		return generateEvent(data, &TeamAddEvent{})
 	case "watch":
-		return newWatch(r)
+		return generateEvent(data, &WatchEvent{})
 	}
-	return nil, &newEventError{"Not a recgonized event type"}
+	return nil, &newEventError{"Not a recognized event type"}
 }

plugins/inputs/github_webhooks/github_webhooks_test.go

@@ -7,231 +7,89 @@ import (
 	"testing"
 )
 
-func TestCommitCommentEvent(t *testing.T) {
+func GithubWebhookRequest(event string, jsonString string, t *testing.T) {
 	gh := NewGithubWebhooks()
-	jsonString := CommitCommentEventJSON()
 	req, _ := http.NewRequest("POST", "/", strings.NewReader(jsonString))
-	req.Header.Add("X-Github-Event", "commit_comment")
+	req.Header.Add("X-Github-Event", event)
 	w := httptest.NewRecorder()
 	gh.eventHandler(w, req)
 	if w.Code != http.StatusOK {
-		t.Errorf("POST commit_comment returned HTTP status code %v.\nExpected %v", w.Code, http.StatusOK)
+		t.Errorf("POST "+event+" returned HTTP status code %v.\nExpected %v", w.Code, http.StatusOK)
 	}
 }
 
+func TestCommitCommentEvent(t *testing.T) {
+	GithubWebhookRequest("commit_comment", CommitCommentEventJSON(), t)
+}
+
 func TestDeleteEvent(t *testing.T) {
-	gh := NewGithubWebhooks()
-	jsonString := DeleteEventJSON()
-	req, _ := http.NewRequest("POST", "/", strings.NewReader(jsonString))
-	req.Header.Add("X-Github-Event", "delete")
-	w := httptest.NewRecorder()
-	gh.eventHandler(w, req)
-	if w.Code != http.StatusOK {
-		t.Errorf("POST commit_comment returned HTTP status code %v.\nExpected %v", w.Code, http.StatusOK)
-	}
+	GithubWebhookRequest("delete", DeleteEventJSON(), t)
 }
 
 func TestDeploymentEvent(t *testing.T) {
-	gh := NewGithubWebhooks()
-	jsonString := DeploymentEventJSON()
-	req, _ := http.NewRequest("POST", "/", strings.NewReader(jsonString))
-	req.Header.Add("X-Github-Event", "deployment")
-	w := httptest.NewRecorder()
-	gh.eventHandler(w, req)
-	if w.Code != http.StatusOK {
-		t.Errorf("POST commit_comment returned HTTP status code %v.\nExpected %v", w.Code, http.StatusOK)
-	}
+	GithubWebhookRequest("deployment", DeploymentEventJSON(), t)
 }
 
 func TestDeploymentStatusEvent(t *testing.T) {
-	gh := NewGithubWebhooks()
-	jsonString := DeploymentStatusEventJSON()
-	req, _ := http.NewRequest("POST", "/", strings.NewReader(jsonString))
-	req.Header.Add("X-Github-Event", "deployment_status")
-	w := httptest.NewRecorder()
-	gh.eventHandler(w, req)
-	if w.Code != http.StatusOK {
-		t.Errorf("POST commit_comment returned HTTP status code %v.\nExpected %v", w.Code, http.StatusOK)
-	}
+	GithubWebhookRequest("deployment_status", DeploymentStatusEventJSON(), t)
 }
 
 func TestForkEvent(t *testing.T) {
-	gh := NewGithubWebhooks()
-	jsonString := ForkEventJSON()
-	req, _ := http.NewRequest("POST", "/", strings.NewReader(jsonString))
-	req.Header.Add("X-Github-Event", "fork")
-	w := httptest.NewRecorder()
-	gh.eventHandler(w, req)
-	if w.Code != http.StatusOK {
-		t.Errorf("POST commit_comment returned HTTP status code %v.\nExpected %v", w.Code, http.StatusOK)
-	}
+	GithubWebhookRequest("fork", ForkEventJSON(), t)
 }
 
 func TestGollumEvent(t *testing.T) {
-	gh := NewGithubWebhooks()
-	jsonString := GollumEventJSON()
-	req, _ := http.NewRequest("POST", "/", strings.NewReader(jsonString))
-	req.Header.Add("X-Github-Event", "gollum")
-	w := httptest.NewRecorder()
-	gh.eventHandler(w, req)
-	if w.Code != http.StatusOK {
-		t.Errorf("POST commit_comment returned HTTP status code %v.\nExpected %v", w.Code, http.StatusOK)
-	}
+	GithubWebhookRequest("gollum", GollumEventJSON(), t)
 }
 
 func TestIssueCommentEvent(t *testing.T) {
-	gh := NewGithubWebhooks()
-	jsonString := IssueCommentEventJSON()
-	req, _ := http.NewRequest("POST", "/", strings.NewReader(jsonString))
-	req.Header.Add("X-Github-Event", "issue_comment")
-	w := httptest.NewRecorder()
-	gh.eventHandler(w, req)
-	if w.Code != http.StatusOK {
-		t.Errorf("POST commit_comment returned HTTP status code %v.\nExpected %v", w.Code, http.StatusOK)
-	}
+	GithubWebhookRequest("issue_comment", IssueCommentEventJSON(), t)
 }
 
 func TestIssuesEvent(t *testing.T) {
-	gh := NewGithubWebhooks()
-	jsonString := IssuesEventJSON()
-	req, _ := http.NewRequest("POST", "/", strings.NewReader(jsonString))
-	req.Header.Add("X-Github-Event", "issues")
-	w := httptest.NewRecorder()
-	gh.eventHandler(w, req)
-	if w.Code != http.StatusOK {
-		t.Errorf("POST commit_comment returned HTTP status code %v.\nExpected %v", w.Code, http.StatusOK)
-	}
+	GithubWebhookRequest("issues", IssuesEventJSON(), t)
 }
 
 func TestMemberEvent(t *testing.T) {
-	gh := NewGithubWebhooks()
-	jsonString := MemberEventJSON()
-	req, _ := http.NewRequest("POST", "/", strings.NewReader(jsonString))
-	req.Header.Add("X-Github-Event", "member")
-	w := httptest.NewRecorder()
-	gh.eventHandler(w, req)
-	if w.Code != http.StatusOK {
-		t.Errorf("POST commit_comment returned HTTP status code %v.\nExpected %v", w.Code, http.StatusOK)
-	}
+	GithubWebhookRequest("member", MemberEventJSON(), t)
 }
 
 func TestMembershipEvent(t *testing.T) {
-	gh := NewGithubWebhooks()
-	jsonString := MembershipEventJSON()
-	req, _ := http.NewRequest("POST", "/", strings.NewReader(jsonString))
-	req.Header.Add("X-Github-Event", "membership")
-	w := httptest.NewRecorder()
-	gh.eventHandler(w, req)
-	if w.Code != http.StatusOK {
-		t.Errorf("POST commit_comment returned HTTP status code %v.\nExpected %v", w.Code, http.StatusOK)
-	}
+	GithubWebhookRequest("membership", MembershipEventJSON(), t)
 }
 
 func TestPageBuildEvent(t *testing.T) {
-	gh := NewGithubWebhooks()
-	jsonString := PageBuildEventJSON()
-	req, _ := http.NewRequest("POST", "/", strings.NewReader(jsonString))
-	req.Header.Add("X-Github-Event", "page_build")
-	w := httptest.NewRecorder()
-	gh.eventHandler(w, req)
-	if w.Code != http.StatusOK {
-		t.Errorf("POST commit_comment returned HTTP status code %v.\nExpected %v", w.Code, http.StatusOK)
-	}
+	GithubWebhookRequest("page_build", PageBuildEventJSON(), t)
 }
 
 func TestPublicEvent(t *testing.T) {
-	gh := NewGithubWebhooks()
-	jsonString := PublicEventJSON()
-	req, _ := http.NewRequest("POST", "/", strings.NewReader(jsonString))
-	req.Header.Add("X-Github-Event", "public")
-	w := httptest.NewRecorder()
-	gh.eventHandler(w, req)
-	if w.Code != http.StatusOK {
-		t.Errorf("POST commit_comment returned HTTP status code %v.\nExpected %v", w.Code, http.StatusOK)
-	}
+	GithubWebhookRequest("public", PublicEventJSON(), t)
 }
 
 func TestPullRequestReviewCommentEvent(t *testing.T) {
-	gh := NewGithubWebhooks()
-	jsonString := PullRequestReviewCommentEventJSON()
-	req, _ := http.NewRequest("POST", "/", strings.NewReader(jsonString))
-	req.Header.Add("X-Github-Event", "pull_request_review_comment")
-	w := httptest.NewRecorder()
-	gh.eventHandler(w, req)
-	if w.Code != http.StatusOK {
-		t.Errorf("POST commit_comment returned HTTP status code %v.\nExpected %v", w.Code, http.StatusOK)
-	}
+	GithubWebhookRequest("pull_request_review_comment", PullRequestReviewCommentEventJSON(), t)
 }
 
 func TestPushEvent(t *testing.T) {
-	gh := NewGithubWebhooks()
-	jsonString := PushEventJSON()
-	req, _ := http.NewRequest("POST", "/", strings.NewReader(jsonString))
-	req.Header.Add("X-Github-Event", "push")
-	w := httptest.NewRecorder()
-	gh.eventHandler(w, req)
-	if w.Code != http.StatusOK {
-		t.Errorf("POST commit_comment returned HTTP status code %v.\nExpected %v", w.Code, http.StatusOK)
-	}
+	GithubWebhookRequest("push", PushEventJSON(), t)
 }
 
 func TestReleaseEvent(t *testing.T) {
-	gh := NewGithubWebhooks()
-	jsonString := ReleaseEventJSON()
-	req, _ := http.NewRequest("POST", "/", strings.NewReader(jsonString))
-	req.Header.Add("X-Github-Event", "release")
-	w := httptest.NewRecorder()
-	gh.eventHandler(w, req)
-	if w.Code != http.StatusOK {
-		t.Errorf("POST commit_comment returned HTTP status code %v.\nExpected %v", w.Code, http.StatusOK)
-	}
+	GithubWebhookRequest("release", ReleaseEventJSON(), t)
 }
 
 func TestRepositoryEvent(t *testing.T) {
-	gh := NewGithubWebhooks()
-	jsonString := RepositoryEventJSON()
-	req, _ := http.NewRequest("POST", "/", strings.NewReader(jsonString))
-	req.Header.Add("X-Github-Event", "repository")
-	w := httptest.NewRecorder()
-	gh.eventHandler(w, req)
-	if w.Code != http.StatusOK {
-		t.Errorf("POST commit_comment returned HTTP status code %v.\nExpected %v", w.Code, http.StatusOK)
-	}
+	GithubWebhookRequest("repository", RepositoryEventJSON(), t)
 }
 
 func TestStatusEvent(t *testing.T) {
-	gh := NewGithubWebhooks()
-
-	jsonString := StatusEventJSON()
-	req, _ := http.NewRequest("POST", "/", strings.NewReader(jsonString))
-	req.Header.Add("X-Github-Event", "status")
-	w := httptest.NewRecorder()
-	gh.eventHandler(w, req)
-	if w.Code != http.StatusOK {
-		t.Errorf("POST commit_comment returned HTTP status code %v.\nExpected %v", w.Code, http.StatusOK)
-	}
+	GithubWebhookRequest("status", StatusEventJSON(), t)
 }
 
 func TestTeamAddEvent(t *testing.T) {
-	gh := NewGithubWebhooks()
-	jsonString := TeamAddEventJSON()
-	req, _ := http.NewRequest("POST", "/", strings.NewReader(jsonString))
-	req.Header.Add("X-Github-Event", "team_add")
-	w := httptest.NewRecorder()
-	gh.eventHandler(w, req)
-	if w.Code != http.StatusOK {
-		t.Errorf("POST commit_comment returned HTTP status code %v.\nExpected %v", w.Code, http.StatusOK)
-	}
+	GithubWebhookRequest("team_add", TeamAddEventJSON(), t)
 }
 
 func TestWatchEvent(t *testing.T) {
-	gh := NewGithubWebhooks()
-	jsonString := WatchEventJSON()
-	req, _ := http.NewRequest("POST", "/", strings.NewReader(jsonString))
-	req.Header.Add("X-Github-Event", "watch")
-	w := httptest.NewRecorder()
-	gh.eventHandler(w, req)
-	if w.Code != http.StatusOK {
-		t.Errorf("POST commit_comment returned HTTP status code %v.\nExpected %v", w.Code, http.StatusOK)
-	}
+	GithubWebhookRequest("watch", WatchEventJSON(), t)
 }

plugins/inputs/haproxy/haproxy.go

@@ -6,9 +6,11 @@ import (
 	"github.com/influxdata/telegraf"
 	"github.com/influxdata/telegraf/plugins/inputs"
 	"io"
+	"net"
 	"net/http"
 	"net/url"
 	"strconv"
+	"strings"
 	"sync"
 	"time"
 )
@@ -47,7 +49,7 @@ const (
 	HF_THROTTLE       = 29 //29. throttle [...S]: current throttle percentage for the server, when slowstart is active, or no value if not in slowstart.
 	HF_LBTOT          = 30 //30. lbtot [..BS]: total number of times a server was selected, either for new sessions, or when re-dispatching. The server counter is the number of times that server was selected.
 	HF_TRACKED        = 31 //31. tracked [...S]: id of proxy/server if tracking is enabled.
-	HF_TYPE           = 32 //32. type [LFBS]: (0                                                                                                                                                                                                  = frontend, 1 = backend, 2 = server, 3 = socket/listener)
+	HF_TYPE           = 32 //32. type [LFBS]: (0 = frontend, 1 = backend, 2 = server, 3 = socket/listener)
 	HF_RATE           = 33 //33. rate [.FBS]: number of sessions per second over last elapsed second
 	HF_RATE_LIM       = 34 //34. rate_lim [.F..]: configured limit on new sessions per second
 	HF_RATE_MAX       = 35 //35. rate_max [.FBS]: max number of new sessions per second
@@ -91,8 +93,8 @@ var sampleConfig = `
 
   ## If no servers are specified, then default to 127.0.0.1:1936
   servers = ["http://myhaproxy.com:1936", "http://anotherhaproxy.com:1936"]
-  ## Or you can also use local socket(not work yet)
-  ## servers = ["socket://run/haproxy/admin.sock"]
+  ## Or you can also use local socket
+  ## servers = ["socket:/run/haproxy/admin.sock"]
 `
 
 func (r *haproxy) SampleConfig() string {
@@ -127,10 +129,42 @@ func (g *haproxy) Gather(acc telegraf.Accumulator) error {
 	return outerr
 }
 
-func (g *haproxy) gatherServer(addr string, acc telegraf.Accumulator) error {
-	if g.client == nil {
+func (g *haproxy) gatherServerSocket(addr string, acc telegraf.Accumulator) error {
+	var socketPath string
+	socketAddr := strings.Split(addr, ":")
 
-		client := &http.Client{}
+	if len(socketAddr) >= 2 {
+		socketPath = socketAddr[1]
+	} else {
+		socketPath = socketAddr[0]
+	}
+
+	c, err := net.Dial("unix", socketPath)
+
+	if err != nil {
+		return fmt.Errorf("Could not connect to socket '%s': %s", addr, err)
+	}
+
+	_, errw := c.Write([]byte("show stat\n"))
+
+	if errw != nil {
+		return fmt.Errorf("Could not write to socket '%s': %s", addr, errw)
+	}
+
+	return importCsvResult(c, acc, socketPath)
+}
+
+func (g *haproxy) gatherServer(addr string, acc telegraf.Accumulator) error {
+	if !strings.HasPrefix(addr, "http") {
+		return g.gatherServerSocket(addr, acc)
+	}
+
+	if g.client == nil {
+		tr := &http.Transport{ResponseHeaderTimeout: time.Duration(3 * time.Second)}
+		client := &http.Client{
+			Transport: tr,
+			Timeout:   time.Duration(4 * time.Second),
+		}
 		g.client = client
 	}
 

plugins/inputs/haproxy/haproxy_test.go

@@ -1,17 +1,42 @@
 package haproxy
 
 import (
+	"crypto/rand"
+	"encoding/binary"
 	"fmt"
+	"net"
+	"net/http"
+	"net/http/httptest"
 	"strings"
 	"testing"
 
 	"github.com/influxdata/telegraf/testutil"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
-	"net/http"
-	"net/http/httptest"
 )
 
+type statServer struct{}
+
+func (s statServer) serverSocket(l net.Listener) {
+	for {
+		conn, err := l.Accept()
+		if err != nil {
+			return
+		}
+
+		go func(c net.Conn) {
+			buf := make([]byte, 1024)
+			n, _ := c.Read(buf)
+
+			data := buf[:n]
+			if string(data) == "show stat\n" {
+				c.Write([]byte(csvOutputSample))
+				c.Close()
+			}
+		}(conn)
+	}
+}
+
 func TestHaproxyGeneratesMetricsWithAuthentication(t *testing.T) {
 	//We create a fake server to return test data
 	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@@ -146,6 +171,69 @@ func TestHaproxyGeneratesMetricsWithoutAuthentication(t *testing.T) {
 	acc.AssertContainsTaggedFields(t, "haproxy", fields, tags)
 }
 
+func TestHaproxyGeneratesMetricsUsingSocket(t *testing.T) {
+	var randomNumber int64
+	binary.Read(rand.Reader, binary.LittleEndian, &randomNumber)
+	sock, err := net.Listen("unix", fmt.Sprintf("/tmp/test-haproxy%d.sock", randomNumber))
+	if err != nil {
+		t.Fatal("Cannot initialize socket ")
+	}
+
+	defer sock.Close()
+
+	s := statServer{}
+	go s.serverSocket(sock)
+
+	r := &haproxy{
+		Servers: []string{sock.Addr().String()},
+	}
+
+	var acc testutil.Accumulator
+
+	err = r.Gather(&acc)
+	require.NoError(t, err)
+
+	tags := map[string]string{
+		"proxy":  "be_app",
+		"server": sock.Addr().String(),
+		"sv":     "host0",
+	}
+
+	fields := map[string]interface{}{
+		"active_servers":    uint64(1),
+		"backup_servers":    uint64(0),
+		"bin":               uint64(510913516),
+		"bout":              uint64(2193856571),
+		"check_duration":    uint64(10),
+		"cli_abort":         uint64(73),
+		"ctime":             uint64(2),
+		"downtime":          uint64(0),
+		"dresp":             uint64(0),
+		"econ":              uint64(0),
+		"eresp":             uint64(1),
+		"http_response.1xx": uint64(0),
+		"http_response.2xx": uint64(119534),
+		"http_response.3xx": uint64(48051),
+		"http_response.4xx": uint64(2345),
+		"http_response.5xx": uint64(1056),
+		"lbtot":             uint64(171013),
+		"qcur":              uint64(0),
+		"qmax":              uint64(0),
+		"qtime":             uint64(0),
+		"rate":              uint64(3),
+		"rate_max":          uint64(12),
+		"rtime":             uint64(312),
+		"scur":              uint64(1),
+		"smax":              uint64(32),
+		"srv_abort":         uint64(1),
+		"stot":              uint64(171014),
+		"ttime":             uint64(2341),
+		"wredis":            uint64(0),
+		"wretr":             uint64(1),
+	}
+	acc.AssertContainsTaggedFields(t, "haproxy", fields, tags)
+}
+
 //When not passing server config, we default to localhost
 //We just want to make sure we did request stat from localhost
 func TestHaproxyDefaultGetFromLocalhost(t *testing.T) {

plugins/inputs/http_response/README.md

@@ -0,0 +1,44 @@
+# Example Input Plugin
+
+This input plugin will test HTTP/HTTPS connections.
+
+### Configuration:
+
+```
+# HTTP/HTTPS request given an address a method and a timeout
+[[inputs.http_response]]
+  ## Server address (default http://localhost)
+  address = "http://github.com"
+  ## Set response_timeout (default 5 seconds)
+  response_timeout = "5s"
+  ## HTTP Request Method
+  method = "GET"
+  ## Whether to follow redirects from the server (defaults to false)
+  follow_redirects = true
+  ## HTTP Request Headers (all values must be strings)
+  # [inputs.http_response.headers]
+  #   Host = "github.com"
+  ## Optional HTTP Request Body
+  # body = '''
+  # {'fake':'data'}
+  # '''
+```
+
+### Measurements & Fields:
+
+- http_response
+    - response_time (float, seconds)
+    - http_response_code (int) #The code received
+
+### Tags:
+
+- All measurements have the following tags:
+    - server
+    - method
+
+### Example Output:
+
+```
+$ ./telegraf -config telegraf.conf -input-filter http_response -test
+http_response,method=GET,server=http://www.github.com http_response_code=200i,response_time=6.223266528 1459419354977857955
+```

plugins/inputs/http_response/http_response.go

@@ -0,0 +1,151 @@
+package http_response
+
+import (
+	"errors"
+	"io"
+	"net/http"
+	"net/url"
+	"strings"
+	"time"
+
+	"github.com/influxdata/telegraf"
+	"github.com/influxdata/telegraf/internal"
+	"github.com/influxdata/telegraf/plugins/inputs"
+)
+
+// HTTPResponse struct
+type HTTPResponse struct {
+	Address         string
+	Body            string
+	Method          string
+	ResponseTimeout internal.Duration
+	Headers         map[string]string
+	FollowRedirects bool
+}
+
+// Description returns the plugin Description
+func (h *HTTPResponse) Description() string {
+	return "HTTP/HTTPS request given an address a method and a timeout"
+}
+
+var sampleConfig = `
+  ## Server address (default http://localhost)
+  address = "http://github.com"
+  ## Set response_timeout (default 5 seconds)
+  response_timeout = "5s"
+  ## HTTP Request Method
+  method = "GET"
+  ## Whether to follow redirects from the server (defaults to false)
+  follow_redirects = true
+  ## HTTP Request Headers (all values must be strings)
+  # [inputs.http_response.headers]
+  #   Host = "github.com"
+  ## Optional HTTP Request Body
+  # body = '''
+  # {'fake':'data'}
+  # '''
+`
+
+// SampleConfig returns the plugin SampleConfig
+func (h *HTTPResponse) SampleConfig() string {
+	return sampleConfig
+}
+
+// ErrRedirectAttempted indicates that a redirect occurred
+var ErrRedirectAttempted = errors.New("redirect")
+
+// CreateHttpClient creates an http client which will timeout at the specified
+// timeout period and can follow redirects if specified
+func CreateHttpClient(followRedirects bool, ResponseTimeout time.Duration) *http.Client {
+	client := &http.Client{
+		Timeout: ResponseTimeout,
+	}
+
+	if followRedirects == false {
+		client.CheckRedirect = func(req *http.Request, via []*http.Request) error {
+			return ErrRedirectAttempted
+		}
+	}
+	return client
+}
+
+// HTTPGather gathers all fields and returns any errors it encounters
+func (h *HTTPResponse) HTTPGather() (map[string]interface{}, error) {
+	// Prepare fields
+	fields := make(map[string]interface{})
+
+	client := CreateHttpClient(h.FollowRedirects, h.ResponseTimeout.Duration)
+
+	var body io.Reader
+	if h.Body != "" {
+		body = strings.NewReader(h.Body)
+	}
+	request, err := http.NewRequest(h.Method, h.Address, body)
+	if err != nil {
+		return nil, err
+	}
+
+	for key, val := range h.Headers {
+		request.Header.Add(key, val)
+		if key == "Host" {
+			request.Host = val
+		}
+	}
+
+	// Start Timer
+	start := time.Now()
+	resp, err := client.Do(request)
+	if err != nil {
+		if h.FollowRedirects {
+			return nil, err
+		}
+		if urlError, ok := err.(*url.Error); ok &&
+			urlError.Err == ErrRedirectAttempted {
+			err = nil
+		} else {
+			return nil, err
+		}
+	}
+	fields["response_time"] = time.Since(start).Seconds()
+	fields["http_response_code"] = resp.StatusCode
+	return fields, nil
+}
+
+// Gather gets all metric fields and tags and returns any errors it encounters
+func (h *HTTPResponse) Gather(acc telegraf.Accumulator) error {
+	// Set default values
+	if h.ResponseTimeout.Duration < time.Second {
+		h.ResponseTimeout.Duration = time.Second * 5
+	}
+	// Check send and expected string
+	if h.Method == "" {
+		h.Method = "GET"
+	}
+	if h.Address == "" {
+		h.Address = "http://localhost"
+	}
+	addr, err := url.Parse(h.Address)
+	if err != nil {
+		return err
+	}
+	if addr.Scheme != "http" && addr.Scheme != "https" {
+		return errors.New("Only http and https are supported")
+	}
+	// Prepare data
+	tags := map[string]string{"server": h.Address, "method": h.Method}
+	var fields map[string]interface{}
+	// Gather data
+	fields, err = h.HTTPGather()
+	if err != nil {
+		return err
+	}
+	// Add metrics
+	acc.AddFields("http_response", fields, tags)
+	return nil
+}
+
+func init() {
+	inputs.Add("http_response", func() telegraf.Input {
+		return &HTTPResponse{}
+	})
+}

plugins/inputs/http_response/http_response_test.go

@@ -0,0 +1,256 @@
+package http_response
+
+import (
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+	"time"
+
+	"github.com/influxdata/telegraf/internal"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func setUpTestMux() http.Handler {
+	mux := http.NewServeMux()
+	mux.HandleFunc("/redirect", func(w http.ResponseWriter, req *http.Request) {
+		http.Redirect(w, req, "/good", http.StatusMovedPermanently)
+	})
+	mux.HandleFunc("/good", func(w http.ResponseWriter, req *http.Request) {
+		fmt.Fprintf(w, "hit the good page!")
+	})
+	mux.HandleFunc("/badredirect", func(w http.ResponseWriter, req *http.Request) {
+		http.Redirect(w, req, "/badredirect", http.StatusMovedPermanently)
+	})
+	mux.HandleFunc("/mustbepostmethod", func(w http.ResponseWriter, req *http.Request) {
+		if req.Method != "POST" {
+			http.Error(w, "method wasn't post", http.StatusMethodNotAllowed)
+			return
+		}
+		fmt.Fprintf(w, "used post correctly!")
+	})
+	mux.HandleFunc("/musthaveabody", func(w http.ResponseWriter, req *http.Request) {
+		body, err := ioutil.ReadAll(req.Body)
+		req.Body.Close()
+		if err != nil {
+			http.Error(w, "couldn't read request body", http.StatusBadRequest)
+			return
+		}
+		if string(body) == "" {
+			http.Error(w, "body was empty", http.StatusBadRequest)
+			return
+		}
+		fmt.Fprintf(w, "sent a body!")
+	})
+	mux.HandleFunc("/twosecondnap", func(w http.ResponseWriter, req *http.Request) {
+		time.Sleep(time.Second * 2)
+		return
+	})
+	return mux
+}
+
+func TestHeaders(t *testing.T) {
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		cHeader := r.Header.Get("Content-Type")
+		assert.Equal(t, "Hello", r.Host)
+		assert.Equal(t, "application/json", cHeader)
+		w.WriteHeader(http.StatusOK)
+	}))
+	defer ts.Close()
+
+	h := &HTTPResponse{
+		Address:         ts.URL,
+		Method:          "GET",
+		ResponseTimeout: internal.Duration{Duration: time.Second * 2},
+		Headers: map[string]string{
+			"Content-Type": "application/json",
+			"Host":         "Hello",
+		},
+	}
+	fields, err := h.HTTPGather()
+	require.NoError(t, err)
+	assert.NotEmpty(t, fields)
+	if assert.NotNil(t, fields["http_response_code"]) {
+		assert.Equal(t, http.StatusOK, fields["http_response_code"])
+	}
+	assert.NotNil(t, fields["response_time"])
+}
+
+func TestFields(t *testing.T) {
+	mux := setUpTestMux()
+	ts := httptest.NewServer(mux)
+	defer ts.Close()
+
+	h := &HTTPResponse{
+		Address:         ts.URL + "/good",
+		Body:            "{ 'test': 'data'}",
+		Method:          "GET",
+		ResponseTimeout: internal.Duration{Duration: time.Second * 20},
+		Headers: map[string]string{
+			"Content-Type": "application/json",
+		},
+		FollowRedirects: true,
+	}
+	fields, err := h.HTTPGather()
+	require.NoError(t, err)
+	assert.NotEmpty(t, fields)
+	if assert.NotNil(t, fields["http_response_code"]) {
+		assert.Equal(t, http.StatusOK, fields["http_response_code"])
+	}
+	assert.NotNil(t, fields["response_time"])
+}
+
+func TestRedirects(t *testing.T) {
+	mux := setUpTestMux()
+	ts := httptest.NewServer(mux)
+	defer ts.Close()
+
+	h := &HTTPResponse{
+		Address:         ts.URL + "/redirect",
+		Body:            "{ 'test': 'data'}",
+		Method:          "GET",
+		ResponseTimeout: internal.Duration{Duration: time.Second * 20},
+		Headers: map[string]string{
+			"Content-Type": "application/json",
+		},
+		FollowRedirects: true,
+	}
+	fields, err := h.HTTPGather()
+	require.NoError(t, err)
+	assert.NotEmpty(t, fields)
+	if assert.NotNil(t, fields["http_response_code"]) {
+		assert.Equal(t, http.StatusOK, fields["http_response_code"])
+	}
+
+	h = &HTTPResponse{
+		Address:         ts.URL + "/badredirect",
+		Body:            "{ 'test': 'data'}",
+		Method:          "GET",
+		ResponseTimeout: internal.Duration{Duration: time.Second * 20},
+		Headers: map[string]string{
+			"Content-Type": "application/json",
+		},
+		FollowRedirects: true,
+	}
+	fields, err = h.HTTPGather()
+	require.Error(t, err)
+}
+
+func TestMethod(t *testing.T) {
+	mux := setUpTestMux()
+	ts := httptest.NewServer(mux)
+	defer ts.Close()
+
+	h := &HTTPResponse{
+		Address:         ts.URL + "/mustbepostmethod",
+		Body:            "{ 'test': 'data'}",
+		Method:          "POST",
+		ResponseTimeout: internal.Duration{Duration: time.Second * 20},
+		Headers: map[string]string{
+			"Content-Type": "application/json",
+		},
+		FollowRedirects: true,
+	}
+	fields, err := h.HTTPGather()
+	require.NoError(t, err)
+	assert.NotEmpty(t, fields)
+	if assert.NotNil(t, fields["http_response_code"]) {
+		assert.Equal(t, http.StatusOK, fields["http_response_code"])
+	}
+
+	h = &HTTPResponse{
+		Address:         ts.URL + "/mustbepostmethod",
+		Body:            "{ 'test': 'data'}",
+		Method:          "GET",
+		ResponseTimeout: internal.Duration{Duration: time.Second * 20},
+		Headers: map[string]string{
+			"Content-Type": "application/json",
+		},
+		FollowRedirects: true,
+	}
+	fields, err = h.HTTPGather()
+	require.NoError(t, err)
+	assert.NotEmpty(t, fields)
+	if assert.NotNil(t, fields["http_response_code"]) {
+		assert.Equal(t, http.StatusMethodNotAllowed, fields["http_response_code"])
+	}
+
+	//check that lowercase methods work correctly
+	h = &HTTPResponse{
+		Address:         ts.URL + "/mustbepostmethod",
+		Body:            "{ 'test': 'data'}",
+		Method:          "head",
+		ResponseTimeout: internal.Duration{Duration: time.Second * 20},
+		Headers: map[string]string{
+			"Content-Type": "application/json",
+		},
+		FollowRedirects: true,
+	}
+	fields, err = h.HTTPGather()
+	require.NoError(t, err)
+	assert.NotEmpty(t, fields)
+	if assert.NotNil(t, fields["http_response_code"]) {
+		assert.Equal(t, http.StatusMethodNotAllowed, fields["http_response_code"])
+	}
+}
+
+func TestBody(t *testing.T) {
+	mux := setUpTestMux()
+	ts := httptest.NewServer(mux)
+	defer ts.Close()
+
+	h := &HTTPResponse{
+		Address:         ts.URL + "/musthaveabody",
+		Body:            "{ 'test': 'data'}",
+		Method:          "GET",
+		ResponseTimeout: internal.Duration{Duration: time.Second * 20},
+		Headers: map[string]string{
+			"Content-Type": "application/json",
+		},
+		FollowRedirects: true,
+	}
+	fields, err := h.HTTPGather()
+	require.NoError(t, err)
+	assert.NotEmpty(t, fields)
+	if assert.NotNil(t, fields["http_response_code"]) {
+		assert.Equal(t, http.StatusOK, fields["http_response_code"])
+	}
+
+	h = &HTTPResponse{
+		Address:         ts.URL + "/musthaveabody",
+		Method:          "GET",
+		ResponseTimeout: internal.Duration{Duration: time.Second * 20},
+		Headers: map[string]string{
+			"Content-Type": "application/json",
+		},
+		FollowRedirects: true,
+	}
+	fields, err = h.HTTPGather()
+	require.NoError(t, err)
+	assert.NotEmpty(t, fields)
+	if assert.NotNil(t, fields["http_response_code"]) {
+		assert.Equal(t, http.StatusBadRequest, fields["http_response_code"])
+	}
+}
+
+func TestTimeout(t *testing.T) {
+	mux := setUpTestMux()
+	ts := httptest.NewServer(mux)
+	defer ts.Close()
+
+	h := &HTTPResponse{
+		Address:         ts.URL + "/twosecondnap",
+		Body:            "{ 'test': 'data'}",
+		Method:          "GET",
+		ResponseTimeout: internal.Duration{Duration: time.Second * 1},
+		Headers: map[string]string{
+			"Content-Type": "application/json",
+		},
+		FollowRedirects: true,
+	}
+	_, err := h.HTTPGather()
+	require.Error(t, err)
+}

plugins/inputs/httpjson/README.md

@@ -6,7 +6,7 @@ For example, if you have a service called _mycollector_, which has HTTP endpoint
 plugin like this:
 
 ```
-[[httpjson.services]]
+[[inputs.httpjson]]
   name = "mycollector"
 
   servers = [
@@ -24,7 +24,7 @@ plugin like this:
 You can also specify which keys from server response should be considered tags:
 
 ```
-[[httpjson.services]]
+[[inputs.httpjson]]
   ...
 
   tag_keys = [
@@ -36,10 +36,10 @@ You can also specify which keys from server response should be considered tags:
 You can also specify additional request parameters for the service:
 
 ```
-[[httpjson.services]]
+[[inputs.httpjson]]
   ...
 
- [httpjson.services.parameters]
+ [inputs.httpjson.parameters]
     event_type = "cpu_spike"
     threshold = "0.75"
 
@@ -48,10 +48,10 @@ You can also specify additional request parameters for the service:
 You can also specify additional request header parameters for the service:
 
 ```
-[[httpjson.services]]
+[[inputs.httpjson]]
   ...
 
- [httpjson.services.headers]
+ [inputs.httpjson.headers]
     X-Auth-Token = "my-xauth-token"
     apiVersion = "v1"
 ```
@@ -61,18 +61,14 @@ You can also specify additional request header parameters for the service:
 Let's say that we have a service named "mycollector" configured like this:
 
 ```
-[httpjson]
-  [[httpjson.services]]
-    name = "mycollector"
-
-    servers = [
-      "http://my.service.com/_stats"
-    ]
-
-    # HTTP method to use (case-sensitive)
-    method = "GET"
-
-    tag_keys = ["service"]
+[[inputs.httpjson]]
+  name = "mycollector"
+  servers = [
+    "http://my.service.com/_stats"
+  ]
+  # HTTP method to use (case-sensitive)
+  method = "GET"
+  tag_keys = ["service"]
 ```
 
 which responds with the following JSON:
@@ -102,26 +98,21 @@ There is also the option to collect JSON from multiple services, here is an
 example doing that.
 
 ```
-[httpjson]
-  [[httpjson.services]]
-    name = "mycollector1"
+[[inputs.httpjson]]
+  name = "mycollector1"
+  servers = [
+    "http://my.service1.com/_stats"
+  ]
+  # HTTP method to use (case-sensitive)
+  method = "GET"
 
-    servers = [
-      "http://my.service1.com/_stats"
-    ]
-
-    # HTTP method to use (case-sensitive)
-    method = "GET"
-
-  [[httpjson.services]]
-    name = "mycollector2"
-
-    servers = [
-      "http://service.net/json/stats"
-    ]
-
-    # HTTP method to use (case-sensitive)
-    method = "POST"
+[[inputs.httpjson]]
+  name = "mycollector2"
+  servers = [
+    "http://service.net/json/stats"
+  ]
+  # HTTP method to use (case-sensitive)
+  method = "POST"
 ```
 
 The services respond with the following JSON:

plugins/inputs/httpjson/httpjson.go

@@ -1,7 +1,6 @@
 package httpjson
 
 import (
-	"bytes"
 	"errors"
 	"fmt"
 	"io/ioutil"
@@ -12,6 +11,7 @@ import (
 	"time"
 
 	"github.com/influxdata/telegraf"
+	"github.com/influxdata/telegraf/internal"
 	"github.com/influxdata/telegraf/plugins/inputs"
 	"github.com/influxdata/telegraf/plugins/parsers"
 )
@@ -23,7 +23,17 @@ type HttpJson struct {
 	TagKeys    []string
 	Parameters map[string]string
 	Headers    map[string]string
-	client     HTTPClient
+
+	// Path to CA file
+	SSLCA string `toml:"ssl_ca"`
+	// Path to host cert file
+	SSLCert string `toml:"ssl_cert"`
+	// Path to cert key file
+	SSLKey string `toml:"ssl_key"`
+	// Use SSL but skip chain & host verification
+	InsecureSkipVerify bool
+
+	client HTTPClient
 }
 
 type HTTPClient interface {
@@ -36,16 +46,27 @@ type HTTPClient interface {
 	// http.Response:  HTTP respons object
 	// error        :  Any error that may have occurred
 	MakeRequest(req *http.Request) (*http.Response, error)
+
+	SetHTTPClient(client *http.Client)
+	HTTPClient() *http.Client
 }
 
 type RealHTTPClient struct {
 	client *http.Client
 }
 
-func (c RealHTTPClient) MakeRequest(req *http.Request) (*http.Response, error) {
+func (c *RealHTTPClient) MakeRequest(req *http.Request) (*http.Response, error) {
 	return c.client.Do(req)
 }
 
+func (c *RealHTTPClient) SetHTTPClient(client *http.Client) {
+	c.client = client
+}
+
+func (c *RealHTTPClient) HTTPClient() *http.Client {
+	return c.client
+}
+
 var sampleConfig = `
   ## NOTE This plugin only reads numerical measurements, strings and booleans
   ## will be ignored.
@@ -77,6 +98,13 @@ var sampleConfig = `
   # [inputs.httpjson.headers]
   #   X-Auth-Token = "my-xauth-token"
   #   apiVersion = "v1"
+
+  ## Optional SSL Config
+  # ssl_ca = "/etc/telegraf/ca.pem"
+  # ssl_cert = "/etc/telegraf/cert.pem"
+  # ssl_key = "/etc/telegraf/key.pem"
+  ## Use SSL but skip chain & host verification
+  # insecure_skip_verify = false
 `
 
 func (h *HttpJson) SampleConfig() string {
@@ -91,6 +119,23 @@ func (h *HttpJson) Description() string {
 func (h *HttpJson) Gather(acc telegraf.Accumulator) error {
 	var wg sync.WaitGroup
 
+	if h.client.HTTPClient() == nil {
+		tlsCfg, err := internal.GetTLSConfig(
+			h.SSLCert, h.SSLKey, h.SSLCA, h.InsecureSkipVerify)
+		if err != nil {
+			return err
+		}
+		tr := &http.Transport{
+			ResponseHeaderTimeout: time.Duration(3 * time.Second),
+			TLSClientConfig:       tlsCfg,
+		}
+		client := &http.Client{
+			Transport: tr,
+			Timeout:   time.Duration(4 * time.Second),
+		}
+		h.client.SetHTTPClient(client)
+	}
+
 	errorChannel := make(chan error, len(h.Servers))
 
 	for _, server := range h.Servers {
@@ -182,15 +227,14 @@ func (h *HttpJson) sendRequest(serverURL string) (string, float64, error) {
 		return "", -1, fmt.Errorf("Invalid server URL \"%s\"", serverURL)
 	}
 
-	params := url.Values{}
 	data := url.Values{}
-
 	switch {
 	case h.Method == "GET":
-		requestURL.RawQuery = params.Encode()
+		params := requestURL.Query()
 		for k, v := range h.Parameters {
 			params.Add(k, v)
 		}
+		requestURL.RawQuery = params.Encode()
 
 	case h.Method == "POST":
 		requestURL.RawQuery = ""
@@ -200,7 +244,8 @@ func (h *HttpJson) sendRequest(serverURL string) (string, float64, error) {
 	}
 
 	// Create + send request
-	req, err := http.NewRequest(h.Method, requestURL.String(), bytes.NewBufferString(data.Encode()))
+	req, err := http.NewRequest(h.Method, requestURL.String(),
+		strings.NewReader(data.Encode()))
 	if err != nil {
 		return "", -1, err
 	}
@@ -244,6 +289,8 @@ func (h *HttpJson) sendRequest(serverURL string) (string, float64, error) {
 
 func init() {
 	inputs.Add("httpjson", func() telegraf.Input {
-		return &HttpJson{client: RealHTTPClient{client: &http.Client{}}}
+		return &HttpJson{
+			client: &RealHTTPClient{},
+		}
 	})
 }

plugins/inputs/httpjson/httpjson_test.go

@@ -1,8 +1,10 @@
 package httpjson
 
 import (
+	"fmt"
 	"io/ioutil"
 	"net/http"
+	"net/http/httptest"
 	"strings"
 	"testing"
 
@@ -27,6 +29,75 @@ const validJSON = `
 		"another_list": [4]
 	}`
 
+const validJSON2 = `{
+  "user":{
+    "hash_rate":0,
+    "expected_24h_rewards":0,
+    "total_rewards":0.000595109232,
+    "paid_rewards":0,
+    "unpaid_rewards":0.000595109232,
+    "past_24h_rewards":0,
+    "total_work":"5172625408",
+    "blocks_found":0
+  },
+  "workers":{
+    "brminer.1":{
+      "hash_rate":0,
+      "hash_rate_24h":0,
+      "valid_shares":"6176",
+      "stale_shares":"0",
+      "invalid_shares":"0",
+      "rewards":4.5506464e-5,
+      "rewards_24h":0,
+      "reset_time":1455409950
+    },
+    "brminer.2":{
+      "hash_rate":0,
+      "hash_rate_24h":0,
+      "valid_shares":"0",
+      "stale_shares":"0",
+      "invalid_shares":"0",
+      "rewards":0,
+      "rewards_24h":0,
+      "reset_time":1455936726
+    },
+    "brminer.3":{
+      "hash_rate":0,
+      "hash_rate_24h":0,
+      "valid_shares":"0",
+      "stale_shares":"0",
+      "invalid_shares":"0",
+      "rewards":0,
+      "rewards_24h":0,
+      "reset_time":1455936733
+    }
+  },
+  "pool":{
+    "hash_rate":114100000,
+    "active_users":843,
+    "total_work":"5015346808842682368",
+    "pps_ratio":1.04,
+    "pps_rate":7.655e-9
+  },
+  "network":{
+    "hash_rate":1426117703,
+    "block_number":944895,
+    "time_per_block":156,
+    "difficulty":51825.72835216,
+    "next_difficulty":51916.15249019,
+    "retarget_time":95053
+  },
+  "market":{
+    "ltc_btc":0.00798,
+    "ltc_usd":3.37801,
+    "ltc_eur":3.113,
+    "ltc_gbp":2.32807,
+    "ltc_rub":241.796,
+    "ltc_cny":21.3883,
+    "btc_usd":422.852
+  }
+}`
+
 const validJSONTags = `
 	{
 		"value": 15,
@@ -54,7 +125,7 @@ type mockHTTPClient struct {
 // Mock implementation of MakeRequest. Usually returns an http.Response with
 // hard-coded responseBody and statusCode. However, if the request uses a
 // nonstandard method, it uses status code 405 (method not allowed)
-func (c mockHTTPClient) MakeRequest(req *http.Request) (*http.Response, error) {
+func (c *mockHTTPClient) MakeRequest(req *http.Request) (*http.Response, error) {
 	resp := http.Response{}
 	resp.StatusCode = c.statusCode
 
@@ -76,6 +147,13 @@ func (c mockHTTPClient) MakeRequest(req *http.Request) (*http.Response, error) {
 	return &resp, nil
 }
 
+func (c *mockHTTPClient) SetHTTPClient(_ *http.Client) {
+}
+
+func (c *mockHTTPClient) HTTPClient() *http.Client {
+	return nil
+}
+
 // Generates a pointer to an HttpJson object that uses a mock HTTP client.
 // Parameters:
 //     response  : Body of the response that the mock HTTP client should return
@@ -86,7 +164,7 @@ func (c mockHTTPClient) MakeRequest(req *http.Request) (*http.Response, error) {
 func genMockHttpJson(response string, statusCode int) []*HttpJson {
 	return []*HttpJson{
 		&HttpJson{
-			client: mockHTTPClient{responseBody: response, statusCode: statusCode},
+			client: &mockHTTPClient{responseBody: response, statusCode: statusCode},
 			Servers: []string{
 				"http://server1.example.com/metrics/",
 				"http://server2.example.com/metrics/",
@@ -103,7 +181,7 @@ func genMockHttpJson(response string, statusCode int) []*HttpJson {
 			},
 		},
 		&HttpJson{
-			client: mockHTTPClient{responseBody: response, statusCode: statusCode},
+			client: &mockHTTPClient{responseBody: response, statusCode: statusCode},
 			Servers: []string{
 				"http://server3.example.com/metrics/",
 				"http://server4.example.com/metrics/",
@@ -149,6 +227,222 @@ func TestHttpJson200(t *testing.T) {
 	}
 }
 
+// Test that GET Parameters from the url string are applied properly
+func TestHttpJsonGET_URL(t *testing.T) {
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		key := r.FormValue("api_key")
+		assert.Equal(t, "mykey", key)
+		w.WriteHeader(http.StatusOK)
+		fmt.Fprintln(w, validJSON2)
+	}))
+	defer ts.Close()
+
+	a := HttpJson{
+		Servers: []string{ts.URL + "?api_key=mykey"},
+		Name:    "",
+		Method:  "GET",
+		client:  &RealHTTPClient{client: &http.Client{}},
+	}
+
+	var acc testutil.Accumulator
+	err := a.Gather(&acc)
+	require.NoError(t, err)
+
+	// remove response_time from gathered fields because it's non-deterministic
+	delete(acc.Metrics[0].Fields, "response_time")
+
+	fields := map[string]interface{}{
+		"market_btc_usd":                  float64(422.852),
+		"market_ltc_btc":                  float64(0.00798),
+		"market_ltc_cny":                  float64(21.3883),
+		"market_ltc_eur":                  float64(3.113),
+		"market_ltc_gbp":                  float64(2.32807),
+		"market_ltc_rub":                  float64(241.796),
+		"market_ltc_usd":                  float64(3.37801),
+		"network_block_number":            float64(944895),
+		"network_difficulty":              float64(51825.72835216),
+		"network_hash_rate":               float64(1.426117703e+09),
+		"network_next_difficulty":         float64(51916.15249019),
+		"network_retarget_time":           float64(95053),
+		"network_time_per_block":          float64(156),
+		"pool_active_users":               float64(843),
+		"pool_hash_rate":                  float64(1.141e+08),
+		"pool_pps_rate":                   float64(7.655e-09),
+		"pool_pps_ratio":                  float64(1.04),
+		"user_blocks_found":               float64(0),
+		"user_expected_24h_rewards":       float64(0),
+		"user_hash_rate":                  float64(0),
+		"user_paid_rewards":               float64(0),
+		"user_past_24h_rewards":           float64(0),
+		"user_total_rewards":              float64(0.000595109232),
+		"user_unpaid_rewards":             float64(0.000595109232),
+		"workers_brminer.1_hash_rate":     float64(0),
+		"workers_brminer.1_hash_rate_24h": float64(0),
+		"workers_brminer.1_reset_time":    float64(1.45540995e+09),
+		"workers_brminer.1_rewards":       float64(4.5506464e-05),
+		"workers_brminer.1_rewards_24h":   float64(0),
+		"workers_brminer.2_hash_rate":     float64(0),
+		"workers_brminer.2_hash_rate_24h": float64(0),
+		"workers_brminer.2_reset_time":    float64(1.455936726e+09),
+		"workers_brminer.2_rewards":       float64(0),
+		"workers_brminer.2_rewards_24h":   float64(0),
+		"workers_brminer.3_hash_rate":     float64(0),
+		"workers_brminer.3_hash_rate_24h": float64(0),
+		"workers_brminer.3_reset_time":    float64(1.455936733e+09),
+		"workers_brminer.3_rewards":       float64(0),
+		"workers_brminer.3_rewards_24h":   float64(0),
+	}
+
+	acc.AssertContainsFields(t, "httpjson", fields)
+}
+
+// Test that GET Parameters are applied properly
+func TestHttpJsonGET(t *testing.T) {
+	params := map[string]string{
+		"api_key": "mykey",
+	}
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		key := r.FormValue("api_key")
+		assert.Equal(t, "mykey", key)
+		w.WriteHeader(http.StatusOK)
+		fmt.Fprintln(w, validJSON2)
+	}))
+	defer ts.Close()
+
+	a := HttpJson{
+		Servers:    []string{ts.URL},
+		Name:       "",
+		Method:     "GET",
+		Parameters: params,
+		client:     &RealHTTPClient{client: &http.Client{}},
+	}
+
+	var acc testutil.Accumulator
+	err := a.Gather(&acc)
+	require.NoError(t, err)
+
+	// remove response_time from gathered fields because it's non-deterministic
+	delete(acc.Metrics[0].Fields, "response_time")
+
+	fields := map[string]interface{}{
+		"market_btc_usd":                  float64(422.852),
+		"market_ltc_btc":                  float64(0.00798),
+		"market_ltc_cny":                  float64(21.3883),
+		"market_ltc_eur":                  float64(3.113),
+		"market_ltc_gbp":                  float64(2.32807),
+		"market_ltc_rub":                  float64(241.796),
+		"market_ltc_usd":                  float64(3.37801),
+		"network_block_number":            float64(944895),
+		"network_difficulty":              float64(51825.72835216),
+		"network_hash_rate":               float64(1.426117703e+09),
+		"network_next_difficulty":         float64(51916.15249019),
+		"network_retarget_time":           float64(95053),
+		"network_time_per_block":          float64(156),
+		"pool_active_users":               float64(843),
+		"pool_hash_rate":                  float64(1.141e+08),
+		"pool_pps_rate":                   float64(7.655e-09),
+		"pool_pps_ratio":                  float64(1.04),
+		"user_blocks_found":               float64(0),
+		"user_expected_24h_rewards":       float64(0),
+		"user_hash_rate":                  float64(0),
+		"user_paid_rewards":               float64(0),
+		"user_past_24h_rewards":           float64(0),
+		"user_total_rewards":              float64(0.000595109232),
+		"user_unpaid_rewards":             float64(0.000595109232),
+		"workers_brminer.1_hash_rate":     float64(0),
+		"workers_brminer.1_hash_rate_24h": float64(0),
+		"workers_brminer.1_reset_time":    float64(1.45540995e+09),
+		"workers_brminer.1_rewards":       float64(4.5506464e-05),
+		"workers_brminer.1_rewards_24h":   float64(0),
+		"workers_brminer.2_hash_rate":     float64(0),
+		"workers_brminer.2_hash_rate_24h": float64(0),
+		"workers_brminer.2_reset_time":    float64(1.455936726e+09),
+		"workers_brminer.2_rewards":       float64(0),
+		"workers_brminer.2_rewards_24h":   float64(0),
+		"workers_brminer.3_hash_rate":     float64(0),
+		"workers_brminer.3_hash_rate_24h": float64(0),
+		"workers_brminer.3_reset_time":    float64(1.455936733e+09),
+		"workers_brminer.3_rewards":       float64(0),
+		"workers_brminer.3_rewards_24h":   float64(0),
+	}
+
+	acc.AssertContainsFields(t, "httpjson", fields)
+}
+
+// Test that POST Parameters are applied properly
+func TestHttpJsonPOST(t *testing.T) {
+	params := map[string]string{
+		"api_key": "mykey",
+	}
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		body, err := ioutil.ReadAll(r.Body)
+		assert.NoError(t, err)
+		assert.Equal(t, "api_key=mykey", string(body))
+		w.WriteHeader(http.StatusOK)
+		fmt.Fprintln(w, validJSON2)
+	}))
+	defer ts.Close()
+
+	a := HttpJson{
+		Servers:    []string{ts.URL},
+		Name:       "",
+		Method:     "POST",
+		Parameters: params,
+		client:     &RealHTTPClient{client: &http.Client{}},
+	}
+
+	var acc testutil.Accumulator
+	err := a.Gather(&acc)
+	require.NoError(t, err)
+
+	// remove response_time from gathered fields because it's non-deterministic
+	delete(acc.Metrics[0].Fields, "response_time")
+
+	fields := map[string]interface{}{
+		"market_btc_usd":                  float64(422.852),
+		"market_ltc_btc":                  float64(0.00798),
+		"market_ltc_cny":                  float64(21.3883),
+		"market_ltc_eur":                  float64(3.113),
+		"market_ltc_gbp":                  float64(2.32807),
+		"market_ltc_rub":                  float64(241.796),
+		"market_ltc_usd":                  float64(3.37801),
+		"network_block_number":            float64(944895),
+		"network_difficulty":              float64(51825.72835216),
+		"network_hash_rate":               float64(1.426117703e+09),
+		"network_next_difficulty":         float64(51916.15249019),
+		"network_retarget_time":           float64(95053),
+		"network_time_per_block":          float64(156),
+		"pool_active_users":               float64(843),
+		"pool_hash_rate":                  float64(1.141e+08),
+		"pool_pps_rate":                   float64(7.655e-09),
+		"pool_pps_ratio":                  float64(1.04),
+		"user_blocks_found":               float64(0),
+		"user_expected_24h_rewards":       float64(0),
+		"user_hash_rate":                  float64(0),
+		"user_paid_rewards":               float64(0),
+		"user_past_24h_rewards":           float64(0),
+		"user_total_rewards":              float64(0.000595109232),
+		"user_unpaid_rewards":             float64(0.000595109232),
+		"workers_brminer.1_hash_rate":     float64(0),
+		"workers_brminer.1_hash_rate_24h": float64(0),
+		"workers_brminer.1_reset_time":    float64(1.45540995e+09),
+		"workers_brminer.1_rewards":       float64(4.5506464e-05),
+		"workers_brminer.1_rewards_24h":   float64(0),
+		"workers_brminer.2_hash_rate":     float64(0),
+		"workers_brminer.2_hash_rate_24h": float64(0),
+		"workers_brminer.2_reset_time":    float64(1.455936726e+09),
+		"workers_brminer.2_rewards":       float64(0),
+		"workers_brminer.2_rewards_24h":   float64(0),
+		"workers_brminer.3_hash_rate":     float64(0),
+		"workers_brminer.3_hash_rate_24h": float64(0),
+		"workers_brminer.3_reset_time":    float64(1.455936733e+09),
+		"workers_brminer.3_rewards":       float64(0),
+		"workers_brminer.3_rewards_24h":   float64(0),
+	}
+
+	acc.AssertContainsFields(t, "httpjson", fields)
+}
+
 // Test response to HTTP 500
 func TestHttpJson500(t *testing.T) {
 	httpjson := genMockHttpJson(validJSON, 500)

plugins/inputs/influxdb/README.md

@@ -1,6 +1,103 @@
 # influxdb plugin
 
-The influxdb plugin collects InfluxDB-formatted data from JSON endpoints.
+The InfluxDB plugin will collect metrics on the given InfluxDB servers.
+
+This plugin can also gather metrics from endpoints that expose
+InfluxDB-formatted endpoints. See below for more information.
+
+### Configuration:
+
+```toml
+# Read InfluxDB-formatted JSON metrics from one or more HTTP endpoints
+[[inputs.influxdb]]
+  ## Works with InfluxDB debug endpoints out of the box,
+  ## but other services can use this format too.
+  ## See the influxdb plugin's README for more details.
+
+  ## Multiple URLs from which to read InfluxDB-formatted JSON
+  urls = [
+    "http://localhost:8086/debug/vars"
+  ]
+```
+
+### Measurements & Fields
+
+- influxdb
+  - n_shards
+- influxdb_database
+- influxdb_httpd
+- influxdb_measurement
+- influxdb_memstats
+  - heap_inuse
+  - heap_released
+  - mspan_inuse
+  - total_alloc
+  - sys
+  - mallocs
+  - frees
+  - heap_idle
+  - pause_total_ns
+  - lookups
+  - heap_sys
+  - mcache_sys
+  - next_gc
+  - gcc_pu_fraction
+  - other_sys
+  - alloc
+  - stack_inuse
+  - stack_sys
+  - buck_hash_sys
+  - gc_sys
+  - num_gc
+  - heap_alloc
+  - heap_objects
+  - mspan_sys
+  - mcache_inuse
+  - last_gc
+- influxdb_shard
+- influxdb_subscriber
+- influxdb_tsm1_cache
+- influxdb_tsm1_wal
+- influxdb_write
+
+### Example Output:
+
+```
+telegraf -config ~/ws/telegraf.conf -input-filter influxdb -test
+* Plugin: influxdb, Collection 1
+> influxdb_database,database=_internal,host=tyrion,url=http://localhost:8086/debug/vars numMeasurements=10,numSeries=29 1463590500247354636
+> influxdb_httpd,bind=:8086,host=tyrion,url=http://localhost:8086/debug/vars req=7,reqActive=1,reqDurationNs=14227734 1463590500247354636
+> influxdb_measurement,database=_internal,host=tyrion,measurement=database,url=http://localhost:8086/debug/vars numSeries=1 1463590500247354636
+> influxdb_measurement,database=_internal,host=tyrion,measurement=httpd,url=http://localhost:8086/debug/vars numSeries=1 1463590500247354636
+> influxdb_measurement,database=_internal,host=tyrion,measurement=measurement,url=http://localhost:8086/debug/vars numSeries=10 1463590500247354636
+> influxdb_measurement,database=_internal,host=tyrion,measurement=runtime,url=http://localhost:8086/debug/vars numSeries=1 1463590500247354636
+> influxdb_measurement,database=_internal,host=tyrion,measurement=shard,url=http://localhost:8086/debug/vars numSeries=4 1463590500247354636
+> influxdb_measurement,database=_internal,host=tyrion,measurement=subscriber,url=http://localhost:8086/debug/vars numSeries=1 1463590500247354636
+> influxdb_measurement,database=_internal,host=tyrion,measurement=tsm1_cache,url=http://localhost:8086/debug/vars numSeries=4 1463590500247354636
+> influxdb_measurement,database=_internal,host=tyrion,measurement=tsm1_filestore,url=http://localhost:8086/debug/vars numSeries=2 1463590500247354636
+> influxdb_measurement,database=_internal,host=tyrion,measurement=tsm1_wal,url=http://localhost:8086/debug/vars numSeries=4 1463590500247354636
+> influxdb_measurement,database=_internal,host=tyrion,measurement=write,url=http://localhost:8086/debug/vars numSeries=1 1463590500247354636
+> influxdb_memstats,host=tyrion,url=http://localhost:8086/debug/vars alloc=7642384i,buck_hash_sys=1463471i,frees=1169558i,gc_sys=653312i,gcc_pu_fraction=0.00003825652361068311,heap_alloc=7642384i,heap_idle=9912320i,heap_inuse=9125888i,heap_objects=48276i,heap_released=0i,heap_sys=19038208i,last_gc=1463590480877651621i,lookups=90i,mallocs=1217834i,mcache_inuse=4800i,mcache_sys=16384i,mspan_inuse=70920i,mspan_sys=81920i,next_gc=11679787i,num_gc=141i,other_sys=1244233i,pause_total_ns=24034027i,stack_inuse=884736i,stack_sys=884736i,sys=23382264i,total_alloc=679012200i 1463590500277918755
+> influxdb_shard,database=_internal,engine=tsm1,host=tyrion,id=4,path=/Users/sparrc/.influxdb/data/_internal/monitor/4,retentionPolicy=monitor,url=http://localhost:8086/debug/vars fieldsCreate=65,seriesCreate=26,writePointsOk=7274,writeReq=280 1463590500247354636
+> influxdb_subscriber,host=tyrion,url=http://localhost:8086/debug/vars pointsWritten=7274 1463590500247354636
+> influxdb_tsm1_cache,database=_internal,host=tyrion,path=/Users/sparrc/.influxdb/data/_internal/monitor/1,retentionPolicy=monitor,url=http://localhost:8086/debug/vars WALCompactionTimeMs=0,cacheAgeMs=2809192,cachedBytes=0,diskBytes=0,memBytes=0,snapshotCount=0 1463590500247354636
+> influxdb_tsm1_cache,database=_internal,host=tyrion,path=/Users/sparrc/.influxdb/data/_internal/monitor/2,retentionPolicy=monitor,url=http://localhost:8086/debug/vars WALCompactionTimeMs=0,cacheAgeMs=2809184,cachedBytes=0,diskBytes=0,memBytes=0,snapshotCount=0 1463590500247354636
+> influxdb_tsm1_cache,database=_internal,host=tyrion,path=/Users/sparrc/.influxdb/data/_internal/monitor/3,retentionPolicy=monitor,url=http://localhost:8086/debug/vars WALCompactionTimeMs=0,cacheAgeMs=2809180,cachedBytes=0,diskBytes=0,memBytes=42368,snapshotCount=0 1463590500247354636
+> influxdb_tsm1_cache,database=_internal,host=tyrion,path=/Users/sparrc/.influxdb/data/_internal/monitor/4,retentionPolicy=monitor,url=http://localhost:8086/debug/vars WALCompactionTimeMs=0,cacheAgeMs=2799155,cachedBytes=0,diskBytes=0,memBytes=331216,snapshotCount=0 1463590500247354636
+> influxdb_tsm1_filestore,database=_internal,host=tyrion,path=/Users/sparrc/.influxdb/data/_internal/monitor/1,retentionPolicy=monitor,url=http://localhost:8086/debug/vars diskBytes=37892 1463590500247354636
+> influxdb_tsm1_filestore,database=_internal,host=tyrion,path=/Users/sparrc/.influxdb/data/_internal/monitor/2,retentionPolicy=monitor,url=http://localhost:8086/debug/vars diskBytes=52907 1463590500247354636
+> influxdb_tsm1_wal,database=_internal,host=tyrion,path=/Users/sparrc/.influxdb/wal/_internal/monitor/1,retentionPolicy=monitor,url=http://localhost:8086/debug/vars currentSegmentDiskBytes=0,oldSegmentsDiskBytes=0 1463590500247354636
+> influxdb_tsm1_wal,database=_internal,host=tyrion,path=/Users/sparrc/.influxdb/wal/_internal/monitor/2,retentionPolicy=monitor,url=http://localhost:8086/debug/vars currentSegmentDiskBytes=0,oldSegmentsDiskBytes=0 1463590500247354636
+> influxdb_tsm1_wal,database=_internal,host=tyrion,path=/Users/sparrc/.influxdb/wal/_internal/monitor/3,retentionPolicy=monitor,url=http://localhost:8086/debug/vars currentSegmentDiskBytes=0,oldSegmentsDiskBytes=65651 1463590500247354636
+> influxdb_tsm1_wal,database=_internal,host=tyrion,path=/Users/sparrc/.influxdb/wal/_internal/monitor/4,retentionPolicy=monitor,url=http://localhost:8086/debug/vars currentSegmentDiskBytes=495687,oldSegmentsDiskBytes=0 1463590500247354636
+> influxdb_write,host=tyrion,url=http://localhost:8086/debug/vars pointReq=7274,pointReqLocal=7274,req=280,subWriteOk=280,writeOk=280 1463590500247354636
+> influxdb_shard,host=tyrion n_shards=4i 1463590500247354636
+```
+
+### InfluxDB-formatted endpoints
+
+The influxdb plugin can collect InfluxDB-formatted data from JSON endpoints.
+Whether associated with an Influx database or not.
 
 With a configuration of:
 
@@ -11,62 +108,3 @@ With a configuration of:
     "http://192.168.2.1:8086/debug/vars"
   ]
 ```
-
-And if 127.0.0.1 responds with this JSON:
-
-```json
-{
-  "k1": {
-    "name": "fruit",
-    "tags": {
-      "kind": "apple"
-    },
-    "values": {
-      "inventory": 371,
-      "sold": 112
-    }
-  },
-  "k2": {
-    "name": "fruit",
-    "tags": {
-      "kind": "banana"
-    },
-    "values": {
-      "inventory": 1000,
-      "sold": 403
-    }
-  }
-}
-```
-
-And if 192.168.2.1 responds like so:
-
-```json
-{
-  "k3": {
-    "name": "transactions",
-    "tags": {},
-    "values": {
-      "total": 100,
-      "balance": 184.75
-    }
-  }
-}
-```
-
-Then the collected metrics will be:
-
-```
-influxdb_fruit,url='http://127.0.0.1:8086/debug/vars',kind='apple' inventory=371.0,sold=112.0
-influxdb_fruit,url='http://127.0.0.1:8086/debug/vars',kind='banana' inventory=1000.0,sold=403.0
-
-influxdb_transactions,url='http://192.168.2.1:8086/debug/vars' total=100.0,balance=184.75
-```
-
-There are two important details to note about the collected metrics:
-
-1. Even though the values in JSON are being displayed as integers, the metrics are reported as floats.
-JSON encoders usually don't print the fractional part for round floats.
-Because you cannot change the type of an existing field in InfluxDB, we assume all numbers are floats.
-
-2. The top-level keys' names (in the example above, `"k1"`, `"k2"`, and `"k3"`) are not considered when recording the metrics.

plugins/inputs/influxdb/influxdb.go

@@ -7,6 +7,7 @@ import (
 	"net/http"
 	"strings"
 	"sync"
+	"time"
 
 	"github.com/influxdata/telegraf"
 	"github.com/influxdata/telegraf/plugins/inputs"
@@ -70,6 +71,44 @@ type point struct {
 	Values map[string]interface{} `json:"values"`
 }
 
+type memstats struct {
+	Alloc         int64   `json:"Alloc"`
+	TotalAlloc    int64   `json:"TotalAlloc"`
+	Sys           int64   `json:"Sys"`
+	Lookups       int64   `json:"Lookups"`
+	Mallocs       int64   `json:"Mallocs"`
+	Frees         int64   `json:"Frees"`
+	HeapAlloc     int64   `json:"HeapAlloc"`
+	HeapSys       int64   `json:"HeapSys"`
+	HeapIdle      int64   `json:"HeapIdle"`
+	HeapInuse     int64   `json:"HeapInuse"`
+	HeapReleased  int64   `json:"HeapReleased"`
+	HeapObjects   int64   `json:"HeapObjects"`
+	StackInuse    int64   `json:"StackInuse"`
+	StackSys      int64   `json:"StackSys"`
+	MSpanInuse    int64   `json:"MSpanInuse"`
+	MSpanSys      int64   `json:"MSpanSys"`
+	MCacheInuse   int64   `json:"MCacheInuse"`
+	MCacheSys     int64   `json:"MCacheSys"`
+	BuckHashSys   int64   `json:"BuckHashSys"`
+	GCSys         int64   `json:"GCSys"`
+	OtherSys      int64   `json:"OtherSys"`
+	NextGC        int64   `json:"NextGC"`
+	LastGC        int64   `json:"LastGC"`
+	PauseTotalNs  int64   `json:"PauseTotalNs"`
+	NumGC         int64   `json:"NumGC"`
+	GCCPUFraction float64 `json:"GCCPUFraction"`
+}
+
+var tr = &http.Transport{
+	ResponseHeaderTimeout: time.Duration(3 * time.Second),
+}
+
+var client = &http.Client{
+	Transport: tr,
+	Timeout:   time.Duration(4 * time.Second),
+}
+
 // Gathers data from a particular URL
 // Parameters:
 //     acc    : The telegraf Accumulator to use
@@ -81,7 +120,10 @@ func (i *InfluxDB) gatherURL(
 	acc telegraf.Accumulator,
 	url string,
 ) error {
-	resp, err := http.Get(url)
+	shardCounter := 0
+	now := time.Now()
+
+	resp, err := client.Get(url)
 	if err != nil {
 		return err
 	}
@@ -108,12 +150,52 @@ func (i *InfluxDB) gatherURL(
 			break
 		}
 
-		// Read in a string key. We don't do anything with the top-level keys, so it's discarded.
-		_, err := dec.Token()
+		// Read in a string key. We don't do anything with the top-level keys,
+		// so it's discarded.
+		key, err := dec.Token()
 		if err != nil {
 			return err
 		}
 
+		if key.(string) == "memstats" {
+			var m memstats
+			if err := dec.Decode(&m); err != nil {
+				continue
+			}
+			acc.AddFields("influxdb_memstats",
+				map[string]interface{}{
+					"alloc":           m.Alloc,
+					"total_alloc":     m.TotalAlloc,
+					"sys":             m.Sys,
+					"lookups":         m.Lookups,
+					"mallocs":         m.Mallocs,
+					"frees":           m.Frees,
+					"heap_alloc":      m.HeapAlloc,
+					"heap_sys":        m.HeapSys,
+					"heap_idle":       m.HeapIdle,
+					"heap_inuse":      m.HeapInuse,
+					"heap_released":   m.HeapReleased,
+					"heap_objects":    m.HeapObjects,
+					"stack_inuse":     m.StackInuse,
+					"stack_sys":       m.StackSys,
+					"mspan_inuse":     m.MSpanInuse,
+					"mspan_sys":       m.MSpanSys,
+					"mcache_inuse":    m.MCacheInuse,
+					"mcache_sys":      m.MCacheSys,
+					"buck_hash_sys":   m.BuckHashSys,
+					"gc_sys":          m.GCSys,
+					"other_sys":       m.OtherSys,
+					"next_gc":         m.NextGC,
+					"last_gc":         m.LastGC,
+					"pause_total_ns":  m.PauseTotalNs,
+					"num_gc":          m.NumGC,
+					"gcc_pu_fraction": m.GCCPUFraction,
+				},
+				map[string]string{
+					"url": url,
+				})
+		}
+
 		// Attempt to parse a whole object into a point.
 		// It might be a non-object, like a string or array.
 		// If we fail to decode it into a point, ignore it and move on.
@@ -122,7 +204,12 @@ func (i *InfluxDB) gatherURL(
 			continue
 		}
 
-		// If the object was a point, but was not fully initialized, ignore it and move on.
+		if p.Name == "shard" {
+			shardCounter++
+		}
+
+		// If the object was a point, but was not fully initialized,
+		// ignore it and move on.
 		if p.Name == "" || p.Tags == nil || p.Values == nil || len(p.Values) == 0 {
 			continue
 		}
@@ -134,9 +221,18 @@ func (i *InfluxDB) gatherURL(
 			"influxdb_"+p.Name,
 			p.Values,
 			p.Tags,
+			now,
 		)
 	}
 
+	acc.AddFields("influxdb",
+		map[string]interface{}{
+			"n_shards": shardCounter,
+		},
+		nil,
+		now,
+	)
+
 	return nil
 }
 

plugins/inputs/influxdb/influxdb_test.go

@@ -11,7 +11,148 @@ import (
 )
 
 func TestBasic(t *testing.T) {
-	js := `
+	fakeServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path == "/endpoint" {
+			_, _ = w.Write([]byte(basicJSON))
+		} else {
+			w.WriteHeader(http.StatusNotFound)
+		}
+	}))
+	defer fakeServer.Close()
+
+	plugin := &influxdb.InfluxDB{
+		URLs: []string{fakeServer.URL + "/endpoint"},
+	}
+
+	var acc testutil.Accumulator
+	require.NoError(t, plugin.Gather(&acc))
+
+	require.Len(t, acc.Metrics, 3)
+	fields := map[string]interface{}{
+		// JSON will truncate floats to integer representations.
+		// Since there's no distinction in JSON, we can't assume it's an int.
+		"i": -1.0,
+		"f": 0.5,
+		"b": true,
+		"s": "string",
+	}
+	tags := map[string]string{
+		"id":  "ex1",
+		"url": fakeServer.URL + "/endpoint",
+	}
+	acc.AssertContainsTaggedFields(t, "influxdb_foo", fields, tags)
+
+	fields = map[string]interface{}{
+		"x": "x",
+	}
+	tags = map[string]string{
+		"id":  "ex2",
+		"url": fakeServer.URL + "/endpoint",
+	}
+	acc.AssertContainsTaggedFields(t, "influxdb_bar", fields, tags)
+
+	acc.AssertContainsTaggedFields(t, "influxdb",
+		map[string]interface{}{
+			"n_shards": 0,
+		}, map[string]string{})
+}
+
+func TestInfluxDB(t *testing.T) {
+	fakeInfluxServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path == "/endpoint" {
+			_, _ = w.Write([]byte(influxReturn))
+		} else {
+			w.WriteHeader(http.StatusNotFound)
+		}
+	}))
+	defer fakeInfluxServer.Close()
+
+	plugin := &influxdb.InfluxDB{
+		URLs: []string{fakeInfluxServer.URL + "/endpoint"},
+	}
+
+	var acc testutil.Accumulator
+	require.NoError(t, plugin.Gather(&acc))
+
+	require.Len(t, acc.Metrics, 34)
+
+	fields := map[string]interface{}{
+		"heap_inuse":      int64(18046976),
+		"heap_released":   int64(3473408),
+		"mspan_inuse":     int64(97440),
+		"total_alloc":     int64(201739016),
+		"sys":             int64(38537464),
+		"mallocs":         int64(570251),
+		"frees":           int64(381008),
+		"heap_idle":       int64(15802368),
+		"pause_total_ns":  int64(5132914),
+		"lookups":         int64(77),
+		"heap_sys":        int64(33849344),
+		"mcache_sys":      int64(16384),
+		"next_gc":         int64(20843042),
+		"gcc_pu_fraction": float64(4.287178819113636e-05),
+		"other_sys":       int64(1229737),
+		"alloc":           int64(17034016),
+		"stack_inuse":     int64(753664),
+		"stack_sys":       int64(753664),
+		"buck_hash_sys":   int64(1461583),
+		"gc_sys":          int64(1112064),
+		"num_gc":          int64(27),
+		"heap_alloc":      int64(17034016),
+		"heap_objects":    int64(189243),
+		"mspan_sys":       int64(114688),
+		"mcache_inuse":    int64(4800),
+		"last_gc":         int64(1460434886475114239),
+	}
+
+	tags := map[string]string{
+		"url": fakeInfluxServer.URL + "/endpoint",
+	}
+	acc.AssertContainsTaggedFields(t, "influxdb_memstats", fields, tags)
+
+	acc.AssertContainsTaggedFields(t, "influxdb",
+		map[string]interface{}{
+			"n_shards": 2,
+		}, map[string]string{})
+}
+
+func TestErrorHandling(t *testing.T) {
+	badServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path == "/endpoint" {
+			_, _ = w.Write([]byte("not json"))
+		} else {
+			w.WriteHeader(http.StatusNotFound)
+		}
+	}))
+	defer badServer.Close()
+
+	plugin := &influxdb.InfluxDB{
+		URLs: []string{badServer.URL + "/endpoint"},
+	}
+
+	var acc testutil.Accumulator
+	require.Error(t, plugin.Gather(&acc))
+}
+
+func TestErrorHandling404(t *testing.T) {
+	badServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path == "/endpoint" {
+			_, _ = w.Write([]byte(basicJSON))
+		} else {
+			w.WriteHeader(http.StatusNotFound)
+		}
+	}))
+	defer badServer.Close()
+
+	plugin := &influxdb.InfluxDB{
+		URLs: []string{badServer.URL},
+	}
+
+	var acc testutil.Accumulator
+	require.Error(t, plugin.Gather(&acc))
+}
+
+const basicJSON = `
 {
   "_1": {
     "name": "foo",
@@ -55,43 +196,48 @@ func TestBasic(t *testing.T) {
   }
 }
 `
-	fakeServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		if r.URL.Path == "/endpoint" {
-			_, _ = w.Write([]byte(js))
-		} else {
-			w.WriteHeader(http.StatusNotFound)
-		}
-	}))
-	defer fakeServer.Close()
 
-	plugin := &influxdb.InfluxDB{
-		URLs: []string{fakeServer.URL + "/endpoint"},
-	}
-
-	var acc testutil.Accumulator
-	require.NoError(t, plugin.Gather(&acc))
-
-	require.Len(t, acc.Metrics, 2)
-	fields := map[string]interface{}{
-		// JSON will truncate floats to integer representations.
-		// Since there's no distinction in JSON, we can't assume it's an int.
-		"i": -1.0,
-		"f": 0.5,
-		"b": true,
-		"s": "string",
-	}
-	tags := map[string]string{
-		"id":  "ex1",
-		"url": fakeServer.URL + "/endpoint",
-	}
-	acc.AssertContainsTaggedFields(t, "influxdb_foo", fields, tags)
-
-	fields = map[string]interface{}{
-		"x": "x",
-	}
-	tags = map[string]string{
-		"id":  "ex2",
-		"url": fakeServer.URL + "/endpoint",
-	}
-	acc.AssertContainsTaggedFields(t, "influxdb_bar", fields, tags)
-}
+const influxReturn = `
+{
+"cluster": {"name": "cluster", "tags": {}, "values": {}},
+"cmdline": ["influxd"],
+"cq": {"name": "cq", "tags": {}, "values": {}},
+"database:_internal": {"name": "database", "tags": {"database": "_internal"}, "values": {"numMeasurements": 8, "numSeries": 12}},
+"database:udp": {"name": "database", "tags": {"database": "udp"}, "values": {"numMeasurements": 14, "numSeries": 38}},
+"hh:/Users/csparr/.influxdb/hh": {"name": "hh", "tags": {"path": "/Users/csparr/.influxdb/hh"}, "values": {}},
+"httpd::8086": {"name": "httpd", "tags": {"bind": ":8086"}, "values": {"req": 7, "reqActive": 1, "reqDurationNs": 4488799}},
+"measurement:cpu_idle.udp": {"name": "measurement", "tags": {"database": "udp", "measurement": "cpu_idle"}, "values": {"numSeries": 1}},
+"measurement:cpu_usage.udp": {"name": "measurement", "tags": {"database": "udp", "measurement": "cpu_usage"}, "values": {"numSeries": 1}},
+"measurement:database._internal": {"name": "measurement", "tags": {"database": "_internal", "measurement": "database"}, "values": {"numSeries": 2}},
+"measurement:database.udp": {"name": "measurement", "tags": {"database": "udp", "measurement": "database"}, "values": {"numSeries": 2}},
+"measurement:httpd.udp": {"name": "measurement", "tags": {"database": "udp", "measurement": "httpd"}, "values": {"numSeries": 1}},
+"measurement:measurement.udp": {"name": "measurement", "tags": {"database": "udp", "measurement": "measurement"}, "values": {"numSeries": 22}},
+"measurement:mem.udp": {"name": "measurement", "tags": {"database": "udp", "measurement": "mem"}, "values": {"numSeries": 1}},
+"measurement:net.udp": {"name": "measurement", "tags": {"database": "udp", "measurement": "net"}, "values": {"numSeries": 1}},
+"measurement:runtime._internal": {"name": "measurement", "tags": {"database": "_internal", "measurement": "runtime"}, "values": {"numSeries": 1}},
+"measurement:runtime.udp": {"name": "measurement", "tags": {"database": "udp", "measurement": "runtime"}, "values": {"numSeries": 1}},
+"measurement:shard._internal": {"name": "measurement", "tags": {"database": "_internal", "measurement": "shard"}, "values": {"numSeries": 2}},
+"measurement:shard.udp": {"name": "measurement", "tags": {"database": "udp", "measurement": "shard"}, "values": {"numSeries": 1}},
+"measurement:subscriber._internal": {"name": "measurement", "tags": {"database": "_internal", "measurement": "subscriber"}, "values": {"numSeries": 1}},
+"measurement:subscriber.udp": {"name": "measurement", "tags": {"database": "udp", "measurement": "subscriber"}, "values": {"numSeries": 1}},
+"measurement:swap_used.udp": {"name": "measurement", "tags": {"database": "udp", "measurement": "swap_used"}, "values": {"numSeries": 1}},
+"measurement:tsm1_cache._internal": {"name": "measurement", "tags": {"database": "_internal", "measurement": "tsm1_cache"}, "values": {"numSeries": 2}},
+"measurement:tsm1_cache.udp": {"name": "measurement", "tags": {"database": "udp", "measurement": "tsm1_cache"}, "values": {"numSeries": 2}},
+"measurement:tsm1_wal._internal": {"name": "measurement", "tags": {"database": "_internal", "measurement": "tsm1_wal"}, "values": {"numSeries": 2}},
+"measurement:tsm1_wal.udp": {"name": "measurement", "tags": {"database": "udp", "measurement": "tsm1_wal"}, "values": {"numSeries": 2}},
+"measurement:udp._internal": {"name": "measurement", "tags": {"database": "_internal", "measurement": "udp"}, "values": {"numSeries": 1}},
+"measurement:write._internal": {"name": "measurement", "tags": {"database": "_internal", "measurement": "write"}, "values": {"numSeries": 1}},
+"measurement:write.udp": {"name": "measurement", "tags": {"database": "udp", "measurement": "write"}, "values": {"numSeries": 1}},
+"memstats": {"Alloc":17034016,"TotalAlloc":201739016,"Sys":38537464,"Lookups":77,"Mallocs":570251,"Frees":381008,"HeapAlloc":17034016,"HeapSys":33849344,"HeapIdle":15802368,"HeapInuse":18046976,"HeapReleased":3473408,"HeapObjects":189243,"StackInuse":753664,"StackSys":753664,"MSpanInuse":97440,"MSpanSys":114688,"MCacheInuse":4800,"MCacheSys":16384,"BuckHashSys":1461583,"GCSys":1112064,"OtherSys":1229737,"NextGC":20843042,"LastGC":1460434886475114239,"PauseTotalNs":5132914,"PauseNs":[195052,117751,139370,156933,263089,165249,713747,103904,122015,294408,213753,170864,175845,114221,121563,122409,113098,162219,229257,126726,250774,254235,117206,293588,144279,124306,127053,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"PauseEnd":[1460433856394860455,1460433856398162739,1460433856405888337,1460433856411784017,1460433856417924684,1460433856428385687,1460433856443782908,1460433856456522851,1460433857392743223,1460433866484394564,1460433866494076235,1460433896472438632,1460433957839825106,1460433976473440328,1460434016473413006,1460434096471892794,1460434126470792929,1460434246480428250,1460434366554468369,1460434396471249528,1460434456471205885,1460434476479487292,1460434536471435965,1460434616469784776,1460434736482078216,1460434856544251733,1460434886475114239,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"NumGC":27,"GCCPUFraction":4.287178819113636e-05,"EnableGC":true,"DebugGC":false,"BySize":[{"Size":0,"Mallocs":0,"Frees":0},{"Size":8,"Mallocs":1031,"Frees":955},{"Size":16,"Mallocs":308485,"Frees":142064},{"Size":32,"Mallocs":64937,"Frees":54321},{"Size":48,"Mallocs":33012,"Frees":29754},{"Size":64,"Mallocs":20299,"Frees":18173},{"Size":80,"Mallocs":8186,"Frees":7597},{"Size":96,"Mallocs":9806,"Frees":8982},{"Size":112,"Mallocs":5671,"Frees":4850},{"Size":128,"Mallocs":2972,"Frees":2684},{"Size":144,"Mallocs":4106,"Frees":3719},{"Size":160,"Mallocs":1324,"Frees":911},{"Size":176,"Mallocs":2574,"Frees":2391},{"Size":192,"Mallocs":4053,"Frees":3863},{"Size":208,"Mallocs":442,"Frees":307},{"Size":224,"Mallocs":336,"Frees":172},{"Size":240,"Mallocs":143,"Frees":125},{"Size":256,"Mallocs":542,"Frees":497},{"Size":288,"Mallocs":15971,"Frees":14761},{"Size":320,"Mallocs":245,"Frees":30},{"Size":352,"Mallocs":1299,"Frees":1065},{"Size":384,"Mallocs":138,"Frees":2},{"Size":416,"Mallocs":54,"Frees":47},{"Size":448,"Mallocs":75,"Frees":29},{"Size":480,"Mallocs":6,"Frees":4},{"Size":512,"Mallocs":452,"Frees":422},{"Size":576,"Mallocs":486,"Frees":395},{"Size":640,"Mallocs":81,"Frees":67},{"Size":704,"Mallocs":421,"Frees":397},{"Size":768,"Mallocs":469,"Frees":468},{"Size":896,"Mallocs":1049,"Frees":1010},{"Size":1024,"Mallocs":1078,"Frees":960},{"Size":1152,"Mallocs":750,"Frees":498},{"Size":1280,"Mallocs":84,"Frees":72},{"Size":1408,"Mallocs":218,"Frees":187},{"Size":1536,"Mallocs":73,"Frees":48},{"Size":1664,"Mallocs":43,"Frees":30},{"Size":2048,"Mallocs":153,"Frees":57},{"Size":2304,"Mallocs":41,"Frees":30},{"Size":2560,"Mallocs":18,"Frees":15},{"Size":2816,"Mallocs":164,"Frees":157},{"Size":3072,"Mallocs":0,"Frees":0},{"Size":3328,"Mallocs":13,"Frees":6},{"Size":4096,"Mallocs":101,"Frees":82},{"Size":4608,"Mallocs":32,"Frees":26},{"Size":5376,"Mallocs":165,"Frees":151},{"Size":6144,"Mallocs":15,"Frees":9},{"Size":6400,"Mallocs":1,"Frees":1},{"Size":6656,"Mallocs":1,"Frees":0},{"Size":6912,"Mallocs":0,"Frees":0},{"Size":8192,"Mallocs":13,"Frees":13},{"Size":8448,"Mallocs":0,"Frees":0},{"Size":8704,"Mallocs":1,"Frees":1},{"Size":9472,"Mallocs":6,"Frees":4},{"Size":10496,"Mallocs":0,"Frees":0},{"Size":12288,"Mallocs":41,"Frees":35},{"Size":13568,"Mallocs":0,"Frees":0},{"Size":14080,"Mallocs":0,"Frees":0},{"Size":16384,"Mallocs":4,"Frees":4},{"Size":16640,"Mallocs":0,"Frees":0},{"Size":17664,"Mallocs":0,"Frees":0}]},
+"queryExecutor": {"name": "queryExecutor", "tags": {}, "values": {}},
+"shard:/Users/csparr/.influxdb/data/_internal/monitor/2:2": {"name": "shard", "tags": {"database": "_internal", "engine": "tsm1", "id": "2", "path": "/Users/csparr/.influxdb/data/_internal/monitor/2", "retentionPolicy": "monitor"}, "values": {}},
+"shard:/Users/csparr/.influxdb/data/udp/default/1:1": {"name": "shard", "tags": {"database": "udp", "engine": "tsm1", "id": "1", "path": "/Users/csparr/.influxdb/data/udp/default/1", "retentionPolicy": "default"}, "values": {"fieldsCreate": 61, "seriesCreate": 33, "writePointsOk": 3613, "writeReq": 110}},
+"subscriber": {"name": "subscriber", "tags": {}, "values": {"pointsWritten": 3613}},
+"tsm1_cache:/Users/csparr/.influxdb/data/_internal/monitor/2": {"name": "tsm1_cache", "tags": {"database": "_internal", "path": "/Users/csparr/.influxdb/data/_internal/monitor/2", "retentionPolicy": "monitor"}, "values": {"WALCompactionTimeMs": 0, "cacheAgeMs": 1103932, "cachedBytes": 0, "diskBytes": 0, "memBytes": 40480, "snapshotCount": 0}},
+"tsm1_cache:/Users/csparr/.influxdb/data/udp/default/1": {"name": "tsm1_cache", "tags": {"database": "udp", "path": "/Users/csparr/.influxdb/data/udp/default/1", "retentionPolicy": "default"}, "values": {"WALCompactionTimeMs": 0, "cacheAgeMs": 1103029, "cachedBytes": 0, "diskBytes": 0, "memBytes": 2359472, "snapshotCount": 0}},
+"tsm1_filestore:/Users/csparr/.influxdb/data/_internal/monitor/2": {"name": "tsm1_filestore", "tags": {"database": "_internal", "path": "/Users/csparr/.influxdb/data/_internal/monitor/2", "retentionPolicy": "monitor"}, "values": {}},
+"tsm1_filestore:/Users/csparr/.influxdb/data/udp/default/1": {"name": "tsm1_filestore", "tags": {"database": "udp", "path": "/Users/csparr/.influxdb/data/udp/default/1", "retentionPolicy": "default"}, "values": {}},
+"tsm1_wal:/Users/csparr/.influxdb/wal/_internal/monitor/2": {"name": "tsm1_wal", "tags": {"database": "_internal", "path": "/Users/csparr/.influxdb/wal/_internal/monitor/2", "retentionPolicy": "monitor"}, "values": {"currentSegmentDiskBytes": 0, "oldSegmentsDiskBytes": 69532}},
+"tsm1_wal:/Users/csparr/.influxdb/wal/udp/default/1": {"name": "tsm1_wal", "tags": {"database": "udp", "path": "/Users/csparr/.influxdb/wal/udp/default/1", "retentionPolicy": "default"}, "values": {"currentSegmentDiskBytes": 193728, "oldSegmentsDiskBytes": 1008330}},
+"write": {"name": "write", "tags": {}, "values": {"pointReq": 3613, "pointReqLocal": 3613, "req": 110, "subWriteOk": 110, "writeOk": 110}}
+}`

plugins/inputs/ipmi_sensor/README.md

@@ -0,0 +1,42 @@
+# Telegraf ipmi plugin
+
+Get bare metal metrics using the command line utility `ipmitool`
+
+see ipmitool(https://sourceforge.net/projects/ipmitool/files/ipmitool/)
+
+The plugin will use the following command to collect remote host sensor stats:
+
+ipmitool -I lan -H 192.168.1.1 -U USERID -P PASSW0RD sdr
+
+## Measurements
+
+- ipmi_sensor:
+
+    * Tags: `name`, `server`, `unit`
+    * Fields:
+      - status
+      - value
+
+## Configuration
+
+```toml
+[[inputs.ipmi_sensor]]
+  ## specify servers via a url matching:
+  ##  [username[:password]@][protocol[(address)]]
+  ##  e.g.
+  ##    root:passwd@lan(127.0.0.1)
+  ##
+  servers = ["USERID:PASSW0RD@lan(10.20.2.203)"]
+```
+
+## Output
+
+```
+> ipmi_sensor,server=10.20.2.203,unit=degrees_c,name=ambient_temp status=1i,value=20 1458488465012559455
+> ipmi_sensor,server=10.20.2.203,unit=feet,name=altitude status=1i,value=80 1458488465012688613
+> ipmi_sensor,server=10.20.2.203,unit=watts,name=avg_power status=1i,value=220 1458488465012776511
+> ipmi_sensor,server=10.20.2.203,unit=volts,name=planar_3.3v status=1i,value=3.28 1458488465012861875
+> ipmi_sensor,server=10.20.2.203,unit=volts,name=planar_vbat status=1i,value=3.04 1458488465013072508
+> ipmi_sensor,server=10.20.2.203,unit=rpm,name=fan_1a_tach status=1i,value=2610 1458488465013137932
+> ipmi_sensor,server=10.20.2.203,unit=rpm,name=fan_1b_tach status=1i,value=1775 1458488465013279896
+```

plugins/inputs/ipmi_sensor/command.go

@@ -0,0 +1,35 @@
+package ipmi_sensor
+
+import (
+	"fmt"
+	"os/exec"
+	"strings"
+	"time"
+
+	"github.com/influxdata/telegraf/internal"
+)
+
+type CommandRunner struct{}
+
+func (t CommandRunner) cmd(conn *Connection, args ...string) *exec.Cmd {
+	path := conn.Path
+	opts := append(conn.options(), args...)
+
+	if path == "" {
+		path = "ipmitool"
+	}
+
+	return exec.Command(path, opts...)
+}
+
+func (t CommandRunner) Run(conn *Connection, args ...string) (string, error) {
+	cmd := t.cmd(conn, args...)
+
+	output, err := internal.CombinedOutputTimeout(cmd, time.Second*5)
+	if err != nil {
+		return "", fmt.Errorf("run %s %s: %s (%s)",
+			cmd.Path, strings.Join(cmd.Args, " "), string(output), err)
+	}
+
+	return string(output), err
+}

plugins/inputs/ipmi_sensor/connection.go

@@ -0,0 +1,89 @@
+package ipmi_sensor
+
+import (
+	"fmt"
+	"net"
+	"strconv"
+	"strings"
+)
+
+// Connection properties for a Client
+type Connection struct {
+	Hostname  string
+	Username  string
+	Password  string
+	Path      string
+	Port      int
+	Interface string
+}
+
+func NewConnection(server string) *Connection {
+	conn := &Connection{}
+	inx1 := strings.Index(server, "@")
+	inx2 := strings.Index(server, "(")
+	inx3 := strings.Index(server, ")")
+
+	connstr := server
+
+	if inx1 > 0 {
+		security := server[0:inx1]
+		connstr = server[inx1+1 : len(server)]
+		up := strings.SplitN(security, ":", 2)
+		conn.Username = up[0]
+		conn.Password = up[1]
+	}
+
+	if inx2 > 0 {
+		inx2 = strings.Index(connstr, "(")
+		inx3 = strings.Index(connstr, ")")
+
+		conn.Interface = connstr[0:inx2]
+		conn.Hostname = connstr[inx2+1 : inx3]
+	}
+
+	return conn
+}
+
+func (t *Connection) options() []string {
+	intf := t.Interface
+	if intf == "" {
+		intf = "lan"
+	}
+
+	options := []string{
+		"-H", t.Hostname,
+		"-U", t.Username,
+		"-P", t.Password,
+		"-I", intf,
+	}
+
+	if t.Port != 0 {
+		options = append(options, "-p", strconv.Itoa(t.Port))
+	}
+
+	return options
+}
+
+// RemoteIP returns the remote (bmc) IP address of the Connection
+func (c *Connection) RemoteIP() string {
+	if net.ParseIP(c.Hostname) == nil {
+		addrs, err := net.LookupHost(c.Hostname)
+		if err != nil && len(addrs) > 0 {
+			return addrs[0]
+		}
+	}
+	return c.Hostname
+}
+
+// LocalIP returns the local (client) IP address of the Connection
+func (c *Connection) LocalIP() string {
+	conn, err := net.Dial("udp", fmt.Sprintf("%s:%d", c.Hostname, c.Port))
+	if err != nil {
+		// don't bother returning an error, since this value will never
+		// make it to the bmc if we can't connect to it.
+		return c.Hostname
+	}
+	_ = conn.Close()
+	host, _, _ := net.SplitHostPort(conn.LocalAddr().String())
+	return host
+}

plugins/inputs/ipmi_sensor/ipmi.go

@@ -0,0 +1,129 @@
+package ipmi_sensor
+
+import (
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/influxdata/telegraf"
+	"github.com/influxdata/telegraf/plugins/inputs"
+)
+
+type Ipmi struct {
+	Servers []string
+	runner  Runner
+}
+
+var sampleConfig = `
+  ## specify servers via a url matching:
+  ##  [username[:password]@][protocol[(address)]]
+  ##  e.g.
+  ##    root:passwd@lan(127.0.0.1)
+  ##
+  servers = ["USERID:PASSW0RD@lan(192.168.1.1)"]
+`
+
+func NewIpmi() *Ipmi {
+	return &Ipmi{
+		runner: CommandRunner{},
+	}
+}
+
+func (m *Ipmi) SampleConfig() string {
+	return sampleConfig
+}
+
+func (m *Ipmi) Description() string {
+	return "Read metrics from one or many bare metal servers"
+}
+
+func (m *Ipmi) Gather(acc telegraf.Accumulator) error {
+	if m.runner == nil {
+		m.runner = CommandRunner{}
+	}
+	for _, serv := range m.Servers {
+		err := m.gatherServer(serv, acc)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (m *Ipmi) gatherServer(serv string, acc telegraf.Accumulator) error {
+	conn := NewConnection(serv)
+
+	res, err := m.runner.Run(conn, "sdr")
+	if err != nil {
+		return err
+	}
+
+	// each line will look something like
+	// Planar VBAT      | 3.05 Volts        | ok
+	lines := strings.Split(res, "\n")
+	for i := 0; i < len(lines); i++ {
+		vals := strings.Split(lines[i], "|")
+		if len(vals) != 3 {
+			continue
+		}
+
+		tags := map[string]string{
+			"server": conn.Hostname,
+			"name":   transform(vals[0]),
+		}
+
+		fields := make(map[string]interface{})
+		if strings.EqualFold("ok", trim(vals[2])) {
+			fields["status"] = 1
+		} else {
+			fields["status"] = 0
+		}
+
+		val1 := trim(vals[1])
+
+		if strings.Index(val1, " ") > 0 {
+			// split middle column into value and unit
+			valunit := strings.SplitN(val1, " ", 2)
+			fields["value"] = Atofloat(valunit[0])
+			if len(valunit) > 1 {
+				tags["unit"] = transform(valunit[1])
+			}
+		} else {
+			fields["value"] = 0.0
+		}
+
+		acc.AddFields("ipmi_sensor", fields, tags, time.Now())
+	}
+
+	return nil
+}
+
+type Runner interface {
+	Run(conn *Connection, args ...string) (string, error)
+}
+
+func Atofloat(val string) float64 {
+	f, err := strconv.ParseFloat(val, 64)
+	if err != nil {
+		return 0.0
+	} else {
+		return f
+	}
+}
+
+func trim(s string) string {
+	return strings.TrimSpace(s)
+}
+
+func transform(s string) string {
+	s = trim(s)
+	s = strings.ToLower(s)
+	return strings.Replace(s, " ", "_", -1)
+}
+
+func init() {
+	inputs.Add("ipmi_sensor", func() telegraf.Input {
+		return &Ipmi{}
+	})
+}

plugins/inputs/ipmi_sensor/ipmi_test.go

@@ -0,0 +1,275 @@
+package ipmi_sensor
+
+import (
+	"testing"
+
+	"github.com/influxdata/telegraf/testutil"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+const serv = "USERID:PASSW0RD@lan(192.168.1.1)"
+
+const cmdReturn = `
+Ambient Temp     | 20 degrees C      | ok
+Altitude         | 80 feet           | ok
+Avg Power        | 210 Watts         | ok
+Planar 3.3V      | 3.29 Volts        | ok
+Planar 5V        | 4.90 Volts        | ok
+Planar 12V       | 12.04 Volts       | ok
+Planar VBAT      | 3.05 Volts        | ok
+Fan 1A Tach      | 2610 RPM          | ok
+Fan 1B Tach      | 1775 RPM          | ok
+Fan 2A Tach      | 2001 RPM          | ok
+Fan 2B Tach      | 1275 RPM          | ok
+Fan 3A Tach      | 2929 RPM          | ok
+Fan 3B Tach      | 2125 RPM          | ok
+Fan 1            | 0x00              | ok
+Fan 2            | 0x00              | ok
+Fan 3            | 0x00              | ok
+Front Panel      | 0x00              | ok
+Video USB        | 0x00              | ok
+DASD Backplane 1 | 0x00              | ok
+SAS Riser        | 0x00              | ok
+PCI Riser 1      | 0x00              | ok
+PCI Riser 2      | 0x00              | ok
+CPU 1            | 0x00              | ok
+CPU 2            | 0x00              | ok
+All CPUs         | 0x00              | ok
+One of The CPUs  | 0x00              | ok
+IOH Temp Status  | 0x00              | ok
+CPU 1 OverTemp   | 0x00              | ok
+CPU 2 OverTemp   | 0x00              | ok
+CPU Fault Reboot | 0x00              | ok
+Aux Log          | 0x00              | ok
+NMI State        | 0x00              | ok
+ABR Status       | 0x00              | ok
+Firmware Error   | 0x00              | ok
+PCIs             | 0x00              | ok
+CPUs             | 0x00              | ok
+DIMMs            | 0x00              | ok
+Sys Board Fault  | 0x00              | ok
+Power Supply 1   | 0x00              | ok
+Power Supply 2   | 0x00              | ok
+PS 1 Fan Fault   | 0x00              | ok
+PS 2 Fan Fault   | 0x00              | ok
+VT Fault         | 0x00              | ok
+Pwr Rail A Fault | 0x00              | ok
+Pwr Rail B Fault | 0x00              | ok
+Pwr Rail C Fault | 0x00              | ok
+Pwr Rail D Fault | 0x00              | ok
+Pwr Rail E Fault | 0x00              | ok
+PS 1 Therm Fault | 0x00              | ok
+PS 2 Therm Fault | 0x00              | ok
+PS1 12V OV Fault | 0x00              | ok
+PS2 12V OV Fault | 0x00              | ok
+PS1 12V UV Fault | 0x00              | ok
+PS2 12V UV Fault | 0x00              | ok
+PS1 12V OC Fault | 0x00              | ok
+PS2 12V OC Fault | 0x00              | ok
+PS 1 VCO Fault   | 0x00              | ok
+PS 2 VCO Fault   | 0x00              | ok
+Power Unit       | 0x00              | ok
+Cooling Zone 1   | 0x00              | ok
+Cooling Zone 2   | 0x00              | ok
+Cooling Zone 3   | 0x00              | ok
+Drive 0          | 0x00              | ok
+Drive 1          | 0x00              | ok
+Drive 2          | 0x00              | ok
+Drive 3          | 0x00              | ok
+Drive 4          | 0x00              | ok
+Drive 5          | 0x00              | ok
+Drive 6          | 0x00              | ok
+Drive 7          | 0x00              | ok
+Drive 8          | 0x00              | ok
+Drive 9          | 0x00              | ok
+Drive 10         | 0x00              | ok
+Drive 11         | 0x00              | ok
+Drive 12         | 0x00              | ok
+Drive 13         | 0x00              | ok
+Drive 14         | 0x00              | ok
+Drive 15         | 0x00              | ok
+All DIMMS        | 0x00              | ok
+One of the DIMMs | 0x00              | ok
+DIMM 1           | 0x00              | ok
+DIMM 2           | 0x00              | ok
+DIMM 3           | 0x00              | ok
+DIMM 4           | 0x00              | ok
+DIMM 5           | 0x00              | ok
+DIMM 6           | 0x00              | ok
+DIMM 7           | 0x00              | ok
+DIMM 8           | 0x00              | ok
+DIMM 9           | 0x00              | ok
+DIMM 10          | 0x00              | ok
+DIMM 11          | 0x00              | ok
+DIMM 12          | 0x00              | ok
+DIMM 13          | 0x00              | ok
+DIMM 14          | 0x00              | ok
+DIMM 15          | 0x00              | ok
+DIMM 16          | 0x00              | ok
+DIMM 17          | 0x00              | ok
+DIMM 18          | 0x00              | ok
+DIMM 1 Temp      | 0x00              | ok
+DIMM 2 Temp      | 0x00              | ok
+DIMM 3 Temp      | 0x00              | ok
+DIMM 4 Temp      | 0x00              | ok
+DIMM 5 Temp      | 0x00              | ok
+DIMM 6 Temp      | 0x00              | ok
+DIMM 7 Temp      | 0x00              | ok
+DIMM 8 Temp      | 0x00              | ok
+DIMM 9 Temp      | 0x00              | ok
+DIMM 10 Temp     | 0x00              | ok
+DIMM 11 Temp     | 0x00              | ok
+DIMM 12 Temp     | 0x00              | ok
+DIMM 13 Temp     | 0x00              | ok
+DIMM 14 Temp     | 0x00              | ok
+DIMM 15 Temp     | 0x00              | ok
+DIMM 16 Temp     | 0x00              | ok
+DIMM 17 Temp     | 0x00              | ok
+DIMM 18 Temp     | 0x00              | ok
+PCI 1            | 0x00              | ok
+PCI 2            | 0x00              | ok
+PCI 3            | 0x00              | ok
+PCI 4            | 0x00              | ok
+All PCI Error    | 0x00              | ok
+One of PCI Error | 0x00              | ok
+IPMI Watchdog    | 0x00              | ok
+Host Power       | 0x00              | ok
+DASD Backplane 2 | 0x00              | ok
+DASD Backplane 3 | Not Readable      | ns
+DASD Backplane 4 | Not Readable      | ns
+Backup Memory    | 0x00              | ok
+Progress         | 0x00              | ok
+Planar Fault     | 0x00              | ok
+SEL Fullness     | 0x00              | ok
+PCI 5            | 0x00              | ok
+OS RealTime Mod  | 0x00              | ok
+`
+
+type runnerMock struct {
+	out string
+	err error
+}
+
+func newRunnerMock(out string, err error) Runner {
+	return &runnerMock{
+		out: out,
+		err: err,
+	}
+}
+
+func (r runnerMock) Run(conn *Connection, args ...string) (out string, err error) {
+	if r.err != nil {
+		return out, r.err
+	}
+	return r.out, nil
+}
+
+func TestIpmi(t *testing.T) {
+	i := &Ipmi{
+		Servers: []string{"USERID:PASSW0RD@lan(192.168.1.1)"},
+		runner:  newRunnerMock(cmdReturn, nil),
+	}
+
+	var acc testutil.Accumulator
+
+	err := i.Gather(&acc)
+
+	require.NoError(t, err)
+
+	assert.Equal(t, acc.NFields(), 266, "non-numeric measurements should be ignored")
+
+	var tests = []struct {
+		fields map[string]interface{}
+		tags   map[string]string
+	}{
+		{
+			map[string]interface{}{
+				"value":  float64(20),
+				"status": int(1),
+			},
+			map[string]string{
+				"name":   "ambient_temp",
+				"server": "192.168.1.1",
+				"unit":   "degrees_c",
+			},
+		},
+		{
+			map[string]interface{}{
+				"value":  float64(80),
+				"status": int(1),
+			},
+			map[string]string{
+				"name":   "altitude",
+				"server": "192.168.1.1",
+				"unit":   "feet",
+			},
+		},
+		{
+			map[string]interface{}{
+				"value":  float64(210),
+				"status": int(1),
+			},
+			map[string]string{
+				"name":   "avg_power",
+				"server": "192.168.1.1",
+				"unit":   "watts",
+			},
+		},
+		{
+			map[string]interface{}{
+				"value":  float64(4.9),
+				"status": int(1),
+			},
+			map[string]string{
+				"name":   "planar_5v",
+				"server": "192.168.1.1",
+				"unit":   "volts",
+			},
+		},
+		{
+			map[string]interface{}{
+				"value":  float64(3.05),
+				"status": int(1),
+			},
+			map[string]string{
+				"name":   "planar_vbat",
+				"server": "192.168.1.1",
+				"unit":   "volts",
+			},
+		},
+		{
+			map[string]interface{}{
+				"value":  float64(2610),
+				"status": int(1),
+			},
+			map[string]string{
+				"name":   "fan_1a_tach",
+				"server": "192.168.1.1",
+				"unit":   "rpm",
+			},
+		},
+		{
+			map[string]interface{}{
+				"value":  float64(1775),
+				"status": int(1),
+			},
+			map[string]string{
+				"name":   "fan_1b_tach",
+				"server": "192.168.1.1",
+				"unit":   "rpm",
+			},
+		},
+	}
+
+	for _, test := range tests {
+		acc.AssertContainsTaggedFields(t, "ipmi_sensor", test.fields, test.tags)
+	}
+}
+
+func TestIpmiConnection(t *testing.T) {
+	conn := NewConnection(serv)
+	assert.Equal(t, "USERID", conn.Username)
+	assert.Equal(t, "lan", conn.Interface)
+
+}

plugins/inputs/jolokia/README.md

@@ -1,51 +1,61 @@
 # Telegraf plugin: Jolokia
 
-#### Plugin arguments:
-- **context** string: Context root used of jolokia url
-- **servers** []Server: List of servers
-  + **name** string: Server's logical name
-  + **host** string: Server's ip address or hostname
-  + **port** string: Server's listening port
-- **metrics** []Metric
-  + **name** string: Name of the measure
-  + **jmx** string: Jmx path that identifies mbeans attributes
-  + **pass** []string: Attributes to retain when collecting values
-  + **drop** []string: Attributes to drop when collecting values
+#### Configuration
+
+```toml
+# Read JMX metrics through Jolokia
+[[inputs.jolokia]]
+  ## This is the context root used to compose the jolokia url
+  context = "/jolokia"
+
+  ## This specifies the mode used
+  # mode = "proxy"
+  #
+  ## When in proxy mode this section is used to specify further
+  ## proxy address configurations.
+  ## Remember to change host address to fit your environment.
+  # [inputs.jolokia.proxy]
+  #   host = "127.0.0.1"
+  #   port = "8080"
+
+
+  ## List of servers exposing jolokia read service
+  [[inputs.jolokia.servers]]
+    name = "as-server-01"
+    host = "127.0.0.1"
+    port = "8080"
+    # username = "myuser"
+    # password = "mypassword"
+
+  ## List of metrics collected on above servers
+  ## Each metric consists in a name, a jmx path and either
+  ## a pass or drop slice attribute.
+  ## This collect all heap memory usage metrics.
+  [[inputs.jolokia.metrics]]
+    name = "heap_memory_usage"
+    mbean  = "java.lang:type=Memory"
+    attribute = "HeapMemoryUsage"
+
+  ## This collect thread counts metrics.
+  [[inputs.jolokia.metrics]]
+    name = "thread_count"
+    mbean  = "java.lang:type=Threading"
+    attribute = "TotalStartedThreadCount,ThreadCount,DaemonThreadCount,PeakThreadCount"
+
+  ## This collect number of class loaded/unloaded counts metrics.
+  [[inputs.jolokia.metrics]]
+    name = "class_count"
+    mbean  = "java.lang:type=ClassLoading"
+    attribute = "LoadedClassCount,UnloadedClassCount,TotalLoadedClassCount"
+```
 
 #### Description
 
-The Jolokia plugin collects JVM metrics exposed as MBean's attributes through jolokia REST endpoint. All metrics
-are collected for each server configured.
+The Jolokia plugin collects JVM metrics exposed as MBean's attributes through
+jolokia REST endpoint. All metrics are collected for each server configured.
 
 See: https://jolokia.org/
 
 # Measurements:
-Jolokia plugin produces one measure for each metric configured, adding Server's `name`, `host` and `port` as tags.
-
-Given a configuration like:
-
-```ini
-[jolokia]
-
-[[jolokia.servers]]
-  name = "as-service-1"
-  host = "127.0.0.1"
-  port = "8080"
-
-[[jolokia.servers]]
-  name = "as-service-2"
-  host = "127.0.0.1"
-  port = "8180"
-
-[[jolokia.metrics]]
-  name = "heap_memory_usage"
-  jmx  = "/java.lang:type=Memory/HeapMemoryUsage"
-  pass = ["used", "max"]
-```
-
-The collected metrics will be:
-
-```
-jolokia_heap_memory_usage name=as-service-1,host=127.0.0.1,port=8080 used=xxx,max=yyy
-jolokia_heap_memory_usage name=as-service-2,host=127.0.0.1,port=8180 used=vvv,max=zzz
-```
+Jolokia plugin produces one measure for each metric configured,
+adding Server's `jolokia_name`, `jolokia_host` and `jolokia_port` as tags.

plugins/inputs/jolokia/jolokia.go

@@ -1,12 +1,14 @@
 package jolokia
 
 import (
+	"bytes"
 	"encoding/json"
 	"errors"
 	"fmt"
 	"io/ioutil"
 	"net/http"
 	"net/url"
+	"time"
 
 	"github.com/influxdata/telegraf"
 	"github.com/influxdata/telegraf/plugins/inputs"
@@ -21,8 +23,10 @@ type Server struct {
 }
 
 type Metric struct {
-	Name string
-	Jmx  string
+	Name      string
+	Mbean     string
+	Attribute string
+	Path      string
 }
 
 type JolokiaClient interface {
@@ -40,20 +44,32 @@ func (c JolokiaClientImpl) MakeRequest(req *http.Request) (*http.Response, error
 type Jolokia struct {
 	jClient JolokiaClient
 	Context string
+	Mode    string
 	Servers []Server
 	Metrics []Metric
+	Proxy   Server
 }
 
-func (j *Jolokia) SampleConfig() string {
-	return `
+const sampleConfig = `
   ## This is the context root used to compose the jolokia url
-  context = "/jolokia/read"
+  context = "/jolokia"
+
+  ## This specifies the mode used
+  # mode = "proxy"
+  #
+  ## When in proxy mode this section is used to specify further
+  ## proxy address configurations.
+  ## Remember to change host address to fit your environment.
+  # [inputs.jolokia.proxy]
+  #   host = "127.0.0.1"
+  #   port = "8080"
+
 
   ## List of servers exposing jolokia read service
   [[inputs.jolokia.servers]]
-    name = "stable"
-    host = "192.168.103.2"
-    port = "8180"
+    name = "as-server-01"
+    host = "127.0.0.1"
+    port = "8080"
     # username = "myuser"
     # password = "mypassword"
 
@@ -63,20 +79,31 @@ func (j *Jolokia) SampleConfig() string {
   ## This collect all heap memory usage metrics.
   [[inputs.jolokia.metrics]]
     name = "heap_memory_usage"
-    jmx  = "/java.lang:type=Memory/HeapMemoryUsage"
+    mbean  = "java.lang:type=Memory"
+    attribute = "HeapMemoryUsage"
+
+  ## This collect thread counts metrics.
+  [[inputs.jolokia.metrics]]
+    name = "thread_count"
+    mbean  = "java.lang:type=Threading"
+    attribute = "TotalStartedThreadCount,ThreadCount,DaemonThreadCount,PeakThreadCount"
+
+  ## This collect number of class loaded/unloaded counts metrics.
+  [[inputs.jolokia.metrics]]
+    name = "class_count"
+    mbean  = "java.lang:type=ClassLoading"
+    attribute = "LoadedClassCount,UnloadedClassCount,TotalLoadedClassCount"
 `
+
+func (j *Jolokia) SampleConfig() string {
+	return sampleConfig
 }
 
 func (j *Jolokia) Description() string {
 	return "Read JMX metrics through Jolokia"
 }
 
-func (j *Jolokia) getAttr(requestUrl *url.URL) (map[string]interface{}, error) {
-	// Create + send request
-	req, err := http.NewRequest("GET", requestUrl.String(), nil)
-	if err != nil {
-		return nil, err
-	}
+func (j *Jolokia) doRequest(req *http.Request) (map[string]interface{}, error) {
 
 	resp, err := j.jClient.MakeRequest(req)
 	if err != nil {
@@ -87,7 +114,7 @@ func (j *Jolokia) getAttr(requestUrl *url.URL) (map[string]interface{}, error) {
 	// Process response
 	if resp.StatusCode != http.StatusOK {
 		err = fmt.Errorf("Response from url \"%s\" has status code %d (%s), expected %d (%s)",
-			requestUrl,
+			req.RequestURI,
 			resp.StatusCode,
 			http.StatusText(resp.StatusCode),
 			http.StatusOK,
@@ -107,51 +134,133 @@ func (j *Jolokia) getAttr(requestUrl *url.URL) (map[string]interface{}, error) {
 		return nil, errors.New("Error decoding JSON response")
 	}
 
+	if status, ok := jsonOut["status"]; ok {
+		if status != float64(200) {
+			return nil, fmt.Errorf("Not expected status value in response body: %3.f",
+				status)
+		}
+	} else {
+		return nil, fmt.Errorf("Missing status in response body")
+	}
+
 	return jsonOut, nil
 }
 
+func (j *Jolokia) prepareRequest(server Server, metric Metric) (*http.Request, error) {
+	var jolokiaUrl *url.URL
+	context := j.Context // Usually "/jolokia"
+
+	// Create bodyContent
+	bodyContent := map[string]interface{}{
+		"type":  "read",
+		"mbean": metric.Mbean,
+	}
+
+	if metric.Attribute != "" {
+		bodyContent["attribute"] = metric.Attribute
+		if metric.Path != "" {
+			bodyContent["path"] = metric.Path
+		}
+	}
+
+	// Add target, only in proxy mode
+	if j.Mode == "proxy" {
+		serviceUrl := fmt.Sprintf("service:jmx:rmi:///jndi/rmi://%s:%s/jmxrmi",
+			server.Host, server.Port)
+
+		target := map[string]string{
+			"url": serviceUrl,
+		}
+
+		if server.Username != "" {
+			target["user"] = server.Username
+		}
+
+		if server.Password != "" {
+			target["password"] = server.Password
+		}
+
+		bodyContent["target"] = target
+
+		proxy := j.Proxy
+
+		// Prepare ProxyURL
+		proxyUrl, err := url.Parse("http://" + proxy.Host + ":" + proxy.Port + context)
+		if err != nil {
+			return nil, err
+		}
+		if proxy.Username != "" || proxy.Password != "" {
+			proxyUrl.User = url.UserPassword(proxy.Username, proxy.Password)
+		}
+
+		jolokiaUrl = proxyUrl
+
+	} else {
+		serverUrl, err := url.Parse("http://" + server.Host + ":" + server.Port + context)
+		if err != nil {
+			return nil, err
+		}
+		if server.Username != "" || server.Password != "" {
+			serverUrl.User = url.UserPassword(server.Username, server.Password)
+		}
+
+		jolokiaUrl = serverUrl
+	}
+
+	requestBody, err := json.Marshal(bodyContent)
+
+	req, err := http.NewRequest("POST", jolokiaUrl.String(), bytes.NewBuffer(requestBody))
+
+	if err != nil {
+		return nil, err
+	}
+
+	req.Header.Add("Content-type", "application/json")
+
+	return req, nil
+}
+
 func (j *Jolokia) Gather(acc telegraf.Accumulator) error {
-	context := j.Context //"/jolokia/read"
 	servers := j.Servers
 	metrics := j.Metrics
 	tags := make(map[string]string)
 
 	for _, server := range servers {
-		tags["server"] = server.Name
-		tags["port"] = server.Port
-		tags["host"] = server.Host
+		tags["jolokia_name"] = server.Name
+		tags["jolokia_port"] = server.Port
+		tags["jolokia_host"] = server.Host
 		fields := make(map[string]interface{})
+
 		for _, metric := range metrics {
-
 			measurement := metric.Name
-			jmxPath := metric.Jmx
 
-			// Prepare URL
-			requestUrl, err := url.Parse("http://" + server.Host + ":" +
-				server.Port + context + jmxPath)
+			req, err := j.prepareRequest(server, metric)
 			if err != nil {
 				return err
 			}
-			if server.Username != "" || server.Password != "" {
-				requestUrl.User = url.UserPassword(server.Username, server.Password)
-			}
 
-			out, _ := j.getAttr(requestUrl)
+			out, err := j.doRequest(req)
 
-			if values, ok := out["value"]; ok {
-				switch t := values.(type) {
-				case map[string]interface{}:
-					for k, v := range t {
-						fields[measurement+"_"+k] = v
-					}
-				case interface{}:
-					fields[measurement] = t
-				}
+			if err != nil {
+				fmt.Printf("Error handling response: %s\n", err)
 			} else {
-				fmt.Printf("Missing key 'value' in '%s' output response\n",
-					requestUrl.String())
+
+				if values, ok := out["value"]; ok {
+					switch t := values.(type) {
+					case map[string]interface{}:
+						for k, v := range t {
+							fields[measurement+"_"+k] = v
+						}
+					case interface{}:
+						fields[measurement] = t
+					}
+				} else {
+					fmt.Printf("Missing key 'value' in output response\n")
+				}
+
 			}
 		}
+
 		acc.AddFields("jolokia", fields, tags)
 	}
 
@@ -160,6 +269,11 @@ func (j *Jolokia) Gather(acc telegraf.Accumulator) error {
 
 func init() {
 	inputs.Add("jolokia", func() telegraf.Input {
-		return &Jolokia{jClient: &JolokiaClientImpl{client: &http.Client{}}}
+		tr := &http.Transport{ResponseHeaderTimeout: time.Duration(3 * time.Second)}
+		client := &http.Client{
+			Transport: tr,
+			Timeout:   time.Duration(4 * time.Second),
+		}
+		return &Jolokia{jClient: &JolokiaClientImpl{client: client}}
 	})
 }

plugins/inputs/jolokia/jolokia_test.go

@@ -47,8 +47,10 @@ const invalidJSON = "I don't think this is JSON"
 const empty = ""
 
 var Servers = []Server{Server{Name: "as1", Host: "127.0.0.1", Port: "8080"}}
-var HeapMetric = Metric{Name: "heap_memory_usage", Jmx: "/java.lang:type=Memory/HeapMemoryUsage"}
-var UsedHeapMetric = Metric{Name: "heap_memory_usage", Jmx: "/java.lang:type=Memory/HeapMemoryUsage"}
+var HeapMetric = Metric{Name: "heap_memory_usage",
+	Mbean: "java.lang:type=Memory", Attribute: "HeapMemoryUsage"}
+var UsedHeapMetric = Metric{Name: "heap_memory_usage",
+	Mbean: "java.lang:type=Memory", Attribute: "HeapMemoryUsage"}
 
 type jolokiaClientStub struct {
 	responseBody string
@@ -94,9 +96,9 @@ func TestHttpJsonMultiValue(t *testing.T) {
 		"heap_memory_usage_used":      203288528.0,
 	}
 	tags := map[string]string{
-		"host":   "127.0.0.1",
-		"port":   "8080",
-		"server": "as1",
+		"jolokia_host": "127.0.0.1",
+		"jolokia_port": "8080",
+		"jolokia_name": "as1",
 	}
 	acc.AssertContainsTaggedFields(t, "jolokia", fields, tags)
 }
@@ -114,3 +116,17 @@ func TestHttpJsonOn404(t *testing.T) {
 	assert.Nil(t, err)
 	assert.Equal(t, 0, len(acc.Metrics))
 }
+
+// Test that the proper values are ignored or collected
+func TestHttpInvalidJson(t *testing.T) {
+
+	jolokia := genJolokiaClientStub(invalidJSON, 200, Servers,
+		[]Metric{UsedHeapMetric})
+
+	var acc testutil.Accumulator
+	acc.SetDebug(true)
+	err := jolokia.Gather(&acc)
+
+	assert.Nil(t, err)
+	assert.Equal(t, 0, len(acc.Metrics))
+}