reviving code from #1447 to a new branch

If unset, query_version is still treated as version 1 for compatibility.

.circleci/config.yml

@@ -0,0 +1,49 @@
+---
+defaults: &defaults
+  docker:
+    - image: 'circleci/golang:1.9.2'
+  working_directory: '/go/src/github.com/influxdata/telegraf'
+version: 2
+jobs:
+  build:
+    <<: *defaults
+    steps:
+      - checkout
+      - run: 'make ci-test'
+  release:
+    <<: *defaults
+    steps:
+      - checkout
+      - run: './scripts/release.sh'
+      - store_artifacts:
+          path: './artifacts'
+          destination: '.'
+  nightly:
+    <<: *defaults
+    steps:
+      - checkout
+      - run: './scripts/release.sh'
+      - store_artifacts:
+          path: './artifacts'
+          destination: '.'
+workflows:
+  version: 2
+  build_and_release:
+    jobs:
+      - 'build'
+      - 'release':
+          requires:
+            - 'build'
+  nightly:
+    jobs:
+      - 'build'
+      - 'nightly':
+          requires:
+            - 'build'
+    triggers:
+      - schedule:
+          cron: "0 18 * * *"
+          filters:
+            branches:
+              only:
+                - master

.gitignore

@@ -1,7 +1,5 @@
 build
-tivan
-.vagrant
 /telegraf
-.idea
+/telegraf.gz
 *~
 *#

CHANGELOG.md

@@ -1,15 +1,111 @@
-## v1.5 [unreleased]
+## v1.6 [unreleased]
+
+### Release Notes
+
+- The `mysql` input plugin has been updated to convert values to the
+  correct data type.  This may cause a `field type error` when inserting into
+  InfluxDB due the change of types.  It is recommended to drop the `mysql`,
+  `mysql_variables`, and `mysql_innodb`:
+  ```
+  DROP MEASUREMENT mysql
+  DROP MEASUREMENT mysql_variables
+  DROP MEASUREMENT mysql_innodb
+  ```
+
+- The `postgresql` plugins now defaults to using a persistent connection to the database.
+  In environments where TCP connections are terminated the `max_lifetime`
+  setting should be set less than the collection `interval` to prevent errors.
+
+- The `sqlserver` input plugin has a new query and data model that can be enabled
+  by setting `query_version = 2`.  It is encouraged to migrate to the new
+  model when possible as the old version is deprecated and will be removed in
+  a future version.
+
+- An option has been added to the `openldap` input plugin that reverses metric
+  name to improve grouping.  This change is enabled when `reverse_metric_names = true`
+  is set.  It is encouraged to enable this option when possible as the old
+  ordering is deprecated.
+
+
+### New Plugins
+
+- [ipset](./plugins/inputs/ipset/README.md) - Thanks to @sajoupa
+- [nats](./plugins/inputs/nats/README.md) - Thanks to @mjs & @levex
+
+### Features
+
+- [#3551](https://github.com/influxdata/telegraf/pull/3551): Add health status mapping from string to int in elasticsearch input.
+- [#3580](https://github.com/influxdata/telegraf/pull/3580): Add control over which stats to gather in basicstats aggregator.
+- [#3596](https://github.com/influxdata/telegraf/pull/3596): Add messages_delivered_get to rabbitmq input.
+- [#3632](https://github.com/influxdata/telegraf/pull/3632): Add wired field to mem input.
+- [#3619](https://github.com/influxdata/telegraf/pull/3619): Add support for gathering exchange metrics to the rabbitmq input.
+- [#3565](https://github.com/influxdata/telegraf/pull/3565): Add support for additional metrics on Linux in zfs input.
+- [#3524](https://github.com/influxdata/telegraf/pull/3524): Add available_entropy field to kernel input plugin.
+- [#3643](https://github.com/influxdata/telegraf/pull/3643): Add user privilege level setting to IPMI sensors.
+- [#2701](https://github.com/influxdata/telegraf/pull/2701): Use persistent connection to postgresql database.
+- [#2846](https://github.com/influxdata/telegraf/pull/2846): Add support for dropwizard input format.
+- [#3666](https://github.com/influxdata/telegraf/pull/3666): Add container health metrics to docker input.
+- [#3687](https://github.com/influxdata/telegraf/pull/3687): Add support for using globs in devices list of diskio input plugin.
+- [#2754](https://github.com/influxdata/telegraf/pull/2754): Allow running as console application on Windows.
+- [#3703](https://github.com/influxdata/telegraf/pull/3703): Add listener counts and node running status to rabbitmq input.
+- [#3674](https://github.com/influxdata/telegraf/pull/3674): Add NATS Monitoring Input Plugin.
+- [#3702](https://github.com/influxdata/telegraf/pull/3702): Add ability to select which queues will be gathered in rabbitmq input.
+- [#3726](https://github.com/influxdata/telegraf/pull/3726): Add support for setting bsd source address to the ping input.
+- [#3346](https://github.com/influxdata/telegraf/pull/3346): Add Ipset input plugin.
+- [#3719](https://github.com/influxdata/telegraf/pull/3719): Add TLS and HTTP basic auth to prometheus_client output.
+- [#3618](https://github.com/influxdata/telegraf/pull/3618): Add new sqlserver output data model.
+- [#3559](https://github.com/influxdata/telegraf/pull/3559): Add native Go method for finding pids to procstat.
+- [#3722](https://github.com/influxdata/telegraf/pull/3722): Add additional metrics and reverse metric names option to openldap.
+- [#3769](https://github.com/influxdata/telegraf/pull/3769): Add TLS support to the mesos input plugin.
+
+### Bugfixes
+
+- [#1896](https://github.com/influxdata/telegraf/issues/1896): Fix various mysql data type conversions.
+
+## v1.5.3 [unreleased]
+
+### Bugfixes
+
+- [#3729](https://github.com/influxdata/telegraf/issues/3729): Set path to / if HOST_MOUNT_PREFIX matches full path.
+- [#3739](https://github.com/influxdata/telegraf/issues/3739): Remove userinfo from url tag in prometheus input.
+
+## v1.5.2 [2018-01-30]
+
+### Bugfixes
+
+- [#3684](https://github.com/influxdata/telegraf/pull/3684): Ignore empty lines in Graphite plaintext.
+- [#3604](https://github.com/influxdata/telegraf/issues/3604): Fix index out of bounds error in solr input plugin.
+- [#3680](https://github.com/influxdata/telegraf/pull/3680): Reconnect before sending graphite metrics if disconnected.
+- [#3693](https://github.com/influxdata/telegraf/pull/3693): Align aggregator period with internal ticker to avoid skipping metrics.
+- [#3629](https://github.com/influxdata/telegraf/issues/3629): Fix a potential deadlock when using aggregators.
+- [#3697](https://github.com/influxdata/telegraf/issues/3697): Limit wait time for writes in mqtt output.
+- [#3698](https://github.com/influxdata/telegraf/issues/3698): Revert change in graphite output where dot in field key was replaced by underscore.
+- [#3710](https://github.com/influxdata/telegraf/issues/3710): Add timeout to wavefront output write.
+- [#3725](https://github.com/influxdata/telegraf/issues/3725): Exclude master_replid fields from redis input.
+
+## v1.5.1 [2018-01-10]
+
+### Bugfixes
+
+- [#3624](https://github.com/influxdata/telegraf/pull/3624): Fix name error in jolokia2_agent sample config.
+- [#3625](https://github.com/influxdata/telegraf/pull/3625): Fix DC/OS login expiration time.
+- [#3593](https://github.com/influxdata/telegraf/pull/3593): Set Content-Type charset in influxdb output and allow it be overridden.
+- [#3594](https://github.com/influxdata/telegraf/pull/3594): Document permissions setup for postfix input.
+- [#3633](https://github.com/influxdata/telegraf/pull/3633): Fix deliver_get field in rabbitmq input.
+- [#3607](https://github.com/influxdata/telegraf/issues/3607): Escape environment variables during config toml parsing.
+
+## v1.5 [2017-12-14]
 
 ### New Plugins
 - [basicstats](./plugins/aggregators/basicstats/README.md) - Thanks to @toni-moreno
 - [bond](./plugins/inputs/bond/README.md) - Thanks to @ildarsv
-- [cratedb](./plugins/outputs/wavefront/README.md) - Thanks to @felixge
+- [cratedb](./plugins/outputs/cratedb/README.md) - Thanks to @felixge
 - [dcos](./plugins/inputs/dcos/README.md) - Thanks to @influxdata
 - [jolokia2](./plugins/inputs/jolokia2/README.md) - Thanks to @dylanmei
 - [nginx_plus](./plugins/inputs/nginx_plus/README.md) - Thanks to @mplonka & @poblahblahblah
 - [opensmtpd](./plugins/inputs/opensmtpd/README.md) - Thanks to @aromeyer
 - [particle](./plugins/inputs/webhooks/particle/README.md) - Thanks to @davidgs
-- [pf](./plugins/inputs/pf/README.md) Thanks to @nferch
+- [pf](./plugins/inputs/pf/README.md) - Thanks to @nferch
 - [postfix](./plugins/inputs/postfix/README.md) - Thanks to @phemmer
 - [smart](./plugins/inputs/smart/README.md) - Thanks to @rickard-von-essen
 - [solr](./plugins/inputs/solr/README.md) - Thanks to @ljagiello
@@ -28,6 +124,11 @@
   plugin is deprecated and will be removed in a future release.  Users of this
   plugin are encouraged to update to the new `jolokia2` plugin.
 
+- In the `postgresql` and `postgresql_extensible` plugins, the type of the oid
+  data type has changed from string to integer.  It is recommended to drop
+  affected fields until a new shard is started. For details on how to
+  workaround this issue please see [#3622](https://github.com/influxdata/telegraf/issues/3622).
+
 ### Features
 
 - [#3170](https://github.com/influxdata/telegraf/pull/3170): Add support for sharding based on metric name.
@@ -78,6 +179,7 @@
 - [#3140](https://github.com/influxdata/telegraf/pull/3140): Add support for glob patterns in net input plugin.
 - [#3405](https://github.com/influxdata/telegraf/pull/3405): Add input plugin for OpenBSD/FreeBSD pf.
 - [#3528](https://github.com/influxdata/telegraf/pull/3528): Add option to amqp output to publish persistent messages.
+- [#3530](https://github.com/influxdata/telegraf/pull/3530): Support I (idle) process state on procfs+Linux.
 
 ### Bugfixes
 
@@ -92,6 +194,9 @@
 - [#3263](https://github.com/influxdata/telegraf/issues/3263): Fix snmp-tools output parsing with Windows EOLs.
 - [#3447](https://github.com/influxdata/telegraf/issues/3447): Add shadow-utils dependency to rpm package.
 - [#3448](https://github.com/influxdata/telegraf/issues/3448): Use deb-systemd-invoke to restart service.
+- [#3553](https://github.com/influxdata/telegraf/issues/3553): Fix kafka_consumer outside range of offsets error.
+- [#3568](https://github.com/influxdata/telegraf/issues/3568): Fix separation of multiple prometheus_client outputs.
+- [#3577](https://github.com/influxdata/telegraf/issues/3577): Don't add system input uptime_format as a counter.
 
 ## v1.4.5 [2017-12-01]
 

CONTRIBUTING.md

@@ -79,7 +79,10 @@ func (s *Simple) Description() string {
 }
 
 func (s *Simple) SampleConfig() string {
-    return "ok = true # indicate if everything is fine"
+    return `
+  ## Indicate if everything is fine
+  ok = true
+`
 }
 
 func (s *Simple) Gather(acc telegraf.Accumulator) error {
@@ -207,7 +210,9 @@ func (s *Simple) Description() string {
 }
 
 func (s *Simple) SampleConfig() string {
-    return "url = localhost"
+    return `
+  ok = true
+`
 }
 
 func (s *Simple) Connect() error {

Godeps

@@ -4,7 +4,7 @@ github.com/amir/raidman c74861fe6a7bb8ede0a010ce4485bdbb4fc4c985
 github.com/apache/thrift 4aaa92ece8503a6da9bc6701604f69acf2b99d07
 github.com/aws/aws-sdk-go c861d27d0304a79f727e9a8a4e2ac1e74602fdc0
 github.com/beorn7/perks 4c0e84591b9aa9e6dcfdf3e020114cd81f89d5f9
-github.com/bsm/sarama-cluster ccdc0803695fbce22f1706d04ded46cd518fd832
+github.com/bsm/sarama-cluster abf039439f66c1ce78017f560b490612552f6472
 github.com/cenkalti/backoff b02f2bbce11d7ea6b97f282ef1771b0fe2f65ef3
 github.com/couchbase/go-couchbase bfe555a140d53dc1adf390f1a1d4b0fd4ceadb28
 github.com/couchbase/gomemcached 4a25d2f4e1dea9ea7dd76dfd943407abf9b07d29
@@ -16,7 +16,7 @@ github.com/docker/go-connections 990a1a1a70b0da4c4cb70e117971a4f0babfbf1a
 github.com/eapache/go-resiliency b86b1ec0dd4209a588dc1285cdd471e73525c0b3
 github.com/eapache/go-xerial-snappy bb955e01b9346ac19dc29eb16586c90ded99a98c
 github.com/eapache/queue 44cc805cf13205b55f69e14bcb69867d1ae92f98
-github.com/eclipse/paho.mqtt.golang d4f545eb108a2d19f9b1a735689dbfb719bc21fb
+github.com/eclipse/paho.mqtt.golang aff15770515e3c57fc6109da73d42b0d46f7f483
 github.com/go-logfmt/logfmt 390ab7935ee28ec6b286364bba9b4dd6410cb3d5
 github.com/go-sql-driver/mysql 2e00b5cd70399450106cec6431c2e2ce3cae5034
 github.com/gobwas/glob bea32b9cd2d6f55753d94a28e959b13f0244797a
@@ -27,6 +27,7 @@ github.com/golang/snappy 7db9049039a047d955fe8c19b83c8ff5abd765c7
 github.com/go-ole/go-ole be49f7c07711fcb603cff39e1de7c67926dc0ba7
 github.com/google/go-cmp f94e52cad91c65a63acc1e75d4be223ea22e99bc
 github.com/gorilla/mux 392c28fe23e1c45ddba891b0320b3b5df220beea
+github.com/go-redis/redis 73b70592cdaa9e6abdfcfbf97b4a90d80728c836
 github.com/go-sql-driver/mysql 2e00b5cd70399450106cec6431c2e2ce3cae5034
 github.com/hailocab/go-hostpool e80d13ce29ede4452c43dea11e79b9bc8a15b478
 github.com/hashicorp/consul 63d2fc68239b996096a1c55a0d4b400ea4c2583f
@@ -44,6 +45,7 @@ github.com/miekg/dns 99f84ae56e75126dd77e5de4fae2ea034a468ca1
 github.com/mitchellh/mapstructure d0303fe809921458f417bcf828397a65db30a7e4
 github.com/multiplay/go-ts3 07477f49b8dfa3ada231afc7b7b17617d42afe8e
 github.com/naoina/go-stringutil 6b638e95a32d0c1131db0e7fe83775cbea4a0d0b
+github.com/nats-io/gnatsd 393bbb7c031433e68707c8810fda0bfcfbe6ab9b
 github.com/nats-io/go-nats ea9585611a4ab58a205b9b125ebd74c389a6b898
 github.com/nats-io/nats ea9585611a4ab58a205b9b125ebd74c389a6b898
 github.com/nats-io/nuid 289cccf02c178dc782430d534e3c1f5b72af807f
@@ -65,13 +67,15 @@ github.com/samuel/go-zookeeper 1d7be4effb13d2d908342d349d71a284a7542693
 github.com/satori/go.uuid 5bf94b69c6b68ee1b541973bb8e1144db23a194b
 github.com/shirou/gopsutil 384a55110aa5ae052eb93ea94940548c1e305a99
 github.com/shirou/w32 3c9377fc6748f222729a8270fe2775d149a249ad
-github.com/Shopify/sarama c01858abb625b73a3af51d0798e4ad42c8147093
+github.com/Shopify/sarama 3b1b38866a79f06deddf0487d5c27ba0697ccd65
 github.com/Sirupsen/logrus 61e43dc76f7ee59a82bdf3d71033dc12bea4c77d
 github.com/soniah/gosnmp 5ad50dc75ab389f8a1c9f8a67d3a1cd85f67ed15
 github.com/StackExchange/wmi f3e2bae1e0cb5aef83e319133eabfee30013a4a5
 github.com/streadway/amqp 63795daa9a446c920826655f26ba31c81c860fd6
 github.com/stretchr/objx 1a9d0bb9f541897e62256577b352fdbc1fb4fd94
 github.com/stretchr/testify 4d4bfba8f1d1027c4fdbe371823030df51419987
+github.com/tidwall/gjson 0623bd8fbdbf97cc62b98d15108832851a658e59
+github.com/tidwall/match 173748da739a410c5b0b813b956f89ff94730b4c
 github.com/vjeantet/grok d73e972b60935c7fec0b4ffbc904ed39ecaf7efe
 github.com/wvanbergen/kafka bc265fedb9ff5b5c5d3c0fdcef4a819b3523d3ee
 github.com/wvanbergen/kazoo-go 968957352185472eacb69215fa3dbfcfdbac1096

Makefile

@@ -2,6 +2,9 @@ PREFIX := /usr/local
 VERSION := $(shell git describe --exact-match --tags 2>/dev/null)
 BRANCH := $(shell git rev-parse --abbrev-ref HEAD)
 COMMIT := $(shell git rev-parse --short HEAD)
+GOFILES ?= $(shell git ls-files '*.go')
+GOFMT ?= $(shell gofmt -l $(GOFILES))
+
 ifdef GOBIN
 PATH := $(GOBIN):$(PATH)
 else
@@ -16,10 +19,18 @@ ifdef VERSION
 endif
 
 all:
+	$(MAKE) fmtcheck
 	$(MAKE) deps
 	$(MAKE) telegraf
 
+ci-test:
+	$(MAKE) deps
+	$(MAKE) fmtcheck
+	$(MAKE) vet
+	$(MAKE) test
+
 deps:
+	go get -u github.com/golang/lint/golint
 	go get github.com/sparrc/gdm
 	gdm restore
 
@@ -36,98 +47,53 @@ install: telegraf
 test:
 	go test -short ./...
 
+fmt:
+	@gofmt -w $(GOFILES)
+
+fmtcheck:
+	@echo '[INFO] running gofmt to identify incorrectly formatted code...'
+	@if [ ! -z $(GOFMT) ]; then \
+		echo "[ERROR] gofmt has found errors in the following files:"  ; \
+		echo "$(GOFMT)" ; \
+		echo "" ;\
+		echo "Run make fmt to fix them." ; \
+		exit 1 ;\
+	fi
+	@echo '[INFO] done.'
+
+lint:
+	golint ./...
+
 test-windows:
 	go test ./plugins/inputs/ping/...
 	go test ./plugins/inputs/win_perf_counters/...
 	go test ./plugins/inputs/win_services/...
+	go test ./plugins/inputs/procstat/...
 
-lint:
-	go vet ./...
+# vet runs the Go source code static analysis tool `vet` to find
+# any common errors.
+vet:
+	@echo 'go vet $$(go list ./...)'
+	@go vet $$(go list ./...) ; if [ $$? -eq 1 ]; then \
+		echo ""; \
+		echo "go vet has found suspicious constructs. Please remediate any reported errors"; \
+		echo "to fix them before submitting code for review."; \
+		exit 1; \
+	fi
 
-test-all: lint
+test-all: vet
 	go test ./...
 
 package:
 	./scripts/build.py --package --platform=all --arch=all
 
 clean:
-	-rm -f telegraf
-	-rm -f telegraf.exe
+	rm -f telegraf
+	rm -f telegraf.exe
 
 docker-image:
 	./scripts/build.py --package --platform=linux --arch=amd64
 	cp build/telegraf*$(COMMIT)*.deb .
 	docker build -f scripts/dev.docker --build-arg "package=telegraf*$(COMMIT)*.deb" -t "telegraf-dev:$(COMMIT)" .
 
-# Run all docker containers necessary for integration tests
-docker-run:
-	docker run --name aerospike -p "3000:3000" -d aerospike/aerospike-server:3.9.0
-	docker run --name zookeeper -p "2181:2181" -d wurstmeister/zookeeper
-	docker run --name kafka \
-		--link zookeeper:zookeeper \
-		-e KAFKA_ADVERTISED_HOST_NAME=localhost \
-		-e KAFKA_ADVERTISED_PORT=9092 \
-		-e KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 \
-		-e KAFKA_CREATE_TOPICS="test:1:1" \
-		-p "9092:9092" \
-		-d wurstmeister/kafka
-	docker run --name elasticsearch -p "9200:9200" -p "9300:9300" -d elasticsearch:5
-	docker run --name mysql -p "3306:3306" -e MYSQL_ALLOW_EMPTY_PASSWORD=yes -d mysql
-	docker run --name memcached -p "11211:11211" -d memcached
-	docker run --name postgres -p "5432:5432" -d postgres
-	docker run --name rabbitmq -p "15672:15672" -p "5672:5672" -d rabbitmq:3-management
-	docker run --name redis -p "6379:6379" -d redis
-	docker run --name nsq -p "4150:4150" -d nsqio/nsq /nsqd
-	docker run --name mqtt -p "1883:1883" -d ncarlier/mqtt
-	docker run --name riemann -p "5555:5555" -d stealthly/docker-riemann
-	docker run --name nats -p "4222:4222" -d nats
-	docker run --name openldap \
-		-e SLAPD_CONFIG_ROOTDN="cn=manager,cn=config" \
-		-e SLAPD_CONFIG_ROOTPW="secret" \
-		-p "389:389" -p "636:636" \
-		-d cobaugh/openldap-alpine
-	docker run --name cratedb \
-		-p "6543:5432" \
-		-d crate crate \
-		-Cnetwork.host=0.0.0.0 \
-		-Ctransport.host=localhost \
-		-Clicense.enterprise=false
-
-# Run docker containers necessary for integration tests; skipping services provided
-# by CircleCI
-docker-run-circle:
-	docker run --name aerospike -p "3000:3000" -d aerospike/aerospike-server:3.9.0
-	docker run --name zookeeper -p "2181:2181" -d wurstmeister/zookeeper
-	docker run --name kafka \
-		--link zookeeper:zookeeper \
-		-e KAFKA_ADVERTISED_HOST_NAME=localhost \
-		-e KAFKA_ADVERTISED_PORT=9092 \
-		-e KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 \
-		-e KAFKA_CREATE_TOPICS="test:1:1" \
-		-p "9092:9092" \
-		-d wurstmeister/kafka
-	docker run --name elasticsearch -p "9200:9200" -p "9300:9300" -d elasticsearch:5
-	docker run --name nsq -p "4150:4150" -d nsqio/nsq /nsqd
-	docker run --name mqtt -p "1883:1883" -d ncarlier/mqtt
-	docker run --name riemann -p "5555:5555" -d stealthly/docker-riemann
-	docker run --name nats -p "4222:4222" -d nats
-	docker run --name openldap \
-		-e SLAPD_CONFIG_ROOTDN="cn=manager,cn=config" \
-		-e SLAPD_CONFIG_ROOTPW="secret" \
-		-p "389:389" -p "636:636" \
-		-d cobaugh/openldap-alpine
-	docker run --name cratedb \
-		-p "6543:5432" \
-		-d crate crate \
-		-Cnetwork.host=0.0.0.0 \
-		-Ctransport.host=localhost \
-		-Clicense.enterprise=false
-
-docker-kill:
-	-docker kill aerospike elasticsearch kafka memcached mqtt mysql nats nsq \
-		openldap postgres rabbitmq redis riemann zookeeper cratedb
-	-docker rm aerospike elasticsearch kafka memcached mqtt mysql nats nsq \
-		openldap postgres rabbitmq redis riemann zookeeper cratedb
-
-.PHONY: deps telegraf telegraf.exe install test test-windows lint test-all \
-	package clean docker-run docker-run-circle docker-kill docker-image
+.PHONY: deps telegraf install test test-windows lint vet test-all package clean docker-image fmtcheck

README.md

@@ -159,6 +159,7 @@ configuration options.
 * [interrupts](./plugins/inputs/interrupts)
 * [ipmi_sensor](./plugins/inputs/ipmi_sensor)
 * [iptables](./plugins/inputs/iptables)
+* [ipset](./plugins/inputs/ipset)
 * [jolokia](./plugins/inputs/jolokia) (deprecated, use [jolokia2](./plugins/inputs/jolokia2))
 * [jolokia2](./plugins/inputs/jolokia2)
 * [kapacitor](./plugins/inputs/kapacitor)
@@ -171,6 +172,7 @@ configuration options.
 * [minecraft](./plugins/inputs/minecraft)
 * [mongodb](./plugins/inputs/mongodb)
 * [mysql](./plugins/inputs/mysql)
+* [nats](./plugins/inputs/nats)
 * [net_response](./plugins/inputs/net_response)
 * [nginx](./plugins/inputs/nginx)
 * [nginx_plus](./plugins/inputs/nginx_plus)
@@ -256,6 +258,7 @@ formats may be used with input plugins supporting the `data_format` option:
 * [Value](./docs/DATA_FORMATS_INPUT.md#value)
 * [Nagios](./docs/DATA_FORMATS_INPUT.md#nagios)
 * [Collectd](./docs/DATA_FORMATS_INPUT.md#collectd)
+* [Dropwizard](./docs/DATA_FORMATS_INPUT.md#dropwizard)
 
 ## Processor Plugins
 

agent/accumulator.go

@@ -26,7 +26,7 @@ type MetricMaker interface {
 func NewAccumulator(
 	maker MetricMaker,
 	metrics chan telegraf.Metric,
-) *accumulator {
+) telegraf.Accumulator {
 	acc := accumulator{
 		maker:     maker,
 		metrics:   metrics,

agent/accumulator_test.go

@@ -115,15 +115,15 @@ func TestAddNoIntervalWithPrecision(t *testing.T) {
 		map[string]interface{}{"value": float64(101)},
 		map[string]string{"acc": "test"}, now)
 
-	testm := <-a.metrics
+	testm := <-metrics
 	actual := testm.String()
 	assert.Contains(t, actual, "acctest value=101")
 
-	testm = <-a.metrics
+	testm = <-metrics
 	actual = testm.String()
 	assert.Contains(t, actual, "acctest,acc=test value=101")
 
-	testm = <-a.metrics
+	testm = <-metrics
 	actual = testm.String()
 	assert.Equal(t,
 		fmt.Sprintf("acctest,acc=test value=101 %d\n", int64(1139572800000000000)),
@@ -147,15 +147,15 @@ func TestAddDisablePrecision(t *testing.T) {
 		map[string]interface{}{"value": float64(101)},
 		map[string]string{"acc": "test"}, now)
 
-	testm := <-a.metrics
+	testm := <-metrics
 	actual := testm.String()
 	assert.Contains(t, actual, "acctest value=101")
 
-	testm = <-a.metrics
+	testm = <-metrics
 	actual = testm.String()
 	assert.Contains(t, actual, "acctest,acc=test value=101")
 
-	testm = <-a.metrics
+	testm = <-metrics
 	actual = testm.String()
 	assert.Equal(t,
 		fmt.Sprintf("acctest,acc=test value=101 %d\n", int64(1139572800082912748)),
@@ -179,15 +179,15 @@ func TestAddNoPrecisionWithInterval(t *testing.T) {
 		map[string]interface{}{"value": float64(101)},
 		map[string]string{"acc": "test"}, now)
 
-	testm := <-a.metrics
+	testm := <-metrics
 	actual := testm.String()
 	assert.Contains(t, actual, "acctest value=101")
 
-	testm = <-a.metrics
+	testm = <-metrics
 	actual = testm.String()
 	assert.Contains(t, actual, "acctest,acc=test value=101")
 
-	testm = <-a.metrics
+	testm = <-metrics
 	actual = testm.String()
 	assert.Equal(t,
 		fmt.Sprintf("acctest,acc=test value=101 %d\n", int64(1139572800000000000)),
@@ -204,7 +204,7 @@ func TestDifferentPrecisions(t *testing.T) {
 	a.AddFields("acctest",
 		map[string]interface{}{"value": float64(101)},
 		map[string]string{"acc": "test"}, now)
-	testm := <-a.metrics
+	testm := <-metrics
 	actual := testm.String()
 	assert.Equal(t,
 		fmt.Sprintf("acctest,acc=test value=101 %d\n", int64(1139572800000000000)),
@@ -214,7 +214,7 @@ func TestDifferentPrecisions(t *testing.T) {
 	a.AddFields("acctest",
 		map[string]interface{}{"value": float64(101)},
 		map[string]string{"acc": "test"}, now)
-	testm = <-a.metrics
+	testm = <-metrics
 	actual = testm.String()
 	assert.Equal(t,
 		fmt.Sprintf("acctest,acc=test value=101 %d\n", int64(1139572800083000000)),
@@ -224,7 +224,7 @@ func TestDifferentPrecisions(t *testing.T) {
 	a.AddFields("acctest",
 		map[string]interface{}{"value": float64(101)},
 		map[string]string{"acc": "test"}, now)
-	testm = <-a.metrics
+	testm = <-metrics
 	actual = testm.String()
 	assert.Equal(t,
 		fmt.Sprintf("acctest,acc=test value=101 %d\n", int64(1139572800082913000)),
@@ -234,7 +234,7 @@ func TestDifferentPrecisions(t *testing.T) {
 	a.AddFields("acctest",
 		map[string]interface{}{"value": float64(101)},
 		map[string]string{"acc": "test"}, now)
-	testm = <-a.metrics
+	testm = <-metrics
 	actual = testm.String()
 	assert.Equal(t,
 		fmt.Sprintf("acctest,acc=test value=101 %d\n", int64(1139572800082912748)),

agent/agent.go

@@ -143,7 +143,7 @@ func (a *Agent) gatherer(
 func gatherWithTimeout(
 	shutdown chan struct{},
 	input *models.RunningInput,
-	acc *accumulator,
+	acc telegraf.Accumulator,
 	timeout time.Duration,
 ) {
 	ticker := time.NewTicker(timeout)
@@ -308,7 +308,13 @@ func (a *Agent) flusher(shutdown chan struct{}, metricC chan telegraf.Metric, ag
 					metrics = processor.Apply(metrics...)
 				}
 				for _, m := range metrics {
-					outMetricC <- m
+					for i, o := range a.Config.Outputs {
+						if i == len(a.Config.Outputs)-1 {
+							o.AddMetric(m)
+						} else {
+							o.AddMetric(m.Copy())
+						}
+					}
 				}
 			}
 		}
@@ -364,8 +370,6 @@ func (a *Agent) Run(shutdown chan struct{}) error {
 	metricC := make(chan telegraf.Metric, 100)
 	aggC := make(chan telegraf.Metric, 100)
 
-	now := time.Now()
-
 	// Start all ServicePlugins
 	for _, input := range a.Config.Inputs {
 		input.SetDefaultTags(a.Config.Tags)
@@ -406,7 +410,7 @@ func (a *Agent) Run(shutdown chan struct{}) error {
 			acc := NewAccumulator(agg, aggC)
 			acc.SetPrecision(a.Config.Agent.Precision.Duration,
 				a.Config.Agent.Interval.Duration)
-			agg.Run(acc, now, shutdown)
+			agg.Run(acc, shutdown)
 		}(aggregator)
 	}
 

appveyor.yml

@@ -23,7 +23,8 @@ install:
   - go env
 
 build_script:
-  - cmd: C:\GnuWin32\bin\make
+  - cmd: C:\GnuWin32\bin\make deps
+  - cmd: C:\GnuWin32\bin\make telegraf
 
 test_script:
   - cmd: C:\GnuWin32\bin\make test-windows

circle.yml

@@ -1,19 +0,0 @@
-machine:
-  services:
-    - docker
-    - memcached
-    - redis
-    - rabbitmq-server
-  post:
-    - sudo rm -rf /usr/local/go
-    - wget https://storage.googleapis.com/golang/go1.9.2.linux-amd64.tar.gz
-    - sudo tar -C /usr/local -xzf go1.9.2.linux-amd64.tar.gz
-    - go version
-
-dependencies:
-  override:
-    - docker info
-
-test:
-  override:
-    - bash scripts/circle-test.sh

cmd/telegraf/telegraf.go

@@ -54,9 +54,10 @@ var fUsage = flag.String("usage", "",
 	"print usage for a plugin, ie, 'telegraf --usage mysql'")
 var fService = flag.String("service", "",
 	"operate on the service")
+var fRunAsConsole = flag.Bool("console", false, "run as console application (windows only)")
 
 var (
-	nextVersion = "1.5.0"
+	nextVersion = "1.6.0"
 	version     string
 	commit      string
 	branch      string
@@ -358,7 +359,7 @@ func main() {
 		return
 	}
 
-	if runtime.GOOS == "windows" {
+	if runtime.GOOS == "windows" && !(*fRunAsConsole) {
 		svcConfig := &service.Config{
 			Name:        "telegraf",
 			DisplayName: "Telegraf Data Collector Service",

docker-compose.yml

@@ -0,0 +1,93 @@
+version: '3'
+
+services:
+  aerospike:
+    image: aerospike/aerospike-server:3.9.0
+    ports:
+      - "3000:3000"
+  zookeeper:
+    image: wurstmeister/zookeeper
+    environment:
+      - JAVA_OPTS="-Xms256m -Xmx256m"
+    ports:
+      - "2181:2181"
+  kafka:
+    image: wurstmeister/kafka
+    environment:
+      - KAFKA_ADVERTISED_HOST_NAME=localhost
+      - KAFKA_ADVERTISED_PORT=9092
+      - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181
+      - KAFKA_CREATE_TOPICS="test:1:1"
+      - JAVA_OPTS="-Xms256m -Xmx256m"
+    ports:
+      - "9092:9092"
+    depends_on:
+      - zookeeper
+  elasticsearch:
+    image: elasticsearch:5
+    environment:
+    - JAVA_OPTS="-Xms256m -Xmx256m"
+    ports:
+      - "9200:9200"
+      - "9300:9300"
+  mysql:
+    image: mysql
+    environment:
+      - MYSQL_ALLOW_EMPTY_PASSWORD=yes
+    ports:
+      - "3306:3306"
+  memcached:
+    image: memcached
+    ports: 
+      - "11211:11211"
+  postgres:
+    image: postgres:alpine
+    ports:
+      - "5432:5432"
+  rabbitmq:
+    image: rabbitmq:3-management
+    ports:
+      - "15672:15672"
+      - "5672:5672"
+  redis:
+    image: redis:alpine
+    ports:
+      - "6379:6379"
+  nsq:
+    image: nsqio/nsq
+    ports:
+      - "4150:4150"
+    command: "/nsqd"
+  mqtt:
+    image: ncarlier/mqtt
+    ports:
+      - "1883:1883"
+  riemann:
+    image: stealthly/docker-riemann
+    ports:
+      - "5555:5555"
+  nats:
+    image: nats
+    ports:
+      - "4222:4222"
+  openldap:
+    image: cobaugh/openldap-alpine
+    environment:
+      - SLAPD_CONFIG_ROOTDN="cn=manager,cn=config"
+      - SLAPD_CONFIG_ROOTPW="secret"
+    ports:
+      - "389:389"
+      - "636:636"
+  crate:
+   image: crate/crate
+   ports:
+    - "4200:4200"
+    - "4230:4230"
+   command:
+    - crate
+    - -Cnetwork.host=0.0.0.0
+    - -Ctransport.host=localhost
+    - -Clicense.enterprise=false
+   environment:
+    - CRATE_HEAP_SIZE=128m
+    - JAVA_OPTS='-Xms256m -Xmx256m'

docs/CONFIGURATION.md

@@ -125,7 +125,7 @@ aggregator and will not get sent to the output plugins.
 * **name_suffix**: Specifies a suffix to attach to the measurement name.
 * **tags**: A map of tags to apply to a specific input's measurements.
 
-The [measurement filtering](#measurement-filtering) parameters be used to
+The [measurement filtering](#measurement-filtering) parameters can be used to
 limit what metrics are handled by the aggregator.  Excluded metrics are passed
 downstream to the next aggregator.
 
@@ -136,7 +136,7 @@ The following config parameters are available for all processors:
 * **order**: This is the order in which the processor(s) get executed. If this
 is not specified then processor execution order will be random.
 
-The [measurement filtering](#measurement-filtering) can parameters may be used
+The [measurement filtering](#measurement-filtering) parameters can be used
 to limit what metrics are handled by the processor.  Excluded metrics are
 passed downstream to the next processor.
 

docs/DATA_FORMATS_INPUT.md

@@ -8,6 +8,7 @@ Telegraf is able to parse the following input data formats into metrics:
 1. [Value](https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md#value), ie: 45 or "booyah"
 1. [Nagios](https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md#nagios) (exec input only)
 1. [Collectd](https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md#collectd)
+1. [Dropwizard](https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md#dropwizard)
 
 Telegraf metrics, like InfluxDB
 [points](https://docs.influxdata.com/influxdb/v0.10/write_protocols/line/),
@@ -479,3 +480,176 @@ You can also change the path to the typesdb or add additional typesdb using
   ## Path of to TypesDB specifications
   collectd_typesdb = ["/usr/share/collectd/types.db"]
 ```
+
+# Dropwizard:
+
+The dropwizard format can parse the JSON representation of a single dropwizard metric registry. By default, tags are parsed from metric names as if they were actual influxdb line protocol keys (`measurement<,tag_set>`) which can be overriden by defining custom [measurement & tag templates](./DATA_FORMATS_INPUT.md#measurement--tag-templates). All field value types are supported, `string`, `number` and `boolean`.
+
+A typical JSON of a dropwizard metric registry:
+
+```json
+{
+	"version": "3.0.0",
+	"counters" : {
+		"measurement,tag1=green" : {
+			"count" : 1
+		}
+	},
+	"meters" : {
+		"measurement" : {
+			"count" : 1,
+			"m15_rate" : 1.0,
+			"m1_rate" : 1.0,
+			"m5_rate" : 1.0,
+			"mean_rate" : 1.0,
+			"units" : "events/second"
+		}
+	},
+	"gauges" : {
+		"measurement" : {
+			"value" : 1
+		}
+	},
+	"histograms" : {
+		"measurement" : {
+			"count" : 1,
+			"max" : 1.0,
+			"mean" : 1.0,
+			"min" : 1.0,
+			"p50" : 1.0,
+			"p75" : 1.0,
+			"p95" : 1.0,
+			"p98" : 1.0,
+			"p99" : 1.0,
+			"p999" : 1.0,
+			"stddev" : 1.0
+		}
+	},
+	"timers" : {
+		"measurement" : {
+			"count" : 1,
+			"max" : 1.0,
+			"mean" : 1.0,
+			"min" : 1.0,
+			"p50" : 1.0,
+			"p75" : 1.0,
+			"p95" : 1.0,
+			"p98" : 1.0,
+			"p99" : 1.0,
+			"p999" : 1.0,
+			"stddev" : 1.0,
+			"m15_rate" : 1.0,
+			"m1_rate" : 1.0,
+			"m5_rate" : 1.0,
+			"mean_rate" : 1.0,
+			"duration_units" : "seconds",
+			"rate_units" : "calls/second"
+		}
+	}
+}
+```
+
+Would get translated into 4 different measurements:
+
+```
+measurement,metric_type=counter,tag1=green count=1
+measurement,metric_type=meter count=1,m15_rate=1.0,m1_rate=1.0,m5_rate=1.0,mean_rate=1.0
+measurement,metric_type=gauge value=1
+measurement,metric_type=histogram count=1,max=1.0,mean=1.0,min=1.0,p50=1.0,p75=1.0,p95=1.0,p98=1.0,p99=1.0,p999=1.0
+measurement,metric_type=timer count=1,max=1.0,mean=1.0,min=1.0,p50=1.0,p75=1.0,p95=1.0,p98=1.0,p99=1.0,p999=1.0,stddev=1.0,m15_rate=1.0,m1_rate=1.0,m5_rate=1.0,mean_rate=1.0
+```
+
+You may also parse a dropwizard registry from any JSON document which contains a dropwizard registry in some inner field. 
+Eg. to parse the following JSON document:
+
+```json
+{
+	"time" : "2017-02-22T14:33:03.662+02:00",
+	"tags" : {
+		"tag1" : "green",
+		"tag2" : "yellow"
+	},
+	"metrics" : {
+		"counters" : 	{ 
+			"measurement" : {
+				"count" : 1
+			}
+		},
+		"meters" : {},
+		"gauges" : {},
+		"histograms" : {},
+		"timers" : {}
+	}
+}
+```
+and translate it into:
+
+```
+measurement,metric_type=counter,tag1=green,tag2=yellow count=1 1487766783662000000
+```
+
+you simply need to use the following additional configuration properties:
+
+```toml
+dropwizard_metric_registry_path = "metrics"
+dropwizard_time_path = "time"
+dropwizard_time_format = "2006-01-02T15:04:05Z07:00"
+dropwizard_tags_path = "tags"
+## tag paths per tag are supported too, eg.
+#[inputs.yourinput.dropwizard_tag_paths]
+#  tag1 = "tags.tag1"
+#  tag2 = "tags.tag2"
+```
+
+
+For more information about the dropwizard json format see
+[here](http://metrics.dropwizard.io/3.1.0/manual/json/).
+
+#### Dropwizard Configuration:
+
+```toml
+[[inputs.exec]]
+  ## Commands array
+  commands = ["curl http://localhost:8080/sys/metrics"]
+  timeout = "5s"
+
+  ## Data format to consume.
+  ## Each data format has its own unique set of configuration options, read
+  ## more about them here:
+  ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md
+  data_format = "dropwizard"
+
+  ## Used by the templating engine to join matched values when cardinality is > 1
+  separator = "_"
+
+  ## Each template line requires a template pattern. It can have an optional
+  ## filter before the template and separated by spaces. It can also have optional extra
+  ## tags following the template. Multiple tags should be separated by commas and no spaces
+  ## similar to the line protocol format. There can be only one default template.
+  ## Templates support below format:
+  ## 1. filter + template
+  ## 2. filter + template + extra tag(s)
+  ## 3. filter + template with field key
+  ## 4. default template
+  ## By providing an empty template array, templating is disabled and measurements are parsed as influxdb line protocol keys (measurement<,tag_set>)
+  templates = []
+
+  ## You may use an appropriate [gjson path](https://github.com/tidwall/gjson#path-syntax) 
+  ## to locate the metric registry within the JSON document
+  # dropwizard_metric_registry_path = "metrics"
+  
+  ## You may use an appropriate [gjson path](https://github.com/tidwall/gjson#path-syntax) 
+  ## to locate the default time of the measurements within the JSON document
+  # dropwizard_time_path = "time"
+  # dropwizard_time_format = "2006-01-02T15:04:05Z07:00"
+  
+  ## You may use an appropriate [gjson path](https://github.com/tidwall/gjson#path-syntax) 
+  ## to locate the tags map within the JSON document
+  # dropwizard_tags_path = "tags"
+
+  ## You may even use tag paths per tag
+  # [inputs.exec.dropwizard_tag_paths]
+  #   tag1 = "tags.tag1"
+  #   tag2 = "tags.tag2"
+
+```

docs/LICENSE_OF_DEPENDENCIES.md

@@ -54,6 +54,7 @@ following works:
 - github.com/miekg/dns [BSD](https://github.com/miekg/dns/blob/master/LICENSE)
 - github.com/naoina/go-stringutil [MIT](https://github.com/naoina/go-stringutil/blob/master/LICENSE)
 - github.com/naoina/toml [MIT](https://github.com/naoina/toml/blob/master/LICENSE)
+- github.com/nats-io/gnatsd [MIT](https://github.com/nats-io/gnatsd/blob/master/LICENSE)
 - github.com/nats-io/go-nats [MIT](https://github.com/nats-io/go-nats/blob/master/LICENSE)
 - github.com/nats-io/nats [MIT](https://github.com/nats-io/nats/blob/master/LICENSE)
 - github.com/nats-io/nuid [MIT](https://github.com/nats-io/nuid/blob/master/LICENSE)
@@ -82,6 +83,8 @@ following works:
 - github.com/streadway/amqp [BSD](https://github.com/streadway/amqp/blob/master/LICENSE)
 - github.com/stretchr/objx [MIT](https://github.com/stretchr/objx/blob/master/LICENSE.md)
 - github.com/stretchr/testify [MIT](https://github.com/stretchr/testify/blob/master/LICENCE.txt)
+- github.com/tidwall/gjson [MIT](https://github.com/tidwall/gjson/blob/master/LICENSE)
+- github.com/tidwall/match [MIT](https://github.com/tidwall/match/blob/master/LICENSE)
 - github.com/mitchellh/mapstructure [MIT](https://github.com/mitchellh/mapstructure/blob/master/LICENSE)
 - github.com/multiplay/go-ts3 [BSD](https://github.com/multiplay/go-ts3/blob/master/LICENSE)
 - github.com/vjeantet/grok [APACHE](https://github.com/vjeantet/grok/blob/master/LICENSE)

etc/telegraf.conf

@@ -1391,7 +1391,7 @@
 #   ## By default, some of the fields are renamed from what haproxy calls them.
 #   ## Setting this option to true results in the plugin keeping the original
 #   ## field names.
-#   # keep_field_names = true
+#   # keep_field_names = false
 #
 #   ## Optional SSL Config
 #   # ssl_ca = "/etc/telegraf/ca.pem"
@@ -1661,7 +1661,7 @@
 #   # insecure_skip_verify = false
 #
 #   ## Add metrics to read
-#   [[inputs.jolokia2.metric]]
+#   [[inputs.jolokia2_agent.metric]]
 #     name  = "java_runtime"
 #     mbean = "java.lang:type=Runtime"
 #     paths = ["Uptime"]

internal/config/config.go

@@ -40,6 +40,11 @@ var (
 
 	// envVarRe is a regex to find environment variables in the config file
 	envVarRe = regexp.MustCompile(`\$\w+`)
+
+	envVarEscaper = strings.NewReplacer(
+		`"`, `\"`,
+		`\`, `\\`,
+	)
 )
 
 // Config specifies the URL/user/password for the database that telegraf
@@ -689,6 +694,11 @@ func trimBOM(f []byte) []byte {
 	return bytes.TrimPrefix(f, []byte("\xef\xbb\xbf"))
 }
 
+// escapeEnv escapes a value for inserting into a TOML string.
+func escapeEnv(value string) string {
+	return envVarEscaper.Replace(value)
+}
+
 // parseFile loads a TOML configuration from a provided path and
 // returns the AST produced from the TOML parser. When loading the file, it
 // will find environment variables and replace them.
@@ -702,8 +712,9 @@ func parseFile(fpath string) (*ast.Table, error) {
 
 	env_vars := envVarRe.FindAll(contents, -1)
 	for _, env_var := range env_vars {
-		env_val := os.Getenv(strings.TrimPrefix(string(env_var), "$"))
-		if env_val != "" {
+		env_val, ok := os.LookupEnv(strings.TrimPrefix(string(env_var), "$"))
+		if ok {
+			env_val = escapeEnv(env_val)
 			contents = bytes.Replace(contents, env_var, []byte(env_val), 1)
 		}
 	}
@@ -1261,6 +1272,47 @@ func buildParser(name string, tbl *ast.Table) (parsers.Parser, error) {
 		}
 	}
 
+	if node, ok := tbl.Fields["dropwizard_metric_registry_path"]; ok {
+		if kv, ok := node.(*ast.KeyValue); ok {
+			if str, ok := kv.Value.(*ast.String); ok {
+				c.DropwizardMetricRegistryPath = str.Value
+			}
+		}
+	}
+	if node, ok := tbl.Fields["dropwizard_time_path"]; ok {
+		if kv, ok := node.(*ast.KeyValue); ok {
+			if str, ok := kv.Value.(*ast.String); ok {
+				c.DropwizardTimePath = str.Value
+			}
+		}
+	}
+	if node, ok := tbl.Fields["dropwizard_time_format"]; ok {
+		if kv, ok := node.(*ast.KeyValue); ok {
+			if str, ok := kv.Value.(*ast.String); ok {
+				c.DropwizardTimeFormat = str.Value
+			}
+		}
+	}
+	if node, ok := tbl.Fields["dropwizard_tags_path"]; ok {
+		if kv, ok := node.(*ast.KeyValue); ok {
+			if str, ok := kv.Value.(*ast.String); ok {
+				c.DropwizardTagsPath = str.Value
+			}
+		}
+	}
+	c.DropwizardTagPathsMap = make(map[string]string)
+	if node, ok := tbl.Fields["dropwizard_tag_paths"]; ok {
+		if subtbl, ok := node.(*ast.Table); ok {
+			for name, val := range subtbl.Fields {
+				if kv, ok := val.(*ast.KeyValue); ok {
+					if str, ok := kv.Value.(*ast.String); ok {
+						c.DropwizardTagPathsMap[name] = str.Value
+					}
+				}
+			}
+		}
+	}
+
 	c.MetricName = name
 
 	delete(tbl.Fields, "data_format")
@@ -1271,6 +1323,11 @@ func buildParser(name string, tbl *ast.Table) (parsers.Parser, error) {
 	delete(tbl.Fields, "collectd_auth_file")
 	delete(tbl.Fields, "collectd_security_level")
 	delete(tbl.Fields, "collectd_typesdb")
+	delete(tbl.Fields, "dropwizard_metric_registry_path")
+	delete(tbl.Fields, "dropwizard_time_path")
+	delete(tbl.Fields, "dropwizard_time_format")
+	delete(tbl.Fields, "dropwizard_tags_path")
+	delete(tbl.Fields, "dropwizard_tag_paths")
 
 	return parsers.NewParser(c)
 }

internal/internal_test.go

@@ -44,6 +44,9 @@ var (
 )
 
 func TestRunTimeout(t *testing.T) {
+	if testing.Short() {
+		t.Skip("Skipping test due to random failures.")
+	}
 	if sleepbin == "" {
 		t.Skip("'sleep' binary not available on OS, skipping.")
 	}
@@ -58,6 +61,8 @@ func TestRunTimeout(t *testing.T) {
 }
 
 func TestCombinedOutputTimeout(t *testing.T) {
+	// TODO: Fix this test
+	t.Skip("Test failing too often, skip for now and revisit later.")
 	if sleepbin == "" {
 		t.Skip("'sleep' binary not available on OS, skipping.")
 	}
@@ -109,6 +114,8 @@ func TestRunError(t *testing.T) {
 }
 
 func TestRandomSleep(t *testing.T) {
+	// TODO: Fix this test
+	t.Skip("Test failing too often, skip for now and revisit later.")
 	// test that zero max returns immediately
 	s := time.Now()
 	RandomSleep(time.Duration(0), make(chan struct{}))

internal/models/running_aggregator.go

@@ -114,7 +114,6 @@ func (r *RunningAggregator) reset() {
 // for period ticks to tell it when to push and reset the aggregator.
 func (r *RunningAggregator) Run(
 	acc telegraf.Accumulator,
-	now time.Time,
 	shutdown chan struct{},
 ) {
 	// The start of the period is truncated to the nearest second.
@@ -133,6 +132,7 @@ func (r *RunningAggregator) Run(
 	// 2nd interval: 00:10 - 00:20.5
 	// etc.
 	//
+	now := time.Now()
 	r.periodStart = now.Truncate(time.Second)
 	truncation := now.Sub(r.periodStart)
 	r.periodEnd = r.periodStart.Add(r.Config.Period)

internal/models/running_aggregator_test.go

@@ -24,7 +24,7 @@ func TestAdd(t *testing.T) {
 	})
 	assert.NoError(t, ra.Config.Filter.Compile())
 	acc := testutil.Accumulator{}
-	go ra.Run(&acc, time.Now(), make(chan struct{}))
+	go ra.Run(&acc, make(chan struct{}))
 
 	m := ra.MakeMetric(
 		"RITest",
@@ -55,7 +55,7 @@ func TestAddMetricsOutsideCurrentPeriod(t *testing.T) {
 	})
 	assert.NoError(t, ra.Config.Filter.Compile())
 	acc := testutil.Accumulator{}
-	go ra.Run(&acc, time.Now(), make(chan struct{}))
+	go ra.Run(&acc, make(chan struct{}))
 
 	// metric before current period
 	m := ra.MakeMetric(
@@ -113,7 +113,7 @@ func TestAddAndPushOnePeriod(t *testing.T) {
 	wg.Add(1)
 	go func() {
 		defer wg.Done()
-		ra.Run(&acc, time.Now(), shutdown)
+		ra.Run(&acc, shutdown)
 	}()
 
 	m := ra.MakeMetric(

internal/templating/engine.go

@@ -0,0 +1,86 @@
+package templating
+
+import (
+	"sort"
+	"strings"
+)
+
+const (
+	// DefaultSeparator is the default separation character to use when separating template parts.
+	DefaultSeparator = "."
+)
+
+// Engine uses a Matcher to retrieve the appropriate template and applies the template
+// to the input string
+type Engine struct {
+	joiner  string
+	matcher *matcher
+}
+
+// Apply extracts the template fields from the given line and returns the measurement
+// name, tags and field name
+func (e *Engine) Apply(line string) (string, map[string]string, string, error) {
+	return e.matcher.match(line).Apply(line, e.joiner)
+}
+
+// NewEngine creates a new templating engine
+func NewEngine(joiner string, defaultTemplate *Template, templates []string) (*Engine, error) {
+	engine := Engine{
+		joiner:  joiner,
+		matcher: newMatcher(defaultTemplate),
+	}
+	templateSpecs := parseTemplateSpecs(templates)
+
+	for _, templateSpec := range templateSpecs {
+		if err := engine.matcher.addSpec(templateSpec); err != nil {
+			return nil, err
+		}
+	}
+
+	return &engine, nil
+}
+
+func parseTemplateSpecs(templates []string) templateSpecs {
+	tmplts := templateSpecs{}
+	for _, pattern := range templates {
+		tmplt := templateSpec{
+			separator: DefaultSeparator,
+		}
+
+		// Format is [separator] [filter] <template> [tag1=value1,tag2=value2]
+		parts := strings.Fields(pattern)
+		partsLength := len(parts)
+		if partsLength < 1 {
+			// ignore
+			continue
+		}
+		if partsLength == 1 {
+			tmplt.template = pattern
+		} else if partsLength == 4 {
+			tmplt.separator = parts[0]
+			tmplt.filter = parts[1]
+			tmplt.template = parts[2]
+			tmplt.tagstring = parts[3]
+		} else {
+			hasTagstring := strings.Contains(parts[partsLength-1], "=")
+			if hasTagstring {
+				tmplt.tagstring = parts[partsLength-1]
+				tmplt.template = parts[partsLength-2]
+				if partsLength == 3 {
+					tmplt.filter = parts[0]
+				}
+			} else {
+				tmplt.template = parts[partsLength-1]
+				if partsLength == 2 {
+					tmplt.filter = parts[0]
+				} else { // length == 3
+					tmplt.separator = parts[0]
+					tmplt.filter = parts[1]
+				}
+			}
+		}
+		tmplts = append(tmplts, tmplt)
+	}
+	sort.Sort(tmplts)
+	return tmplts
+}

internal/templating/matcher.go

@@ -0,0 +1,58 @@
+package templating
+
+import (
+	"strings"
+)
+
+// matcher determines which template should be applied to a given metric
+// based on a filter tree.
+type matcher struct {
+	root            *node
+	defaultTemplate *Template
+}
+
+// newMatcher creates a new matcher.
+func newMatcher(defaultTemplate *Template) *matcher {
+	return &matcher{
+		root:            &node{},
+		defaultTemplate: defaultTemplate,
+	}
+}
+
+func (m *matcher) addSpec(tmplt templateSpec) error {
+	// Parse out the default tags specific to this template
+	tags := map[string]string{}
+	if tmplt.tagstring != "" {
+		for _, kv := range strings.Split(tmplt.tagstring, ",") {
+			parts := strings.Split(kv, "=")
+			tags[parts[0]] = parts[1]
+		}
+	}
+
+	tmpl, err := NewTemplate(tmplt.separator, tmplt.template, tags)
+	if err != nil {
+		return err
+	}
+	m.add(tmplt.filter, tmpl)
+	return nil
+}
+
+// add inserts the template in the filter tree based the given filter
+func (m *matcher) add(filter string, template *Template) {
+	if filter == "" {
+		m.defaultTemplate = template
+		m.root.separator = template.separator
+		return
+	}
+	m.root.insert(filter, template)
+}
+
+// match returns the template that matches the given measurement line.
+// If no template matches, the default template is returned.
+func (m *matcher) match(line string) *Template {
+	tmpl := m.root.search(line)
+	if tmpl != nil {
+		return tmpl
+	}
+	return m.defaultTemplate
+}

internal/templating/node.go

@@ -0,0 +1,122 @@
+package templating
+
+import (
+	"sort"
+	"strings"
+)
+
+// node is an item in a sorted k-ary tree of filter parts. Each child is sorted by its part value.
+// The special value of "*", is always sorted last.
+type node struct {
+	separator string
+	value     string
+	children  nodes
+	template  *Template
+}
+
+// insert inserts the given string template into the tree.  The filter string is separated
+// on the template separator and each part is used as the path in the tree.
+func (n *node) insert(filter string, template *Template) {
+	n.separator = template.separator
+	n.recursiveInsert(strings.Split(filter, n.separator), template)
+}
+
+// recursiveInsert does the actual recursive insertion
+func (n *node) recursiveInsert(values []string, template *Template) {
+	// Add the end, set the template
+	if len(values) == 0 {
+		n.template = template
+		return
+	}
+
+	// See if the the current element already exists in the tree. If so, insert the
+	// into that sub-tree
+	for _, v := range n.children {
+		if v.value == values[0] {
+			v.recursiveInsert(values[1:], template)
+			return
+		}
+	}
+
+	// New element, add it to the tree and sort the children
+	newNode := &node{value: values[0]}
+	n.children = append(n.children, newNode)
+	sort.Sort(&n.children)
+
+	// Now insert the rest of the tree into the new element
+	newNode.recursiveInsert(values[1:], template)
+}
+
+// search searches for a template matching the input string
+func (n *node) search(line string) *Template {
+	separator := n.separator
+	return n.recursiveSearch(strings.Split(line, separator))
+}
+
+// recursiveSearch performs the actual recursive search
+func (n *node) recursiveSearch(lineParts []string) *Template {
+	// Nothing to search
+	if len(lineParts) == 0 || len(n.children) == 0 {
+		return n.template
+	}
+
+	// If last element is a wildcard, don't include it in this search since it's sorted
+	// to the end but lexicographically it would not always be and sort.Search assumes
+	// the slice is sorted.
+	length := len(n.children)
+	if n.children[length-1].value == "*" {
+		length--
+	}
+
+	// Find the index of child with an exact match
+	i := sort.Search(length, func(i int) bool {
+		return n.children[i].value >= lineParts[0]
+	})
+
+	// Found an exact match, so search that child sub-tree
+	if i < len(n.children) && n.children[i].value == lineParts[0] {
+		return n.children[i].recursiveSearch(lineParts[1:])
+	}
+	// Not an exact match, see if we have a wildcard child to search
+	if n.children[len(n.children)-1].value == "*" {
+		return n.children[len(n.children)-1].recursiveSearch(lineParts[1:])
+	}
+	return n.template
+}
+
+// nodes is simply an array of nodes implementing the sorting interface.
+type nodes []*node
+
+// Less returns a boolean indicating whether the filter at position j
+// is less than the filter at position k. Filters are order by string
+// comparison of each component parts.  A wildcard value "*" is never
+// less than a non-wildcard value.
+//
+// For example, the filters:
+//             "*.*"
+//             "servers.*"
+//             "servers.localhost"
+//             "*.localhost"
+//
+// Would be sorted as:
+//             "servers.localhost"
+//             "servers.*"
+//             "*.localhost"
+//             "*.*"
+func (n *nodes) Less(j, k int) bool {
+	if (*n)[j].value == "*" && (*n)[k].value != "*" {
+		return false
+	}
+
+	if (*n)[j].value != "*" && (*n)[k].value == "*" {
+		return true
+	}
+
+	return (*n)[j].value < (*n)[k].value
+}
+
+// Swap swaps two elements of the array
+func (n *nodes) Swap(i, j int) { (*n)[i], (*n)[j] = (*n)[j], (*n)[i] }
+
+// Len returns the length of the array
+func (n *nodes) Len() int { return len(*n) }

internal/templating/template.go

@@ -0,0 +1,148 @@
+package templating
+
+import (
+	"fmt"
+	"strings"
+)
+
+// Template represents a pattern and tags to map a metric string to a influxdb Point
+type Template struct {
+	separator         string
+	parts             []string
+	defaultTags       map[string]string
+	greedyField       bool
+	greedyMeasurement bool
+}
+
+// apply extracts the template fields from the given line and returns the measurement
+// name, tags and field name
+func (t *Template) Apply(line string, joiner string) (string, map[string]string, string, error) {
+	fields := strings.Split(line, t.separator)
+	var (
+		measurement []string
+		tags        = make(map[string][]string)
+		field       []string
+	)
+
+	// Set any default tags
+	for k, v := range t.defaultTags {
+		tags[k] = append(tags[k], v)
+	}
+
+	// See if an invalid combination has been specified in the template:
+	for _, tag := range t.parts {
+		if tag == "measurement*" {
+			t.greedyMeasurement = true
+		} else if tag == "field*" {
+			t.greedyField = true
+		}
+	}
+	if t.greedyField && t.greedyMeasurement {
+		return "", nil, "",
+			fmt.Errorf("either 'field*' or 'measurement*' can be used in each "+
+				"template (but not both together): %q",
+				strings.Join(t.parts, joiner))
+	}
+
+	for i, tag := range t.parts {
+		if i >= len(fields) {
+			continue
+		}
+		if tag == "" {
+			continue
+		}
+
+		switch tag {
+		case "measurement":
+			measurement = append(measurement, fields[i])
+		case "field":
+			field = append(field, fields[i])
+		case "field*":
+			field = append(field, fields[i:]...)
+			break
+		case "measurement*":
+			measurement = append(measurement, fields[i:]...)
+			break
+		default:
+			tags[tag] = append(tags[tag], fields[i])
+		}
+	}
+
+	// Convert to map of strings.
+	outtags := make(map[string]string)
+	for k, values := range tags {
+		outtags[k] = strings.Join(values, joiner)
+	}
+
+	return strings.Join(measurement, joiner), outtags, strings.Join(field, joiner), nil
+}
+
+func NewDefaultTemplateWithPattern(pattern string) (*Template, error) {
+	return NewTemplate(DefaultSeparator, pattern, nil)
+}
+
+// NewTemplate returns a new template ensuring it has a measurement
+// specified.
+func NewTemplate(separator string, pattern string, defaultTags map[string]string) (*Template, error) {
+	parts := strings.Split(pattern, separator)
+	hasMeasurement := false
+	template := &Template{
+		separator:   separator,
+		parts:       parts,
+		defaultTags: defaultTags,
+	}
+
+	for _, part := range parts {
+		if strings.HasPrefix(part, "measurement") {
+			hasMeasurement = true
+		}
+		if part == "measurement*" {
+			template.greedyMeasurement = true
+		} else if part == "field*" {
+			template.greedyField = true
+		}
+	}
+
+	if !hasMeasurement {
+		return nil, fmt.Errorf("no measurement specified for template. %q", pattern)
+	}
+
+	return template, nil
+}
+
+// templateSpec is a template string split in its constituent parts
+type templateSpec struct {
+	separator string
+	filter    string
+	template  string
+	tagstring string
+}
+
+// templateSpecs is simply an array of template specs implementing the sorting interface
+type templateSpecs []templateSpec
+
+// Less reports whether the element with
+// index j should sort before the element with index k.
+func (e templateSpecs) Less(j, k int) bool {
+	if len(e[j].filter) == 0 && len(e[k].filter) == 0 {
+		jlength := len(strings.Split(e[j].template, e[j].separator))
+		klength := len(strings.Split(e[k].template, e[k].separator))
+		return jlength < klength
+	}
+	if len(e[j].filter) == 0 {
+		return true
+	}
+	if len(e[k].filter) == 0 {
+		return false
+	}
+
+	jlength := len(strings.Split(e[j].template, e[j].separator))
+	klength := len(strings.Split(e[k].template, e[k].separator))
+	return jlength < klength
+}
+
+// Swap swaps the elements with indexes i and j.
+func (e templateSpecs) Swap(i, j int) { e[i], e[j] = e[j], e[i] }
+
+// Len is the number of elements in the collection.
+func (e templateSpecs) Len() int { return len(e) }

plugins/aggregators/basicstats/README.md

@@ -8,14 +8,26 @@ emitting the aggregate every `period` seconds.
 ```toml
 # Keep the aggregate basicstats of each metric passing through.
 [[aggregators.basicstats]]
+
   ## General Aggregator Arguments:
+
   ## The period on which to flush & clear the aggregator.
   period = "30s"
+
   ## If true, the original metric will be dropped by the
   ## aggregator and will not get sent to the output plugins.
   drop_original = false
+
+  ## BasicStats Arguments:
+
+  ## Configures which basic stats to push as fields
+  stats = ["count","min","max","mean","stdev","s2"]
 ```
 
+- stats
+    - If not specified, all stats are aggregated and pushed as fields
+    - If empty array, no stats are aggregated
+
 ### Measurements & Fields:
 
 - measurement1

plugins/aggregators/basicstats/basicstats.go

@@ -1,6 +1,7 @@
 package basicstats
 
 import (
+	"log"
 	"math"
 
 	"github.com/influxdata/telegraf"
@@ -8,10 +9,22 @@ import (
 )
 
 type BasicStats struct {
-	cache map[uint64]aggregate
+	Stats []string `toml:"stats"`
+
+	cache       map[uint64]aggregate
+	statsConfig *configuredStats
 }
 
-func NewBasicStats() telegraf.Aggregator {
+type configuredStats struct {
+	count    bool
+	min      bool
+	max      bool
+	mean     bool
+	variance bool
+	stdev    bool
+}
+
+func NewBasicStats() *BasicStats {
 	mm := &BasicStats{}
 	mm.Reset()
 	return mm
@@ -114,25 +127,103 @@ func (m *BasicStats) Add(in telegraf.Metric) {
 }
 
 func (m *BasicStats) Push(acc telegraf.Accumulator) {
+
+	config := getConfiguredStats(m)
+
 	for _, aggregate := range m.cache {
 		fields := map[string]interface{}{}
 		for k, v := range aggregate.fields {
-			fields[k+"_count"] = v.count
-			fields[k+"_min"] = v.min
-			fields[k+"_max"] = v.max
-			fields[k+"_mean"] = v.mean
+
+			if config.count {
+				fields[k+"_count"] = v.count
+			}
+			if config.min {
+				fields[k+"_min"] = v.min
+			}
+			if config.max {
+				fields[k+"_max"] = v.max
+			}
+			if config.mean {
+				fields[k+"_mean"] = v.mean
+			}
+
 			//v.count always >=1
 			if v.count > 1 {
 				variance := v.M2 / (v.count - 1)
-				fields[k+"_s2"] = variance
-				fields[k+"_stdev"] = math.Sqrt(variance)
+
+				if config.variance {
+					fields[k+"_s2"] = variance
+				}
+				if config.stdev {
+					fields[k+"_stdev"] = math.Sqrt(variance)
+				}
 			}
 			//if count == 1 StdDev = infinite => so I won't send data
 		}
-		acc.AddFields(aggregate.name, fields, aggregate.tags)
+
+		if len(fields) > 0 {
+			acc.AddFields(aggregate.name, fields, aggregate.tags)
+		}
 	}
 }
 
+func parseStats(names []string) *configuredStats {
+
+	parsed := &configuredStats{}
+
+	for _, name := range names {
+
+		switch name {
+
+		case "count":
+			parsed.count = true
+		case "min":
+			parsed.min = true
+		case "max":
+			parsed.max = true
+		case "mean":
+			parsed.mean = true
+		case "s2":
+			parsed.variance = true
+		case "stdev":
+			parsed.stdev = true
+
+		default:
+			log.Printf("W! Unrecognized basic stat '%s', ignoring", name)
+		}
+	}
+
+	return parsed
+}
+
+func defaultStats() *configuredStats {
+
+	defaults := &configuredStats{}
+
+	defaults.count = true
+	defaults.min = true
+	defaults.max = true
+	defaults.mean = true
+	defaults.variance = true
+	defaults.stdev = true
+
+	return defaults
+}
+
+func getConfiguredStats(m *BasicStats) *configuredStats {
+
+	if m.statsConfig == nil {
+
+		if m.Stats == nil {
+			m.statsConfig = defaultStats()
+		} else {
+			m.statsConfig = parseStats(m.Stats)
+		}
+	}
+
+	return m.statsConfig
+}
+
 func (m *BasicStats) Reset() {
 	m.cache = make(map[uint64]aggregate)
 }

plugins/aggregators/basicstats/basicstats_test.go

@@ -149,3 +149,211 @@ func TestBasicStatsDifferentPeriods(t *testing.T) {
 	}
 	acc.AssertContainsTaggedFields(t, "m1", expectedFields, expectedTags)
 }
+
+// Test only aggregating count
+func TestBasicStatsWithOnlyCount(t *testing.T) {
+
+	aggregator := NewBasicStats()
+	aggregator.Stats = []string{"count"}
+
+	aggregator.Add(m1)
+	aggregator.Add(m2)
+
+	acc := testutil.Accumulator{}
+	aggregator.Push(&acc)
+
+	expectedFields := map[string]interface{}{
+		"a_count": float64(2),
+		"b_count": float64(2),
+		"c_count": float64(2),
+		"d_count": float64(2),
+		"e_count": float64(1),
+	}
+	expectedTags := map[string]string{
+		"foo": "bar",
+	}
+	acc.AssertContainsTaggedFields(t, "m1", expectedFields, expectedTags)
+}
+
+// Test only aggregating minimum
+func TestBasicStatsWithOnlyMin(t *testing.T) {
+
+	aggregator := NewBasicStats()
+	aggregator.Stats = []string{"min"}
+
+	aggregator.Add(m1)
+	aggregator.Add(m2)
+
+	acc := testutil.Accumulator{}
+	aggregator.Push(&acc)
+
+	expectedFields := map[string]interface{}{
+		"a_min": float64(1),
+		"b_min": float64(1),
+		"c_min": float64(2),
+		"d_min": float64(2),
+		"e_min": float64(200),
+	}
+	expectedTags := map[string]string{
+		"foo": "bar",
+	}
+	acc.AssertContainsTaggedFields(t, "m1", expectedFields, expectedTags)
+}
+
+// Test only aggregating maximum
+func TestBasicStatsWithOnlyMax(t *testing.T) {
+
+	aggregator := NewBasicStats()
+	aggregator.Stats = []string{"max"}
+
+	aggregator.Add(m1)
+	aggregator.Add(m2)
+
+	acc := testutil.Accumulator{}
+	aggregator.Push(&acc)
+
+	expectedFields := map[string]interface{}{
+		"a_max": float64(1),
+		"b_max": float64(3),
+		"c_max": float64(4),
+		"d_max": float64(6),
+		"e_max": float64(200),
+	}
+	expectedTags := map[string]string{
+		"foo": "bar",
+	}
+	acc.AssertContainsTaggedFields(t, "m1", expectedFields, expectedTags)
+}
+
+// Test only aggregating mean
+func TestBasicStatsWithOnlyMean(t *testing.T) {
+
+	aggregator := NewBasicStats()
+	aggregator.Stats = []string{"mean"}
+
+	aggregator.Add(m1)
+	aggregator.Add(m2)
+
+	acc := testutil.Accumulator{}
+	aggregator.Push(&acc)
+
+	expectedFields := map[string]interface{}{
+		"a_mean": float64(1),
+		"b_mean": float64(2),
+		"c_mean": float64(3),
+		"d_mean": float64(4),
+		"e_mean": float64(200),
+	}
+	expectedTags := map[string]string{
+		"foo": "bar",
+	}
+	acc.AssertContainsTaggedFields(t, "m1", expectedFields, expectedTags)
+}
+
+// Test only aggregating variance
+func TestBasicStatsWithOnlyVariance(t *testing.T) {
+
+	aggregator := NewBasicStats()
+	aggregator.Stats = []string{"s2"}
+
+	aggregator.Add(m1)
+	aggregator.Add(m2)
+
+	acc := testutil.Accumulator{}
+	aggregator.Push(&acc)
+
+	expectedFields := map[string]interface{}{
+		"a_s2": float64(0),
+		"b_s2": float64(2),
+		"c_s2": float64(2),
+		"d_s2": float64(8),
+	}
+	expectedTags := map[string]string{
+		"foo": "bar",
+	}
+	acc.AssertContainsTaggedFields(t, "m1", expectedFields, expectedTags)
+}
+
+// Test only aggregating standard deviation
+func TestBasicStatsWithOnlyStandardDeviation(t *testing.T) {
+
+	aggregator := NewBasicStats()
+	aggregator.Stats = []string{"stdev"}
+
+	aggregator.Add(m1)
+	aggregator.Add(m2)
+
+	acc := testutil.Accumulator{}
+	aggregator.Push(&acc)
+
+	expectedFields := map[string]interface{}{
+		"a_stdev": float64(0),
+		"b_stdev": math.Sqrt(2),
+		"c_stdev": math.Sqrt(2),
+		"d_stdev": math.Sqrt(8),
+	}
+	expectedTags := map[string]string{
+		"foo": "bar",
+	}
+	acc.AssertContainsTaggedFields(t, "m1", expectedFields, expectedTags)
+}
+
+// Test only aggregating minimum and maximum
+func TestBasicStatsWithMinAndMax(t *testing.T) {
+
+	aggregator := NewBasicStats()
+	aggregator.Stats = []string{"min", "max"}
+
+	aggregator.Add(m1)
+	aggregator.Add(m2)
+
+	acc := testutil.Accumulator{}
+	aggregator.Push(&acc)
+
+	expectedFields := map[string]interface{}{
+		"a_max": float64(1), //a
+		"a_min": float64(1),
+		"b_max": float64(3), //b
+		"b_min": float64(1),
+		"c_max": float64(4), //c
+		"c_min": float64(2),
+		"d_max": float64(6), //d
+		"d_min": float64(2),
+		"e_max": float64(200), //e
+		"e_min": float64(200),
+	}
+	expectedTags := map[string]string{
+		"foo": "bar",
+	}
+	acc.AssertContainsTaggedFields(t, "m1", expectedFields, expectedTags)
+}
+
+// Test that if an empty array is passed, no points are pushed
+func TestBasicStatsWithNoStats(t *testing.T) {
+
+	aggregator := NewBasicStats()
+	aggregator.Stats = []string{}
+
+	aggregator.Add(m1)
+	aggregator.Add(m2)
+
+	acc := testutil.Accumulator{}
+	aggregator.Push(&acc)
+
+	acc.AssertDoesNotContainMeasurement(t, "m1")
+}
+
+// Test that if an unknown stat is configured, it doesn't explode
+func TestBasicStatsWithUnknownStat(t *testing.T) {
+
+	aggregator := NewBasicStats()
+	aggregator.Stats = []string{"crazy"}
+
+	aggregator.Add(m1)
+	aggregator.Add(m2)
+
+	acc := testutil.Accumulator{}
+	aggregator.Push(&acc)
+
+	acc.AssertDoesNotContainMeasurement(t, "m1")
+}

plugins/inputs/all/all.go

@@ -36,6 +36,7 @@ import (
 	_ "github.com/influxdata/telegraf/plugins/inputs/internal"
 	_ "github.com/influxdata/telegraf/plugins/inputs/interrupts"
 	_ "github.com/influxdata/telegraf/plugins/inputs/ipmi_sensor"
+	_ "github.com/influxdata/telegraf/plugins/inputs/ipset"
 	_ "github.com/influxdata/telegraf/plugins/inputs/iptables"
 	_ "github.com/influxdata/telegraf/plugins/inputs/jolokia"
 	_ "github.com/influxdata/telegraf/plugins/inputs/jolokia2"
@@ -53,6 +54,7 @@ import (
 	_ "github.com/influxdata/telegraf/plugins/inputs/mongodb"
 	_ "github.com/influxdata/telegraf/plugins/inputs/mqtt_consumer"
 	_ "github.com/influxdata/telegraf/plugins/inputs/mysql"
+	_ "github.com/influxdata/telegraf/plugins/inputs/nats"
 	_ "github.com/influxdata/telegraf/plugins/inputs/nats_consumer"
 	_ "github.com/influxdata/telegraf/plugins/inputs/net_response"
 	_ "github.com/influxdata/telegraf/plugins/inputs/nginx"

plugins/inputs/apache/README.md

@@ -2,7 +2,7 @@
 
 The Apache plugin collects server performance information using the [`mod_status`](https://httpd.apache.org/docs/2.4/mod/mod_status.html) module of the [Apache HTTP Server](https://httpd.apache.org/).
 
-Typically, the `mod_status` module is configured to expose a page at the `/server-status?auto` location of the Apache server.  The [ExtendedStatus](https://httpd.apache.org/docs/2.4/mod/core.html#extendedstatus) option must be enabled in order to collect all available fields.  For information about how to configure your server reference the [module documenation](https://httpd.apache.org/docs/2.4/mod/mod_status.html#enable).
+Typically, the `mod_status` module is configured to expose a page at the `/server-status?auto` location of the Apache server.  The [ExtendedStatus](https://httpd.apache.org/docs/2.4/mod/core.html#extendedstatus) option must be enabled in order to collect all available fields.  For information about how to configure your server reference the [module documentation](https://httpd.apache.org/docs/2.4/mod/mod_status.html#enable).
 
 ### Configuration:
 

plugins/inputs/bond/README.md

@@ -1,7 +1,7 @@
 # Bond Input Plugin
 
-The Bond Input plugin collects bond interface status, bond's slaves interfaces
-status and failures count of bond's slaves interfaces.
+The Bond input plugin collects network bond interface status for both the
+network bond interface as well as slave interfaces.
 The plugin collects these metrics from `/proc/net/bonding/*` files.
 
 ### Configuration:

plugins/inputs/dcos/README.md

@@ -106,7 +106,7 @@ the cluster.  For more information on this technique reference
 ### Metrics:
 
 Please consult the [Metrics Reference](https://docs.mesosphere.com/1.10/metrics/reference/)
-for details on interprete field interpretation.
+for details about field interpretation.
 
 - dcos_node
   - tags:

plugins/inputs/dcos/client.go

@@ -325,7 +325,7 @@ func (c *ClusterClient) createLoginToken(sa *ServiceAccount) (string, error) {
 		UID: sa.AccountID,
 		StandardClaims: jwt.StandardClaims{
 			// How long we have to login with this token
-			ExpiresAt: int64(5 * time.Minute / time.Second),
+			ExpiresAt: time.Now().Add(5 * time.Minute).Unix(),
 		},
 	})
 	return token.SignedString(sa.PrivateKey)

plugins/inputs/docker/README.md

@@ -199,6 +199,9 @@ based on the availability of per-cpu stats on your system.
     - network
 - docker_container_blkio specific:
     - device
+- docker_container_health specific:
+    - health_status
+    - failing_streak
 - docker_swarm specific:
     - service_id
     - service_name
@@ -257,4 +260,4 @@ io_serviced_recursive_total=6599i,io_serviced_recursive_write=107i 1453409536840
 >docker_swarm,
 service_id=xaup2o9krw36j2dy1mjx1arjw,service_mode=replicated,service_name=test,\
 tasks_desired=3,tasks_running=3 1508968160000000000
-```
+```

plugins/inputs/docker/docker.go

@@ -391,12 +391,13 @@ func (d *Docker) gatherContainer(
 		}
 	}
 
+	info, err := d.client.ContainerInspect(ctx, container.ID)
+	if err != nil {
+		return fmt.Errorf("Error inspecting docker container: %s", err.Error())
+	}
+
 	// Add whitelisted environment variables to tags
 	if len(d.TagEnvironment) > 0 {
-		info, err := d.client.ContainerInspect(ctx, container.ID)
-		if err != nil {
-			return fmt.Errorf("Error inspecting docker container: %s", err.Error())
-		}
 		for _, envvar := range info.Config.Env {
 			for _, configvar := range d.TagEnvironment {
 				dock_env := strings.SplitN(envvar, "=", 2)
@@ -408,6 +409,14 @@ func (d *Docker) gatherContainer(
 		}
 	}
 
+	if info.State.Health != nil {
+		healthfields := map[string]interface{}{
+			"health_status":  info.State.Health.Status,
+			"failing_streak": info.ContainerJSONBase.State.Health.FailingStreak,
+		}
+		acc.AddFields("docker_container_health", healthfields, tags, time.Now())
+	}
+
 	gatherContainerStats(v, acc, tags, container.ID, d.PerDevice, d.Total, daemonOSType)
 
 	return nil

plugins/inputs/docker/docker_testdata.go

@@ -477,4 +477,12 @@ var containerInspect = types.ContainerJSON{
 			"PATH=/bin:/sbin",
 		},
 	},
+	ContainerJSONBase: &types.ContainerJSONBase{
+		State: &types.ContainerState{
+			Health: &types.Health{
+				FailingStreak: 1,
+				Status:        "Unhealthy",
+			},
+		},
+	},
 }

plugins/inputs/elasticsearch/README.md

@@ -46,6 +46,17 @@ or [cluster-stats](https://www.elastic.co/guide/en/elasticsearch/reference/curre
   # insecure_skip_verify = false
 ```
 
+### Status mappings
+
+When reporting health (green/yellow/red), additional field `status_code`
+is reported. Field contains mapping from status:string to status_code:int
+with following rules:
+
+* `green` - 1
+* `yellow` - 2
+* `red` - 3
+* `unknown` - 0
+
 ### Measurements & Fields:
 
 field data circuit breaker measurement names:

plugins/inputs/elasticsearch/elasticsearch.go

@@ -143,6 +143,19 @@ func NewElasticsearch() *Elasticsearch {
 	}
 }
 
+// perform status mapping
+func mapHealthStatusToCode(s string) int {
+	switch strings.ToLower(s) {
+	case "green":
+		return 1
+	case "yellow":
+		return 2
+	case "red":
+		return 3
+	}
+	return 0
+}
+
 // SampleConfig returns sample configuration for this plugin.
 func (e *Elasticsearch) SampleConfig() string {
 	return sampleConfig
@@ -311,6 +324,7 @@ func (e *Elasticsearch) gatherClusterHealth(url string, acc telegraf.Accumulator
 	measurementTime := time.Now()
 	clusterFields := map[string]interface{}{
 		"status":                healthStats.Status,
+		"status_code":           mapHealthStatusToCode(healthStats.Status),
 		"timed_out":             healthStats.TimedOut,
 		"number_of_nodes":       healthStats.NumberOfNodes,
 		"number_of_data_nodes":  healthStats.NumberOfDataNodes,
@@ -330,6 +344,7 @@ func (e *Elasticsearch) gatherClusterHealth(url string, acc telegraf.Accumulator
 	for name, health := range healthStats.Indices {
 		indexFields := map[string]interface{}{
 			"status":                health.Status,
+			"status_code":           mapHealthStatusToCode(health.Status),
 			"number_of_shards":      health.NumberOfShards,
 			"number_of_replicas":    health.NumberOfReplicas,
 			"active_primary_shards": health.ActivePrimaryShards,

plugins/inputs/elasticsearch/testdata_test.go

@@ -54,6 +54,7 @@ const clusterHealthResponseWithIndices = `
 
 var clusterHealthExpected = map[string]interface{}{
 	"status":                "green",
+	"status_code":           1,
 	"timed_out":             false,
 	"number_of_nodes":       3,
 	"number_of_data_nodes":  3,
@@ -66,6 +67,7 @@ var clusterHealthExpected = map[string]interface{}{
 
 var v1IndexExpected = map[string]interface{}{
 	"status":                "green",
+	"status_code":           1,
 	"number_of_shards":      10,
 	"number_of_replicas":    1,
 	"active_primary_shards": 10,
@@ -77,6 +79,7 @@ var v1IndexExpected = map[string]interface{}{
 
 var v2IndexExpected = map[string]interface{}{
 	"status":                "red",
+	"status_code":           3,
 	"number_of_shards":      10,
 	"number_of_replicas":    1,
 	"active_primary_shards": 0,

plugins/inputs/exec/README.md

@@ -1,175 +1,57 @@
 # Exec Input Plugin
 
-Please also see: [Telegraf Input Data Formats](https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md)
+The `exec` plugin executes the `commands` on every interval and parses metrics from
+their output in any one of the accepted [Input Data Formats](https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md).
 
-### Example 1 - JSON
+This plugin can be used to poll for custom metrics from any source.
 
-#### Configuration
-
-In this example a script called ```/tmp/test.sh```, a script called ```/tmp/test2.sh```, and
-all scripts matching glob pattern ```/tmp/collect_*.sh``` are configured for ```[[inputs.exec]]```
-in JSON format. Glob patterns are matched on every run, so adding new scripts that match the pattern
-will cause them to be picked up immediately.
+### Configuration:
 
 ```toml
-# Read flattened metrics from one or more commands that output JSON to stdout
 [[inputs.exec]]
-  # Shell/commands array
-  # Full command line to executable with parameters, or a glob pattern to run all matching files.
-  commands = ["/tmp/test.sh", "/tmp/test2.sh", "/tmp/collect_*.sh"]
+  ## Commands array
+  commands = [
+    "/tmp/test.sh",
+    "/usr/bin/mycollector --foo=bar",
+    "/tmp/collect_*.sh"
+  ]
 
   ## Timeout for each command to complete.
   timeout = "5s"
 
-  # Data format to consume.
-  # NOTE json only reads numerical measurements, strings and booleans are ignored.
-  data_format = "json"
-
-  # measurement name suffix (for separating different commands)
+  ## measurement name suffix (for separating different commands)
   name_suffix = "_mycollector"
-```
 
-Other options for modifying the measurement names are:
-
-```
-name_prefix = "prefix_"
-```
-
-Let's say that we have the above configuration, and mycollector outputs the
-following JSON:
-
-```json
-{
-    "a": 0.5,
-    "b": {
-        "c": 0.1,
-        "d": 5
-    }
-}
-```
-
-The collected metrics will be stored as fields under the measurement
-"exec_mycollector":
-
-```
-exec_mycollector a=0.5,b_c=0.1,b_d=5 1452815002357578567
-```
-If using JSON, only numeric values are parsed and turned into floats. Booleans
-and strings will be ignored.
-
-### Example 2 - Influx Line-Protocol
-
-In this example an application called ```/usr/bin/line_protocol_collector```
-and a script called ```/tmp/test2.sh``` are configured for ```[[inputs.exec]]```
-in influx line-protocol format.
-
-#### Configuration
-
-```toml
-[[inputs.exec]]
-  # Shell/commands array
-  # compatible with old version
-  # we can still use the old command configuration
-  # command = "/usr/bin/line_protocol_collector"
-  commands = ["/usr/bin/line_protocol_collector","/tmp/test2.sh"]
-
-  ## Timeout for each command to complete.
-  timeout = "5s"
-
-  # Data format to consume.
-  # NOTE json only reads numerical measurements, strings and booleans are ignored.
+  ## Data format to consume.
+  ## Each data format has its own unique set of configuration options, read
+  ## more about them here:
+  ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md
   data_format = "influx"
 ```
 
-The line_protocol_collector application outputs the following line protocol:
+Glob patterns in the `command` option are matched on every run, so adding new
+scripts that match the pattern will cause them to be picked up immediately.
 
-```
-cpu,cpu=cpu0,host=foo,datacenter=us-east usage_idle=99,usage_busy=1
-cpu,cpu=cpu1,host=foo,datacenter=us-east usage_idle=99,usage_busy=1
-cpu,cpu=cpu2,host=foo,datacenter=us-east usage_idle=99,usage_busy=1
-cpu,cpu=cpu3,host=foo,datacenter=us-east usage_idle=99,usage_busy=1
-cpu,cpu=cpu4,host=foo,datacenter=us-east usage_idle=99,usage_busy=1
-cpu,cpu=cpu5,host=foo,datacenter=us-east usage_idle=99,usage_busy=1
-cpu,cpu=cpu6,host=foo,datacenter=us-east usage_idle=99,usage_busy=1
+### Example:
+
+This script produces static values, since no timestamp is specified the values are at the current time.
+```sh
+#!/bin/sh
+echo 'example,tag1=a,tag2=b i=42i,j=43i,k=44i'
 ```
 
-You will get data in InfluxDB exactly as it is defined above,
-tags are cpu=cpuN, host=foo, and datacenter=us-east with fields usage_idle
-and usage_busy. They will receive a timestamp at collection time.
-Each line must end in \n, just as the Influx line protocol does.
-
-
-### Example 3 - Graphite
-
-We can also change the data_format to "graphite" to use the metrics collecting scripts such as (compatible with graphite):
-
-* Nagios [Metrics Plugins](https://exchange.nagios.org/directory/Plugins)
-* Sensu [Metrics Plugins](https://github.com/sensu-plugins)
-
-In this example a script called /tmp/test.sh and a script called /tmp/test2.sh are configured for [[inputs.exec]] in graphite format.
-
-#### Configuration
-
+It can be paired with the following configuration and will be ran at the `interval` of the agent.
 ```toml
-# Read flattened metrics from one or more commands that output JSON to stdout
 [[inputs.exec]]
-  # Shell/commands array
-  commands = ["/tmp/test.sh","/tmp/test2.sh"]
-
-  ## Timeout for each command to complete.
+  commands = ["sh /tmp/test.sh"]
   timeout = "5s"
-
-  # Data format to consume.
-  # NOTE json only reads numerical measurements, strings and booleans are ignored.
-  data_format = "graphite"
-
-  # measurement name suffix (for separating different commands)
-  name_suffix = "_mycollector"
-
-  ## Below configuration will be used for data_format = "graphite", can be ignored for other data_format
-  ## If matching multiple measurement files, this string will be used to join the matched values.
-  separator = "."
-
-  ## Each template line requires a template pattern.  It can have an optional
-  ## filter before the template and separated by spaces.  It can also have optional extra
-  ## tags following the template.  Multiple tags should be separated by commas and no spaces
-  ## similar to the line protocol format.  The can be only one default template.
-  ## Templates support below format:
-  ## 1. filter + template
-  ## 2. filter + template + extra tag
-  ## 3. filter + template with field key
-  ## 4. default template
-  templates = [
-    "*.app env.service.resource.measurement",
-    "stats.* .host.measurement* region=us-west,agent=sensu",
-    "stats2.* .host.measurement.field",
-    "measurement*"
-  ]
-```
-Graphite messages are in this format:
-
-```
-metric_path value timestamp\n
+  data_format = "influx"
 ```
 
-__metric_path__ is the metric namespace that you want to populate.
+### Common Issues:
 
-__value__ is the value that you want to assign to the metric at this time.
+#### Q: My script works when I run it by hand, but not when Telegraf is running as a service.
 
-__timestamp__ is the unix epoch time.
-
-And test.sh/test2.sh will output:
-
-```
-sensu.metric.net.server0.eth0.rx_packets 461295119435 1444234982
-sensu.metric.net.server0.eth0.tx_bytes 1093086493388480 1444234982
-sensu.metric.net.server0.eth0.rx_bytes 1015633926034834 1444234982
-sensu.metric.net.server0.eth0.tx_errors 0 1444234982
-sensu.metric.net.server0.eth0.rx_errors 0 1444234982
-sensu.metric.net.server0.eth0.tx_dropped 0 1444234982
-sensu.metric.net.server0.eth0.rx_dropped 0 1444234982
-```
-
-The templates configuration will be used to parse the graphite metrics to support influxdb/opentsdb tagging store engines.
-
-More detail information about templates, please refer to [The graphite Input](https://github.com/influxdata/influxdb/blob/master/services/graphite/README.md)
+This may be related to the Telegraf service running as a different user.  The
+official packages run Telegraf as the `telegraf` user and group on Linux
+systems.

plugins/inputs/haproxy/README.md

@@ -1,11 +1,14 @@
-# HAproxy Input Plugin
+# HAProxy Input Plugin
 
-[HAproxy](http://www.haproxy.org/) input plugin gathers metrics directly from any running HAproxy instance. It can do so by using CSV generated by HAproxy status page or from admin socket(s).
+The [HAProxy](http://www.haproxy.org/) input plugin gathers
+[statistics](https://cbonte.github.io/haproxy-dconv/1.9/intro.html#3.3.16)
+using the [stats socket](https://cbonte.github.io/haproxy-dconv/1.9/management.html#9.3)
+or [HTTP statistics page](https://cbonte.github.io/haproxy-dconv/1.9/management.html#9) of a HAProxy server.
 
 ### Configuration:
 
 ```toml
-# SampleConfig
+# Read metrics of HAProxy, via socket or HTTP stats page
 [[inputs.haproxy]]
   ## An array of address to gather stats about. Specify an ip on hostname
   ## with optional port. ie localhost, 10.10.3.33:1936, etc.
@@ -23,7 +26,7 @@
   ## By default, some of the fields are renamed from what haproxy calls them.
   ## Setting this option to true results in the plugin keeping the original
   ## field names.
-  # keep_field_names = true
+  # keep_field_names = false
 
   ## Optional SSL Config
   # ssl_ca = "/etc/telegraf/ca.pem"
@@ -33,34 +36,77 @@
   # insecure_skip_verify = false
 ```
 
-#### `servers`
-Server addresses need to explicitly start with 'http' if you wish to use HAproxy status page. Otherwise, address will be assumed to be an UNIX socket and protocol (if present) will be discarded.
+#### HAProxy Configuration
 
-For basic authentication you need to add username and password in the URL: `http://user:password@1.2.3.4/haproxy?stats`.
+The following information may be useful when getting started, but please
+consult the HAProxy documentation for complete and up to date instructions.
 
-Following examples will all resolve to the same socket:
+The [`stats enable`](https://cbonte.github.io/haproxy-dconv/1.8/configuration.html#4-stats%20enable)
+option can be used to add unauthenticated access over HTTP using the default
+settings.  To enable the unix socket begin by reading about the
+[`stats socket`](https://cbonte.github.io/haproxy-dconv/1.8/configuration.html#3.1-stats%20socket)
+option.
+
+
+#### servers
+
+Server addresses must explicitly start with 'http' if you wish to use HAProxy
+status page.  Otherwise, addresses will be assumed to be an UNIX socket and
+any protocol (if present) will be discarded.
+
+When using socket names, wildcard expansion is supported so plugin can gather
+stats from multiple sockets at once.
+
+To use HTTP Basic Auth add the username and password in the userinfo section
+of the URL: `http://user:password@1.2.3.4/haproxy?stats`.  The credentials are
+sent via the `Authorization` header and not using the request URL.
+
+
+#### keep_field_names
+
+By default, some of the fields are renamed from what haproxy calls them.
+Setting the `keep_field_names` parameter to `true` will result in the plugin
+keeping the original field names.
+
+The following renames are made:
+- `pxname` -> `proxy`
+- `svname` -> `sv`
+- `act` -> `active_servers`
+- `bck` -> `backup_servers`
+- `cli_abrt` -> `cli_abort`
+- `srv_abrt` -> `srv_abort`
+- `hrsp_1xx` -> `http_response.1xx`
+- `hrsp_2xx` -> `http_response.2xx`
+- `hrsp_3xx` -> `http_response.3xx`
+- `hrsp_4xx` -> `http_response.4xx`
+- `hrsp_5xx` -> `http_response.5xx`
+- `hrsp_other` -> `http_response.other`
+
+### Metrics:
+
+For more details about collected metrics reference the [HAProxy CSV format
+documentation](https://cbonte.github.io/haproxy-dconv/1.8/management.html#9.1).
+
+- haproxy
+  - tags:
+    - `server` - address of the server data was gathered from
+    - `proxy` - proxy name
+    - `sv` - service name
+    - `type` - proxy session type
+  - fields:
+    - `status` (string)
+    - `check_status` (string)
+    - `last_chk` (string)
+    - `mode` (string)
+    - `tracked` (string)
+    - `agent_status` (string)
+    - `last_agt` (string)
+    - `addr` (string)
+    - `cookie` (string)
+    - `lastsess` (int)
+    - **all other stats** (int)
+
+### Example Output:
 ```
-socket:/var/run/haproxy.sock
-unix:/var/run/haproxy.sock
-foo:/var/run/haproxy.sock
-/var/run/haproxy.sock
+haproxy,server=/run/haproxy/admin.sock,proxy=public,sv=FRONTEND,type=frontend http_response.other=0i,req_rate_max=1i,comp_byp=0i,status="OPEN",rate_lim=0i,dses=0i,req_rate=0i,comp_rsp=0i,bout=9287i,comp_in=0i,mode="http",smax=1i,slim=2000i,http_response.1xx=0i,conn_rate=0i,dreq=0i,ereq=0i,iid=2i,rate_max=1i,http_response.2xx=1i,comp_out=0i,intercepted=1i,stot=2i,pid=1i,http_response.5xx=1i,http_response.3xx=0i,http_response.4xx=0i,conn_rate_max=1i,conn_tot=2i,dcon=0i,bin=294i,rate=0i,sid=0i,req_tot=2i,scur=0i,dresp=0i 1513293519000000000
 ```
-
-When using socket names, wildcard expansion is supported so plugin can gather stats from multiple sockets at once.
-
-If no servers are specified, then the default address of `http://127.0.0.1:1936/haproxy?stats` will be used.
-
-#### `keep_field_names`
-By default, some of the fields are renamed from what haproxy calls them. Setting the `keep_field_names` parameter to `true` will result in the plugin keeping the original field names.
-
-### Measurements & Fields:
-
-Plugin will gather measurements outlined in [HAproxy CSV format documentation](https://cbonte.github.io/haproxy-dconv/1.7/management.html#9.1).
-
-### Tags:
-
-- All measurements have the following tags:
-    - server - address of server data is gathered from
-    - proxy - proxy name as reported in `pxname`
-	- sv - service name as reported in `svname`
-

plugins/inputs/haproxy/haproxy.go

@@ -54,7 +54,7 @@ var sampleConfig = `
   ## By default, some of the fields are renamed from what haproxy calls them.
   ## Setting this option to true results in the plugin keeping the original
   ## field names.
-  # keep_field_names = true
+  # keep_field_names = false
 
   ## Optional SSL Config
   # ssl_ca = "/etc/telegraf/ca.pem"

plugins/inputs/http_listener/http_listener_test.go

@@ -114,7 +114,7 @@ var (
 
 func newTestHTTPListener() *HTTPListener {
 	listener := &HTTPListener{
-		ServiceAddress: ":0",
+		ServiceAddress: "localhost:0",
 	}
 	return listener
 }
@@ -155,7 +155,7 @@ func newTestHTTPSListener() *HTTPListener {
 	})
 
 	listener := &HTTPListener{
-		ServiceAddress:    ":0",
+		ServiceAddress:    "localhost:0",
 		TlsAllowedCacerts: allowedCAFiles,
 		TlsCert:           serviceCertFile,
 		TlsKey:            serviceKeyFile,
@@ -309,7 +309,7 @@ func TestWriteHTTPNoNewline(t *testing.T) {
 
 func TestWriteHTTPMaxLineSizeIncrease(t *testing.T) {
 	listener := &HTTPListener{
-		ServiceAddress: ":0",
+		ServiceAddress: "localhost:0",
 		MaxLineSize:    128 * 1000,
 	}
 
@@ -326,7 +326,7 @@ func TestWriteHTTPMaxLineSizeIncrease(t *testing.T) {
 
 func TestWriteHTTPVerySmallMaxBody(t *testing.T) {
 	listener := &HTTPListener{
-		ServiceAddress: ":0",
+		ServiceAddress: "localhost:0",
 		MaxBodySize:    4096,
 	}
 
@@ -342,7 +342,7 @@ func TestWriteHTTPVerySmallMaxBody(t *testing.T) {
 
 func TestWriteHTTPVerySmallMaxLineSize(t *testing.T) {
 	listener := &HTTPListener{
-		ServiceAddress: ":0",
+		ServiceAddress: "localhost:0",
 		MaxLineSize:    70,
 	}
 
@@ -368,7 +368,7 @@ func TestWriteHTTPVerySmallMaxLineSize(t *testing.T) {
 
 func TestWriteHTTPLargeLinesSkipped(t *testing.T) {
 	listener := &HTTPListener{
-		ServiceAddress: ":0",
+		ServiceAddress: "localhost:0",
 		MaxLineSize:    100,
 	}
 

plugins/inputs/httpjson/README.md

@@ -32,6 +32,13 @@ The httpjson plugin collects data from HTTP URLs which respond with JSON.  It fl
   #   "my_tag_2"
   # ]
 
+  ## Optional SSL Config
+  # ssl_ca = "/etc/telegraf/ca.pem"
+  # ssl_cert = "/etc/telegraf/cert.pem"
+  # ssl_key = "/etc/telegraf/key.pem"
+  ## Use SSL but skip chain & host verification
+  # insecure_skip_verify = false
+
   ## HTTP Request Parameters (all values must be strings).  For "GET" requests, data
   ## will be included in the query.  For "POST" requests, data will be included
   ## in the request body as "x-www-form-urlencoded".
@@ -43,13 +50,6 @@ The httpjson plugin collects data from HTTP URLs which respond with JSON.  It fl
   # [inputs.httpjson.headers]
   #   X-Auth-Token = "my-xauth-token"
   #   apiVersion = "v1"
-
-  ## Optional SSL Config
-  # ssl_ca = "/etc/telegraf/ca.pem"
-  # ssl_cert = "/etc/telegraf/cert.pem"
-  # ssl_key = "/etc/telegraf/key.pem"
-  ## Use SSL but skip chain & host verification
-  # insecure_skip_verify = false
 ```
 
 ### Measurements & Fields:

plugins/inputs/httpjson/httpjson.go

@@ -100,6 +100,13 @@ var sampleConfig = `
   #   "my_tag_2"
   # ]
 
+  ## Optional SSL Config
+  # ssl_ca = "/etc/telegraf/ca.pem"
+  # ssl_cert = "/etc/telegraf/cert.pem"
+  # ssl_key = "/etc/telegraf/key.pem"
+  ## Use SSL but skip chain & host verification
+  # insecure_skip_verify = false
+
   ## HTTP parameters (all values must be strings).  For "GET" requests, data
   ## will be included in the query.  For "POST" requests, data will be included
   ## in the request body as "x-www-form-urlencoded".
@@ -111,13 +118,6 @@ var sampleConfig = `
   # [inputs.httpjson.headers]
   #   X-Auth-Token = "my-xauth-token"
   #   apiVersion = "v1"
-
-  ## Optional SSL Config
-  # ssl_ca = "/etc/telegraf/ca.pem"
-  # ssl_cert = "/etc/telegraf/cert.pem"
-  # ssl_key = "/etc/telegraf/key.pem"
-  ## Use SSL but skip chain & host verification
-  # insecure_skip_verify = false
 `
 
 func (h *HttpJson) SampleConfig() string {

plugins/inputs/ipmi_sensor/README.md

@@ -22,7 +22,10 @@ ipmitool -I lan -H SERVER -U USERID -P PASSW0RD sdr
 [[inputs.ipmi_sensor]]
   ## optionally specify the path to the ipmitool executable
   # path = "/usr/bin/ipmitool"
-  #
+  ##
+  ## optionally force session privilege level. Can be CALLBACK, USER, OPERATOR, ADMINISTRATOR
+  # privilege = "ADMINISTRATOR"
+  ##
   ## optionally specify one or more servers via a url matching
   ##  [username[:password]@][protocol[(address)]]
   ##  e.g.

plugins/inputs/ipmi_sensor/connection.go

@@ -14,10 +14,12 @@ type Connection struct {
 	Password  string
 	Port      int
 	Interface string
+	Privilege string
 }
 
-func NewConnection(server string) *Connection {
+func NewConnection(server string, privilege string) *Connection {
 	conn := &Connection{}
+	conn.Privilege = privilege
 	inx1 := strings.LastIndex(server, "@")
 	inx2 := strings.Index(server, "(")
 	inx3 := strings.Index(server, ")")
@@ -59,7 +61,9 @@ func (t *Connection) options() []string {
 	if t.Port != 0 {
 		options = append(options, "-p", strconv.Itoa(t.Port))
 	}
-
+	if t.Privilege != "" {
+		options = append(options, "-L", t.Privilege)
+	}
 	return options
 }
 

plugins/inputs/ipmi_sensor/connection_test.go

@@ -23,6 +23,7 @@ func TestNewConnection(t *testing.T) {
 				Username:  "USERID",
 				Password:  "PASSW0RD",
 				Interface: "lan",
+				Privilege: "USER",
 			},
 		},
 		{
@@ -32,11 +33,12 @@ func TestNewConnection(t *testing.T) {
 				Username:  "USERID",
 				Password:  "PASS:!@#$%^&*(234)_+W0RD",
 				Interface: "lan",
+				Privilege: "USER",
 			},
 		},
 	}
 
 	for _, v := range testData {
-		assert.Equal(t, v.con, NewConnection(v.addr))
+		assert.Equal(t, v.con, NewConnection(v.addr, "USER"))
 	}
 }

plugins/inputs/ipmi_sensor/ipmi.go

@@ -17,15 +17,19 @@ var (
 )
 
 type Ipmi struct {
-	Path    string
-	Servers []string
-	Timeout internal.Duration
+	Path      string
+	Privilege string
+	Servers   []string
+	Timeout   internal.Duration
 }
 
 var sampleConfig = `
   ## optionally specify the path to the ipmitool executable
   # path = "/usr/bin/ipmitool"
-  #
+  ##
+  ## optionally force session privilege level. Can be CALLBACK, USER, OPERATOR, ADMINISTRATOR
+  # privilege = "ADMINISTRATOR"
+  ##
   ## optionally specify one or more servers via a url matching
   ##  [username[:password]@][protocol[(address)]]
   ##  e.g.
@@ -77,13 +81,11 @@ func (m *Ipmi) Gather(acc telegraf.Accumulator) error {
 func (m *Ipmi) parse(acc telegraf.Accumulator, server string) error {
 	opts := make([]string, 0)
 	hostname := ""
-
 	if server != "" {
-		conn := NewConnection(server)
+		conn := NewConnection(server, m.Privilege)
 		hostname = conn.Hostname
 		opts = conn.options()
 	}
-
 	opts = append(opts, "sdr")
 	cmd := execCommand(m.Path, opts...)
 	out, err := internal.CombinedOutputTimeout(cmd, m.Timeout.Duration)

plugins/inputs/ipmi_sensor/ipmi_test.go

@@ -15,9 +15,10 @@ import (
 
 func TestGather(t *testing.T) {
 	i := &Ipmi{
-		Servers: []string{"USERID:PASSW0RD@lan(192.168.1.1)"},
-		Path:    "ipmitool",
-		Timeout: internal.Duration{Duration: time.Second * 5},
+		Servers:   []string{"USERID:PASSW0RD@lan(192.168.1.1)"},
+		Path:      "ipmitool",
+		Privilege: "USER",
+		Timeout:   internal.Duration{Duration: time.Second * 5},
 	}
 	// overwriting exec commands with mock commands
 	execCommand = fakeExecCommand
@@ -29,7 +30,7 @@ func TestGather(t *testing.T) {
 
 	assert.Equal(t, acc.NFields(), 266, "non-numeric measurements should be ignored")
 
-	conn := NewConnection(i.Servers[0])
+	conn := NewConnection(i.Servers[0], i.Privilege)
 	assert.Equal(t, "USERID", conn.Username)
 	assert.Equal(t, "lan", conn.Interface)
 

plugins/inputs/ipset/README.md

@@ -0,0 +1,62 @@
+# Ipset Plugin
+
+The ipset plugin gathers packets and bytes counters from Linux ipset.
+It uses the output of the command "ipset save".
+Ipsets created without the "counters" option are ignored.
+
+Results are tagged with:
+- ipset name
+- ipset entry
+
+There are 3 ways to grant telegraf the right to run ipset:
+* Run as root (strongly discouraged)
+* Use sudo
+* Configure systemd to run telegraf with CAP_NET_ADMIN and CAP_NET_RAW capabilities.
+
+### Using systemd capabilities
+
+You may run `systemctl edit telegraf.service` and add the following:
+
+```
+[Service]
+CapabilityBoundingSet=CAP_NET_RAW CAP_NET_ADMIN
+AmbientCapabilities=CAP_NET_RAW CAP_NET_ADMIN
+```
+
+### Using sudo
+
+You may edit your sudo configuration with the following:
+
+```sudo
+telegraf ALL=(root) NOPASSWD: /sbin/ipset save
+```
+
+### Configuration
+
+```toml
+  [[inputs.ipset]]
+    ## By default, we only show sets which have already matched at least 1 packet.
+    ## set include_unmatched_sets = true to gather them all.
+    include_unmatched_sets = false
+    ## Adjust your sudo settings appropriately if using this option ("sudo ipset save")
+    ## You can avoid using sudo or root, by setting appropriate privileges for
+    ## the telegraf.service systemd service.
+    use_sudo = false
+    ## The default timeout of 1s for ipset execution can be overridden here:
+    # timeout = "1s"
+
+```
+
+### Example Output
+
+```
+$ sudo ipset save
+create myset hash:net family inet hashsize 1024 maxelem 65536 counters comment
+add myset 10.69.152.1 packets 8 bytes 672 comment "machine A"
+```
+
+```
+$ telegraf --config telegraf.conf --input-filter ipset --test --debug
+* Plugin: inputs.ipset, Collection 1
+> ipset,rule=10.69.152.1,host=trashme,set=myset bytes_total=8i,packets_total=672i 1507615028000000000
+```

plugins/inputs/ipset/ipset.go

@@ -0,0 +1,126 @@
+package ipset
+
+import (
+	"bufio"
+	"bytes"
+	"fmt"
+	"os/exec"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/influxdata/telegraf"
+	"github.com/influxdata/telegraf/internal"
+	"github.com/influxdata/telegraf/plugins/inputs"
+)
+
+// Ipsets is a telegraf plugin to gather packets and bytes counters from ipset
+type Ipset struct {
+	IncludeUnmatchedSets bool
+	UseSudo              bool
+	Timeout              internal.Duration
+	lister               setLister
+}
+
+type setLister func(Timeout internal.Duration, UseSudo bool) (*bytes.Buffer, error)
+
+const measurement = "ipset"
+
+var defaultTimeout = internal.Duration{Duration: time.Second}
+
+// Description returns a short description of the plugin
+func (ipset *Ipset) Description() string {
+	return "Gather packets and bytes counters from Linux ipsets"
+}
+
+// SampleConfig returns sample configuration options.
+func (ipset *Ipset) SampleConfig() string {
+	return `
+  ## By default, we only show sets which have already matched at least 1 packet.
+  ## set include_unmatched_sets = true to gather them all.
+  include_unmatched_sets = false
+  ## Adjust your sudo settings appropriately if using this option ("sudo ipset save")
+  use_sudo = false
+  ## The default timeout of 1s for ipset execution can be overridden here:
+  # timeout = "1s"
+`
+}
+
+func (ips *Ipset) Gather(acc telegraf.Accumulator) error {
+	out, e := ips.lister(ips.Timeout, ips.UseSudo)
+	if e != nil {
+		acc.AddError(e)
+	}
+
+	scanner := bufio.NewScanner(out)
+	for scanner.Scan() {
+		line := scanner.Text()
+		// Ignore sets created without the "counters" option
+		nocomment := strings.Split(line, "\"")[0]
+		if !(strings.Contains(nocomment, "packets") &&
+			strings.Contains(nocomment, "bytes")) {
+			continue
+		}
+
+		data := strings.Fields(line)
+		if len(data) < 7 {
+			acc.AddError(fmt.Errorf("Error parsing line (expected at least 7 fields): %s", line))
+			continue
+		}
+		if data[0] == "add" && (data[4] != "0" || ips.IncludeUnmatchedSets) {
+			tags := map[string]string{
+				"set":  data[1],
+				"rule": data[2],
+			}
+			packets_total, err := strconv.ParseUint(data[4], 10, 64)
+			if err != nil {
+				acc.AddError(err)
+			}
+			bytes_total, err := strconv.ParseUint(data[6], 10, 64)
+			if err != nil {
+				acc.AddError(err)
+			}
+			fields := map[string]interface{}{
+				"packets_total": packets_total,
+				"bytes_total":   bytes_total,
+			}
+			acc.AddCounter(measurement, fields, tags)
+		}
+	}
+	return nil
+}
+
+func setList(Timeout internal.Duration, UseSudo bool) (*bytes.Buffer, error) {
+	// Is ipset installed ?
+	ipsetPath, err := exec.LookPath("ipset")
+	if err != nil {
+		return nil, err
+	}
+	var args []string
+	cmdName := ipsetPath
+	if UseSudo {
+		cmdName = "sudo"
+		args = append(args, ipsetPath)
+	}
+	args = append(args, "save")
+
+	cmd := exec.Command(cmdName, args...)
+
+	var out bytes.Buffer
+	cmd.Stdout = &out
+	err = internal.RunTimeout(cmd, Timeout.Duration)
+	if err != nil {
+		return &out, fmt.Errorf("error running ipset save: %s", err)
+	}
+
+	return &out, nil
+}
+
+func init() {
+	inputs.Add("ipset", func() telegraf.Input {
+		return &Ipset{
+			lister:  setList,
+			Timeout: defaultTimeout,
+		}
+	})
+}

plugins/inputs/ipset/ipset_test.go

@@ -0,0 +1,135 @@
+package ipset
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"reflect"
+	"testing"
+
+	"github.com/influxdata/telegraf/internal"
+	"github.com/influxdata/telegraf/testutil"
+)
+
+func TestIpset(t *testing.T) {
+	tests := []struct {
+		name   string
+		value  string
+		tags   []map[string]string
+		fields [][]map[string]interface{}
+		err    error
+	}{
+		{
+			name:  "0 sets, no results",
+			value: "",
+		},
+		{
+			name: "Empty sets, no values",
+			value: `create myset hash:net family inet hashsize 1024 maxelem 65536
+				create myset2 hash:net,port family inet hashsize 16384 maxelem 524288 counters comment
+				`,
+		},
+		{
+			name: "Non-empty sets, but no counters, no results",
+			value: `create myset hash:net family inet hashsize 1024 maxelem 65536
+				add myset 1.2.3.4
+				`,
+		},
+		{
+			name: "Line with data but not enough fields",
+			value: `create hash:net family inet hashsize 1024 maxelem 65536 counters
+				add myset 4.5.6.7 packets 123 bytes
+				`,
+			err: fmt.Errorf("Error parsing line (expected at least 7 fields): \t\t\t\tadd myset 4.5.6.7 packets 123 bytes"),
+		},
+		{
+			name: "Non-empty sets, counters, no comment",
+			value: `create myset hash:net family inet hashsize 1024 maxelem 65536 counters
+				add myset 1.2.3.4 packets 1328 bytes 79680
+				add myset 2.3.4.5 packets 0 bytes 0
+				add myset 3.4.5.6 packets 3 bytes 222
+				`,
+			tags: []map[string]string{
+				map[string]string{"set": "myset", "rule": "1.2.3.4"},
+				map[string]string{"set": "myset", "rule": "3.4.5.6"},
+			},
+			fields: [][]map[string]interface{}{
+				{map[string]interface{}{"packets_total": uint64(1328), "bytes_total": uint64(79680)}},
+				{map[string]interface{}{"packets_total": uint64(3), "bytes_total": uint64(222)}},
+			},
+		},
+		{
+			name: "Sets with counters and comment",
+			value: `create myset hash:net family inet hashsize 1024 maxelem 65536 counters comment
+				add myset 1.2.3.4 packets 1328 bytes 79680 comment "first IP"
+				add myset 2.3.4.5 packets 0 bytes 0 comment "2nd IP"
+				add myset 3.4.5.6 packets 3 bytes 222 "3rd IP"
+				`,
+			tags: []map[string]string{
+				map[string]string{"set": "myset", "rule": "1.2.3.4"},
+				map[string]string{"set": "myset", "rule": "3.4.5.6"},
+			},
+			fields: [][]map[string]interface{}{
+				{map[string]interface{}{"packets_total": uint64(1328), "bytes_total": uint64(79680)}},
+				{map[string]interface{}{"packets_total": uint64(3), "bytes_total": uint64(222)}},
+			},
+		},
+	}
+
+	for i, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			i++
+			ips := &Ipset{
+				lister: func(Timeout internal.Duration, UseSudo bool) (*bytes.Buffer, error) {
+					return bytes.NewBufferString(tt.value), nil
+				},
+			}
+			acc := new(testutil.Accumulator)
+			err := acc.GatherError(ips.Gather)
+			if !reflect.DeepEqual(tt.err, err) {
+				t.Errorf("%d: expected error '%#v' got '%#v'", i, tt.err, err)
+			}
+			if len(tt.tags) == 0 {
+				n := acc.NFields()
+				if n != 0 {
+					t.Errorf("%d: expected 0 values got %d", i, n)
+				}
+				return
+			}
+			n := 0
+			for j, tags := range tt.tags {
+				for k, fields := range tt.fields[j] {
+					if len(acc.Metrics) < n+1 {
+						t.Errorf("%d: expected at least %d values got %d", i, n+1, len(acc.Metrics))
+						break
+					}
+					m := acc.Metrics[n]
+					if !reflect.DeepEqual(m.Measurement, measurement) {
+						t.Errorf("%d %d %d: expected measurement '%#v' got '%#v'\n", i, j, k, measurement, m.Measurement)
+					}
+					if !reflect.DeepEqual(m.Tags, tags) {
+						t.Errorf("%d %d %d: expected tags\n%#v got\n%#v\n", i, j, k, tags, m.Tags)
+					}
+					if !reflect.DeepEqual(m.Fields, fields) {
+						t.Errorf("%d %d %d: expected fields\n%#v got\n%#v\n", i, j, k, fields, m.Fields)
+					}
+					n++
+				}
+			}
+		})
+	}
+}
+
+func TestIpset_Gather_listerError(t *testing.T) {
+	errFoo := errors.New("error foobar")
+	ips := &Ipset{
+		lister: func(Timeout internal.Duration, UseSudo bool) (*bytes.Buffer, error) {
+			return new(bytes.Buffer), errFoo
+		},
+	}
+	acc := new(testutil.Accumulator)
+	err := acc.GatherError(ips.Gather)
+	if !reflect.DeepEqual(err, errFoo) {
+		t.Errorf("Expected error %#v got\n%#v\n", errFoo, err)
+	}
+}

plugins/inputs/jolokia2/jolokia_agent.go

@@ -46,7 +46,7 @@ func (ja *JolokiaAgent) SampleConfig() string {
   # insecure_skip_verify = false
 
   ## Add metrics to read
-  [[inputs.jolokia2.metric]]
+  [[inputs.jolokia2_agent.metric]]
     name  = "java_runtime"
     mbean = "java.lang:type=Runtime"
     paths = ["Uptime"]

plugins/inputs/mesos/README.md

@@ -11,7 +11,7 @@ For more information, please check the [Mesos Observability Metrics](http://meso
   ## Timeout, in ms.
   timeout = 100
   ## A list of Mesos masters.
-  masters = ["localhost:5050"]
+  masters = ["http://localhost:5050"]
   ## Master metrics groups to be collected, by default, all enabled.
   master_collections = [
     "resources",
@@ -35,6 +35,13 @@ For more information, please check the [Mesos Observability Metrics](http://meso
   #   "tasks",
   #   "messages",
   # ]
+
+  ## Optional SSL Config
+  # ssl_ca = "/etc/telegraf/ca.pem"
+  # ssl_cert = "/etc/telegraf/cert.pem"
+  # ssl_key = "/etc/telegraf/key.pem"
+  ## Use SSL but skip chain & host verification
+  # insecure_skip_verify = false
 ```
 
 By default this plugin is not configured to gather metrics from mesos. Since a mesos cluster can be deployed in numerous ways it does not provide any default
@@ -235,7 +242,8 @@ Mesos slave metric groups
 ### Tags:
 
 - All master/slave measurements have the following tags:
-    - server
+    - server (network location of server: `host:port`)
+    - url (URL origin of server: `scheme://host:port`)
     - role (master/slave)
 
 - All master measurements have the extra tags:

plugins/inputs/mesos/mesos.go

@@ -7,11 +7,14 @@ import (
 	"log"
 	"net"
 	"net/http"
+	"net/url"
 	"strconv"
+	"strings"
 	"sync"
 	"time"
 
 	"github.com/influxdata/telegraf"
+	"github.com/influxdata/telegraf/internal"
 	"github.com/influxdata/telegraf/plugins/inputs"
 	jsonparser "github.com/influxdata/telegraf/plugins/parsers/json"
 )
@@ -30,6 +33,20 @@ type Mesos struct {
 	Slaves     []string
 	SlaveCols  []string `toml:"slave_collections"`
 	//SlaveTasks bool
+
+	// Path to CA file
+	SSLCA string `toml:"ssl_ca"`
+	// Path to host cert file
+	SSLCert string `toml:"ssl_cert"`
+	// Path to cert key file
+	SSLKey string `toml:"ssl_key"`
+	// Use SSL but skip chain & host verification
+	InsecureSkipVerify bool
+
+	initialized bool
+	client      *http.Client
+	masterURLs  []*url.URL
+	slaveURLs   []*url.URL
 }
 
 var allMetrics = map[Role][]string{
@@ -41,7 +58,7 @@ var sampleConfig = `
   ## Timeout, in ms.
   timeout = 100
   ## A list of Mesos masters.
-  masters = ["localhost:5050"]
+  masters = ["http://localhost:5050"]
   ## Master metrics groups to be collected, by default, all enabled.
   master_collections = [
     "resources",
@@ -65,6 +82,13 @@ var sampleConfig = `
   #   "tasks",
   #   "messages",
   # ]
+
+  ## Optional SSL Config
+  # ssl_ca = "/etc/telegraf/ca.pem"
+  # ssl_cert = "/etc/telegraf/cert.pem"
+  # ssl_key = "/etc/telegraf/key.pem"
+  ## Use SSL but skip chain & host verification
+  # insecure_skip_verify = false
 `
 
 // SampleConfig returns a sample configuration block
@@ -77,7 +101,28 @@ func (m *Mesos) Description() string {
 	return "Telegraf plugin for gathering metrics from N Mesos masters"
 }
 
-func (m *Mesos) SetDefaults() {
+func parseURL(s string, role Role) (*url.URL, error) {
+	if !strings.HasPrefix(s, "http://") && !strings.HasPrefix(s, "https://") {
+		host, port, err := net.SplitHostPort(s)
+		// no port specified
+		if err != nil {
+			host = s
+			switch role {
+			case MASTER:
+				port = "5050"
+			case SLAVE:
+				port = "5051"
+			}
+		}
+
+		s = "http://" + host + ":" + port
+		log.Printf("W! [inputs.mesos] Using %q as connection URL; please update your configuration to use an URL", s)
+	}
+
+	return url.Parse(s)
+}
+
+func (m *Mesos) initialize() error {
 	if len(m.MasterCols) == 0 {
 		m.MasterCols = allMetrics[MASTER]
 	}
@@ -87,33 +132,71 @@ func (m *Mesos) SetDefaults() {
 	}
 
 	if m.Timeout == 0 {
-		log.Println("I! [mesos] Missing timeout value, setting default value (100ms)")
+		log.Println("I! [inputs.mesos] Missing timeout value, setting default value (100ms)")
 		m.Timeout = 100
 	}
+
+	rawQuery := "timeout=" + strconv.Itoa(m.Timeout) + "ms"
+
+	m.masterURLs = make([]*url.URL, 0, len(m.Masters))
+	for _, master := range m.Masters {
+		u, err := parseURL(master, MASTER)
+		if err != nil {
+			return err
+		}
+
+		u.RawQuery = rawQuery
+		m.masterURLs = append(m.masterURLs, u)
+	}
+
+	m.slaveURLs = make([]*url.URL, 0, len(m.Slaves))
+	for _, slave := range m.Slaves {
+		u, err := parseURL(slave, SLAVE)
+		if err != nil {
+			return err
+		}
+
+		u.RawQuery = rawQuery
+		m.slaveURLs = append(m.slaveURLs, u)
+	}
+
+	client, err := m.createHttpClient()
+	if err != nil {
+		return err
+	}
+	m.client = client
+
+	return nil
 }
 
 // Gather() metrics from given list of Mesos Masters
 func (m *Mesos) Gather(acc telegraf.Accumulator) error {
-	var wg sync.WaitGroup
-
-	m.SetDefaults()
-
-	for _, v := range m.Masters {
-		wg.Add(1)
-		go func(c string) {
-			acc.AddError(m.gatherMainMetrics(c, ":5050", MASTER, acc))
-			wg.Done()
-			return
-		}(v)
+	if !m.initialized {
+		err := m.initialize()
+		if err != nil {
+			return err
+		}
+		m.initialized = true
 	}
 
-	for _, v := range m.Slaves {
+	var wg sync.WaitGroup
+
+	for _, master := range m.masterURLs {
 		wg.Add(1)
-		go func(c string) {
-			acc.AddError(m.gatherMainMetrics(c, ":5051", SLAVE, acc))
+		go func(master *url.URL) {
+			acc.AddError(m.gatherMainMetrics(master, MASTER, acc))
 			wg.Done()
 			return
-		}(v)
+		}(master)
+	}
+
+	for _, slave := range m.slaveURLs {
+		wg.Add(1)
+		go func(slave *url.URL) {
+			acc.AddError(m.gatherMainMetrics(slave, SLAVE, acc))
+			wg.Done()
+			return
+		}(slave)
 
 		// if !m.SlaveTasks {
 		// 	continue
@@ -121,7 +204,7 @@ func (m *Mesos) Gather(acc telegraf.Accumulator) error {
 
 		// wg.Add(1)
 		// go func(c string) {
-		// 	acc.AddError(m.gatherSlaveTaskMetrics(c, ":5051", acc))
+		// 	acc.AddError(m.gatherSlaveTaskMetrics(slave, acc))
 		// 	wg.Done()
 		// 	return
 		// }(v)
@@ -132,6 +215,24 @@ func (m *Mesos) Gather(acc telegraf.Accumulator) error {
 	return nil
 }
 
+func (m *Mesos) createHttpClient() (*http.Client, error) {
+	tlsCfg, err := internal.GetTLSConfig(
+		m.SSLCert, m.SSLKey, m.SSLCA, m.InsecureSkipVerify)
+	if err != nil {
+		return nil, err
+	}
+
+	client := &http.Client{
+		Transport: &http.Transport{
+			Proxy:           http.ProxyFromEnvironment,
+			TLSClientConfig: tlsCfg,
+		},
+		Timeout: 4 * time.Second,
+	}
+
+	return client, nil
+}
+
 // metricsDiff() returns set names for removal
 func metricsDiff(role Role, w []string) []string {
 	b := []string{}
@@ -393,15 +494,6 @@ func (m *Mesos) filterMetrics(role Role, metrics *map[string]interface{}) {
 	}
 }
 
-var tr = &http.Transport{
-	ResponseHeaderTimeout: time.Duration(3 * time.Second),
-}
-
-var client = &http.Client{
-	Transport: tr,
-	Timeout:   time.Duration(4 * time.Second),
-}
-
 // TaskStats struct for JSON API output /monitor/statistics
 type TaskStats struct {
 	ExecutorID  string                 `json:"executor_id"`
@@ -409,22 +501,15 @@ type TaskStats struct {
 	Statistics  map[string]interface{} `json:"statistics"`
 }
 
-func (m *Mesos) gatherSlaveTaskMetrics(address string, defaultPort string, acc telegraf.Accumulator) error {
+func (m *Mesos) gatherSlaveTaskMetrics(u *url.URL, acc telegraf.Accumulator) error {
 	var metrics []TaskStats
 
-	host, _, err := net.SplitHostPort(address)
-	if err != nil {
-		host = address
-		address = address + defaultPort
-	}
-
 	tags := map[string]string{
-		"server": host,
+		"server": u.Hostname(),
+		"url":    urlTag(u),
 	}
 
-	ts := strconv.Itoa(m.Timeout) + "ms"
-
-	resp, err := client.Get("http://" + address + "/monitor/statistics?timeout=" + ts)
+	resp, err := m.client.Get(withPath(u, "/monitor/statistics").String())
 
 	if err != nil {
 		return err
@@ -459,24 +544,31 @@ func (m *Mesos) gatherSlaveTaskMetrics(address string, defaultPort string, acc t
 	return nil
 }
 
+func withPath(u *url.URL, path string) *url.URL {
+	c := *u
+	c.Path = path
+	return &c
+}
+
+func urlTag(u *url.URL) string {
+	c := *u
+	c.Path = ""
+	c.User = nil
+	c.RawQuery = ""
+	return c.String()
+}
+
 // This should not belong to the object
-func (m *Mesos) gatherMainMetrics(a string, defaultPort string, role Role, acc telegraf.Accumulator) error {
+func (m *Mesos) gatherMainMetrics(u *url.URL, role Role, acc telegraf.Accumulator) error {
 	var jsonOut map[string]interface{}
 
-	host, _, err := net.SplitHostPort(a)
-	if err != nil {
-		host = a
-		a = a + defaultPort
-	}
-
 	tags := map[string]string{
-		"server": host,
+		"server": u.Hostname(),
+		"url":    urlTag(u),
 		"role":   string(role),
 	}
 
-	ts := strconv.Itoa(m.Timeout) + "ms"
-
-	resp, err := client.Get("http://" + a + "/metrics/snapshot?timeout=" + ts)
+	resp, err := m.client.Get(withPath(u, "/metrics/snapshot").String())
 
 	if err != nil {
 		return err

plugins/inputs/mesos/mesos_test.go

@@ -6,10 +6,12 @@ import (
 	"math/rand"
 	"net/http"
 	"net/http/httptest"
+	"net/url"
 	"os"
 	"testing"
 
 	"github.com/influxdata/telegraf/testutil"
+	"github.com/stretchr/testify/require"
 )
 
 var masterMetrics map[string]interface{}
@@ -378,3 +380,19 @@ func TestSlaveFilter(t *testing.T) {
 		}
 	}
 }
+
+func TestWithPathDoesNotModify(t *testing.T) {
+	u, err := url.Parse("http://localhost:5051")
+	require.NoError(t, err)
+	v := withPath(u, "/xyzzy")
+	require.Equal(t, u.String(), "http://localhost:5051")
+	require.Equal(t, v.String(), "http://localhost:5051/xyzzy")
+}
+
+func TestURLTagDoesNotModify(t *testing.T) {
+	u, err := url.Parse("http://a:b@localhost:5051?timeout=1ms")
+	require.NoError(t, err)
+	v := urlTag(u)
+	require.Equal(t, u.String(), "http://a:b@localhost:5051?timeout=1ms")
+	require.Equal(t, v, "http://localhost:5051")
+}

plugins/inputs/mysql/mysql.go

@@ -147,7 +147,7 @@ func (m *Mysql) Gather(acc telegraf.Accumulator) error {
 
 	tlsConfig, err := internal.GetTLSConfig(m.SSLCert, m.SSLKey, m.SSLCA, false)
 	if err != nil {
-		log.Printf("E! MySQL Error registering TLS config: %s", err)
+		return fmt.Errorf("registering TLS config: %s", err)
 	}
 
 	if tlsConfig != nil {
@@ -169,182 +169,6 @@ func (m *Mysql) Gather(acc telegraf.Accumulator) error {
 	return nil
 }
 
-type mapping struct {
-	onServer string
-	inExport string
-}
-
-var mappings = []*mapping{
-	{
-		onServer: "Aborted_",
-		inExport: "aborted_",
-	},
-	{
-		onServer: "Bytes_",
-		inExport: "bytes_",
-	},
-	{
-		onServer: "Com_",
-		inExport: "commands_",
-	},
-	{
-		onServer: "Created_",
-		inExport: "created_",
-	},
-	{
-		onServer: "Handler_",
-		inExport: "handler_",
-	},
-	{
-		onServer: "Innodb_",
-		inExport: "innodb_",
-	},
-	{
-		onServer: "Key_",
-		inExport: "key_",
-	},
-	{
-		onServer: "Open_",
-		inExport: "open_",
-	},
-	{
-		onServer: "Opened_",
-		inExport: "opened_",
-	},
-	{
-		onServer: "Qcache_",
-		inExport: "qcache_",
-	},
-	{
-		onServer: "Table_",
-		inExport: "table_",
-	},
-	{
-		onServer: "Tokudb_",
-		inExport: "tokudb_",
-	},
-	{
-		onServer: "Threads_",
-		inExport: "threads_",
-	},
-	{
-		onServer: "Access_",
-		inExport: "access_",
-	},
-	{
-		onServer: "Aria__",
-		inExport: "aria_",
-	},
-	{
-		onServer: "Binlog__",
-		inExport: "binlog_",
-	},
-	{
-		onServer: "Busy_",
-		inExport: "busy_",
-	},
-	{
-		onServer: "Connection_",
-		inExport: "connection_",
-	},
-	{
-		onServer: "Delayed_",
-		inExport: "delayed_",
-	},
-	{
-		onServer: "Empty_",
-		inExport: "empty_",
-	},
-	{
-		onServer: "Executed_",
-		inExport: "executed_",
-	},
-	{
-		onServer: "Executed_",
-		inExport: "executed_",
-	},
-	{
-		onServer: "Feature_",
-		inExport: "feature_",
-	},
-	{
-		onServer: "Flush_",
-		inExport: "flush_",
-	},
-	{
-		onServer: "Last_",
-		inExport: "last_",
-	},
-	{
-		onServer: "Master_",
-		inExport: "master_",
-	},
-	{
-		onServer: "Max_",
-		inExport: "max_",
-	},
-	{
-		onServer: "Memory_",
-		inExport: "memory_",
-	},
-	{
-		onServer: "Not_",
-		inExport: "not_",
-	},
-	{
-		onServer: "Performance_",
-		inExport: "performance_",
-	},
-	{
-		onServer: "Prepared_",
-		inExport: "prepared_",
-	},
-	{
-		onServer: "Rows_",
-		inExport: "rows_",
-	},
-	{
-		onServer: "Rpl_",
-		inExport: "rpl_",
-	},
-	{
-		onServer: "Select_",
-		inExport: "select_",
-	},
-	{
-		onServer: "Slave_",
-		inExport: "slave_",
-	},
-	{
-		onServer: "Slow_",
-		inExport: "slow_",
-	},
-	{
-		onServer: "Sort_",
-		inExport: "sort_",
-	},
-	{
-		onServer: "Subquery_",
-		inExport: "subquery_",
-	},
-	{
-		onServer: "Tc_",
-		inExport: "tc_",
-	},
-	{
-		onServer: "Threadpool_",
-		inExport: "threadpool_",
-	},
-	{
-		onServer: "wsrep_",
-		inExport: "wsrep_",
-	},
-	{
-		onServer: "Uptime_",
-		inExport: "uptime_",
-	},
-}
-
 var (
 	// status counter
 	generalThreadStates = map[string]uint32{
@@ -717,9 +541,8 @@ func (m *Mysql) gatherGlobalVariables(db *sql.DB, serv string, acc telegraf.Accu
 			fields[key] = string(val)
 			tags[key] = string(val)
 		}
-		// parse value, if it is numeric then save, otherwise ignore
-		if floatVal, ok := parseValue(val); ok {
-			fields[key] = floatVal
+		if value, ok := parseValue(val); ok {
+			fields[key] = value
 		}
 		// Send 20 fields at a time
 		if len(fields) >= 20 {
@@ -769,7 +592,7 @@ func (m *Mysql) gatherSlaveStatuses(db *sql.DB, serv string, acc telegraf.Accumu
 		}
 		// range over columns, and try to parse values
 		for i, col := range cols {
-			// skip unparsable values
+			col = strings.ToLower(col)
 			if value, ok := parseValue(*vals[i].(*sql.RawBytes)); ok {
 				fields["slave_"+col] = value
 			}
@@ -820,98 +643,36 @@ func (m *Mysql) gatherBinaryLogs(db *sql.DB, serv string, acc telegraf.Accumulat
 // the mappings of actual names and names of each status to be exported
 // to output is provided on mappings variable
 func (m *Mysql) gatherGlobalStatuses(db *sql.DB, serv string, acc telegraf.Accumulator) error {
-	// If user forgot the '/', add it
-	if strings.HasSuffix(serv, ")") {
-		serv = serv + "/"
-	} else if serv == "localhost" {
-		serv = ""
-	}
-
 	// run query
 	rows, err := db.Query(globalStatusQuery)
 	if err != nil {
 		return err
 	}
+	defer rows.Close()
 
 	// parse the DSN and save host name as a tag
 	servtag := getDSNTag(serv)
 	tags := map[string]string{"server": servtag}
 	fields := make(map[string]interface{})
 	for rows.Next() {
-		var name string
-		var val interface{}
+		var key string
+		var val sql.RawBytes
 
-		err = rows.Scan(&name, &val)
-		if err != nil {
+		if err = rows.Scan(&key, &val); err != nil {
 			return err
 		}
 
-		var found bool
+		key = strings.ToLower(key)
 
-		// iterate over mappings and gather metrics that is provided on mapping
-		for _, mapped := range mappings {
-			if strings.HasPrefix(name, mapped.onServer) {
-				// convert numeric values to integer
-				i, _ := strconv.Atoi(string(val.([]byte)))
-				fields[mapped.inExport+name[len(mapped.onServer):]] = i
-				found = true
-			}
+		if value, ok := parseValue(val); ok {
+			fields[key] = value
 		}
+
 		// Send 20 fields at a time
 		if len(fields) >= 20 {
 			acc.AddFields("mysql", fields, tags)
 			fields = make(map[string]interface{})
 		}
-
-		if found {
-			continue
-		}
-
-		// search for specific values
-		switch name {
-		case "Queries":
-			i, err := strconv.ParseInt(string(val.([]byte)), 10, 64)
-			if err != nil {
-				acc.AddError(fmt.Errorf("E! Error mysql: parsing %s int value (%s)", name, err))
-			} else {
-				fields["queries"] = i
-			}
-		case "Questions":
-			i, err := strconv.ParseInt(string(val.([]byte)), 10, 64)
-			if err != nil {
-				acc.AddError(fmt.Errorf("E! Error mysql: parsing %s int value (%s)", name, err))
-			} else {
-				fields["questions"] = i
-			}
-		case "Slow_queries":
-			i, err := strconv.ParseInt(string(val.([]byte)), 10, 64)
-			if err != nil {
-				acc.AddError(fmt.Errorf("E! Error mysql: parsing %s int value (%s)", name, err))
-			} else {
-				fields["slow_queries"] = i
-			}
-		case "Connections":
-			i, err := strconv.ParseInt(string(val.([]byte)), 10, 64)
-			if err != nil {
-				acc.AddError(fmt.Errorf("E! Error mysql: parsing %s int value (%s)", name, err))
-			} else {
-				fields["connections"] = i
-			}
-		case "Syncs":
-			i, err := strconv.ParseInt(string(val.([]byte)), 10, 64)
-			if err != nil {
-				acc.AddError(fmt.Errorf("E! Error mysql: parsing %s int value (%s)", name, err))
-			} else {
-				fields["syncs"] = i
-			}
-		case "Uptime":
-			i, err := strconv.ParseInt(string(val.([]byte)), 10, 64)
-			if err != nil {
-				acc.AddError(fmt.Errorf("E! Error mysql: parsing %s int value (%s)", name, err))
-			} else {
-				fields["uptime"] = i
-			}
-		}
 	}
 	// Send any remaining fields
 	if len(fields) > 0 {
@@ -1059,7 +820,7 @@ func (m *Mysql) GatherProcessListStatuses(db *sql.DB, serv string, acc telegraf.
 	for s, c := range stateCounts {
 		fields[newNamespace("threads", s)] = c
 	}
-	acc.AddFields("mysql_info_schema", fields, tags)
+	acc.AddFields("mysql_process_list", fields, tags)
 	return nil
 }
 
@@ -1272,7 +1033,7 @@ func (m *Mysql) gatherInfoSchemaAutoIncStatuses(db *sql.DB, serv string, acc tel
 		fields["auto_increment_column"] = incValue
 		fields["auto_increment_column_max"] = maxInt
 
-		acc.AddFields("mysql_info_schema", fields, tags)
+		acc.AddFields("mysql_table_schema", fields, tags)
 	}
 	return nil
 }
@@ -1287,21 +1048,19 @@ func (m *Mysql) gatherInnoDBMetrics(db *sql.DB, serv string, acc telegraf.Accumu
 	}
 	defer rows.Close()
 
-	var key string
-	var val sql.RawBytes
-
 	// parse DSN and save server tag
 	servtag := getDSNTag(serv)
 	tags := map[string]string{"server": servtag}
 	fields := make(map[string]interface{})
 	for rows.Next() {
+		var key string
+		var val sql.RawBytes
 		if err := rows.Scan(&key, &val); err != nil {
 			return err
 		}
 		key = strings.ToLower(key)
-		// parse value, if it is numeric then save, otherwise ignore
-		if floatVal, ok := parseValue(val); ok {
-			fields[key] = floatVal
+		if value, ok := parseValue(val); ok {
+			fields[key] = value
 		}
 		// Send 20 fields at a time
 		if len(fields) >= 20 {
@@ -1671,23 +1430,17 @@ func (m *Mysql) gatherTableSchema(db *sql.DB, serv string, acc telegraf.Accumula
 			tags["schema"] = tableSchema
 			tags["table"] = tableName
 
-			acc.AddFields(newNamespace("info_schema", "table_rows"),
-				map[string]interface{}{"value": tableRows}, tags)
+			acc.AddFields("mysql_table_schema",
+				map[string]interface{}{"rows": tableRows}, tags)
 
-			dlTags := copyTags(tags)
-			dlTags["component"] = "data_length"
-			acc.AddFields(newNamespace("info_schema", "table_size", "data_length"),
-				map[string]interface{}{"value": dataLength}, dlTags)
+			acc.AddFields("mysql_table_schema",
+				map[string]interface{}{"data_length": dataLength}, tags)
 
-			ilTags := copyTags(tags)
-			ilTags["component"] = "index_length"
-			acc.AddFields(newNamespace("info_schema", "table_size", "index_length"),
-				map[string]interface{}{"value": indexLength}, ilTags)
+			acc.AddFields("mysql_table_schema",
+				map[string]interface{}{"index_length": indexLength}, tags)
 
-			dfTags := copyTags(tags)
-			dfTags["component"] = "data_free"
-			acc.AddFields(newNamespace("info_schema", "table_size", "data_free"),
-				map[string]interface{}{"value": dataFree}, dfTags)
+			acc.AddFields("mysql_table_schema",
+				map[string]interface{}{"data_free": dataFree}, tags)
 
 			versionTags := copyTags(tags)
 			versionTags["type"] = tableType
@@ -1695,24 +1448,34 @@ func (m *Mysql) gatherTableSchema(db *sql.DB, serv string, acc telegraf.Accumula
 			versionTags["row_format"] = rowFormat
 			versionTags["create_options"] = createOptions
 
-			acc.AddFields(newNamespace("info_schema", "table_version"),
-				map[string]interface{}{"value": version}, versionTags)
+			acc.AddFields("mysql_table_schema_version",
+				map[string]interface{}{"table_version": version}, versionTags)
 		}
 	}
 	return nil
 }
 
 // parseValue can be used to convert values such as "ON","OFF","Yes","No" to 0,1
-func parseValue(value sql.RawBytes) (float64, bool) {
-	if bytes.Compare(value, []byte("Yes")) == 0 || bytes.Compare(value, []byte("ON")) == 0 {
+func parseValue(value sql.RawBytes) (interface{}, bool) {
+	if bytes.EqualFold(value, []byte("YES")) || bytes.Compare(value, []byte("ON")) == 0 {
 		return 1, true
 	}
 
-	if bytes.Compare(value, []byte("No")) == 0 || bytes.Compare(value, []byte("OFF")) == 0 {
+	if bytes.EqualFold(value, []byte("NO")) || bytes.Compare(value, []byte("OFF")) == 0 {
 		return 0, true
 	}
-	n, err := strconv.ParseFloat(string(value), 64)
-	return n, err == nil
+
+	if val, err := strconv.ParseInt(string(value), 10, 64); err == nil {
+		return val, true
+	}
+	if val, err := strconv.ParseFloat(string(value), 64); err == nil {
+		return val, true
+	}
+
+	if len(string(value)) > 0 {
+		return string(value), true
+	}
+	return nil, false
 }
 
 // findThreadState can be used to find thread state by command and plain state

plugins/inputs/mysql/mysql_test.go

@@ -127,26 +127,29 @@ func TestMysqlDNSAddTimeout(t *testing.T) {
 		}
 	}
 }
-
 func TestParseValue(t *testing.T) {
 	testCases := []struct {
 		rawByte   sql.RawBytes
-		value     float64
+		output    interface{}
 		boolValue bool
 	}{
-		{sql.RawBytes("Yes"), 1, true},
-		{sql.RawBytes("No"), 0, false},
+		{sql.RawBytes("123"), int64(123), true},
+		{sql.RawBytes("abc"), "abc", true},
+		{sql.RawBytes("10.1"), 10.1, true},
 		{sql.RawBytes("ON"), 1, true},
-		{sql.RawBytes("OFF"), 0, false},
-		{sql.RawBytes("ABC"), 0, false},
+		{sql.RawBytes("OFF"), 0, true},
+		{sql.RawBytes("NO"), 0, true},
+		{sql.RawBytes("YES"), 1, true},
+		{sql.RawBytes("No"), 0, true},
+		{sql.RawBytes("Yes"), 1, true},
+		{sql.RawBytes(""), nil, false},
 	}
 	for _, cases := range testCases {
-		if value, ok := parseValue(cases.rawByte); value != cases.value && ok != cases.boolValue {
-			t.Errorf("want %d with %t, got %d with %t", int(cases.value), cases.boolValue, int(value), ok)
+		if got, ok := parseValue(cases.rawByte); got != cases.output && ok != cases.boolValue {
+			t.Errorf("for %s wanted %t, got %t", string(cases.rawByte), cases.output, got)
 		}
 	}
 }
-
 func TestNewNamespace(t *testing.T) {
 	testCases := []struct {
 		words     []string

plugins/inputs/nats/README.md

@@ -0,0 +1,42 @@
+# NATS Input Plugin
+
+The [NATS](http://www.nats.io/about/) monitoring plugin gathers metrics from
+the NATS [monitoring http server](https://www.nats.io/documentation/server/gnatsd-monitoring/).
+
+### Configuration
+
+```toml
+[[inputs.nats]]
+  ## The address of the monitoring endpoint of the NATS server
+  server = "http://localhost:8222"
+
+  ## Maximum time to receive response
+  # response_timeout = "5s"
+```
+
+### Metrics:
+
+- nats
+  - tags
+    - server
+  - fields:
+    - uptime (integer, nanoseconds)
+    - mem (integer, bytes)
+    - subscriptions (integer, count)
+    - out_bytes (integer, bytes)
+    - connections (integer, count)
+    - in_msgs (integer, bytes)
+    - total_connections (integer, count)
+    - cores (integer, count)
+    - cpu (integer, count)
+    - slow_consumers (integer, count)
+    - routes (integer, count)
+    - remotes (integer, count)
+    - out_msgs (integer, count)
+    - in_bytes (integer, bytes)
+
+### Example Output:
+
+```
+nats,server=http://localhost:8222 uptime=117158348682i,mem=6647808i,subscriptions=0i,out_bytes=0i,connections=0i,in_msgs=0i,total_connections=0i,cores=2i,cpu=0,slow_consumers=0i,routes=0i,remotes=0i,out_msgs=0i,in_bytes=0i 1517015107000000000
+```

plugins/inputs/nats/nats.go

@@ -0,0 +1,114 @@
+// +build !freebsd
+
+package nats
+
+import (
+	"io/ioutil"
+	"net/http"
+	"net/url"
+	"path"
+	"time"
+
+	"encoding/json"
+
+	"github.com/influxdata/telegraf"
+	"github.com/influxdata/telegraf/internal"
+	"github.com/influxdata/telegraf/plugins/inputs"
+
+	gnatsd "github.com/nats-io/gnatsd/server"
+)
+
+type Nats struct {
+	Server          string
+	ResponseTimeout internal.Duration
+
+	client *http.Client
+}
+
+var sampleConfig = `
+  ## The address of the monitoring endpoint of the NATS server
+  server = "http://localhost:8222"
+
+  ## Maximum time to receive response
+  # response_timeout = "5s"
+`
+
+func (n *Nats) SampleConfig() string {
+	return sampleConfig
+}
+
+func (n *Nats) Description() string {
+	return "Provides metrics about the state of a NATS server"
+}
+
+func (n *Nats) Gather(acc telegraf.Accumulator) error {
+	url, err := url.Parse(n.Server)
+	if err != nil {
+		return err
+	}
+	url.Path = path.Join(url.Path, "varz")
+
+	if n.client == nil {
+		n.client = n.createHTTPClient()
+	}
+	resp, err := n.client.Get(url.String())
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+
+	bytes, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return err
+	}
+
+	stats := new(gnatsd.Varz)
+	err = json.Unmarshal([]byte(bytes), &stats)
+	if err != nil {
+		return err
+	}
+
+	acc.AddFields("nats",
+		map[string]interface{}{
+			"in_msgs":           stats.InMsgs,
+			"out_msgs":          stats.OutMsgs,
+			"in_bytes":          stats.InBytes,
+			"out_bytes":         stats.OutBytes,
+			"uptime":            stats.Now.Sub(stats.Start).Nanoseconds(),
+			"cores":             stats.Cores,
+			"cpu":               stats.CPU,
+			"mem":               stats.Mem,
+			"connections":       stats.Connections,
+			"total_connections": stats.TotalConnections,
+			"subscriptions":     stats.Subscriptions,
+			"slow_consumers":    stats.SlowConsumers,
+			"routes":            stats.Routes,
+			"remotes":           stats.Remotes,
+		},
+		map[string]string{"server": n.Server},
+		time.Now())
+
+	return nil
+}
+
+func (n *Nats) createHTTPClient() *http.Client {
+	transport := &http.Transport{
+		Proxy: http.ProxyFromEnvironment,
+	}
+	timeout := n.ResponseTimeout.Duration
+	if timeout == time.Duration(0) {
+		timeout = 5 * time.Second
+	}
+	return &http.Client{
+		Transport: transport,
+		Timeout:   timeout,
+	}
+}
+
+func init() {
+	inputs.Add("nats", func() telegraf.Input {
+		return &Nats{
+			Server: "http://localhost:8222",
+		}
+	})
+}

plugins/inputs/nats/nats_freebsd.go

@@ -0,0 +1,3 @@
+// +build freebsd
+
+package nats

plugins/inputs/nats/nats_test.go

@@ -0,0 +1,114 @@
+// +build !freebsd
+
+package nats
+
+import (
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/influxdata/telegraf/testutil"
+	"github.com/stretchr/testify/require"
+)
+
+var sampleVarz = `
+{
+  "server_id": "n2afhLHLl64Gcaj7S7jaNa",
+  "version": "1.0.0",
+  "go": "go1.8",
+  "host": "0.0.0.0",
+  "auth_required": false,
+  "ssl_required": false,
+  "tls_required": false,
+  "tls_verify": false,
+  "addr": "0.0.0.0",
+  "max_connections": 65536,
+  "ping_interval": 120000000000,
+  "ping_max": 2,
+  "http_host": "0.0.0.0",
+  "http_port": 1337,
+  "https_port": 0,
+  "auth_timeout": 1,
+  "max_control_line": 1024,
+  "cluster": {
+    "addr": "0.0.0.0",
+    "cluster_port": 0,
+    "auth_timeout": 1
+  },
+  "tls_timeout": 0.5,
+  "port": 4222,
+  "max_payload": 1048576,
+  "start": "1861-04-12T10:15:26.841483489-05:00",
+  "now": "2011-10-05T15:24:23.722084098-07:00",
+  "uptime": "150y5md237h8m57s",
+  "mem": 15581184,
+  "cores": 48,
+  "cpu": 9,
+  "connections": 5,
+  "total_connections": 109,
+  "routes": 1,
+  "remotes": 2,
+  "in_msgs": 74148556,
+  "out_msgs": 68863261,
+  "in_bytes": 946267004717,
+  "out_bytes": 948110960598,
+  "slow_consumers": 2,
+  "subscriptions": 4,
+  "http_req_stats": {
+    "/": 1,
+    "/connz": 100847,
+    "/routez": 0,
+    "/subsz": 1,
+    "/varz": 205785
+  },
+  "config_load_time": "2017-07-24T10:15:26.841483489-05:00"
+}
+`
+
+func TestMetricsCorrect(t *testing.T) {
+	var acc testutil.Accumulator
+
+	srv := newTestNatsServer()
+	defer srv.Close()
+
+	n := &Nats{Server: srv.URL}
+	err := n.Gather(&acc)
+	require.NoError(t, err)
+
+	fields := map[string]interface{}{
+		"in_msgs":           int64(74148556),
+		"out_msgs":          int64(68863261),
+		"in_bytes":          int64(946267004717),
+		"out_bytes":         int64(948110960598),
+		"uptime":            int64(4748742536880600609),
+		"cores":             48,
+		"cpu":               float64(9),
+		"mem":               int64(15581184),
+		"connections":       int(5),
+		"total_connections": uint64(109),
+		"subscriptions":     uint32(4),
+		"slow_consumers":    int64(2),
+		"routes":            int(1),
+		"remotes":           int(2),
+	}
+	tags := map[string]string{
+		"server": srv.URL,
+	}
+	acc.AssertContainsTaggedFields(t, "nats", fields, tags)
+}
+
+func newTestNatsServer() *httptest.Server {
+	return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		var rsp string
+
+		switch r.URL.Path {
+		case "/varz":
+			rsp = sampleVarz
+		default:
+			panic("Cannot handle request")
+		}
+
+		fmt.Fprintln(w, rsp)
+	}))
+}

plugins/inputs/openldap/README.md

@@ -4,6 +4,8 @@ This plugin gathers metrics from OpenLDAP's cn=Monitor backend.
 
 ### Configuration:
 
+To use this plugin you must enable the [monitoring](https://www.openldap.org/devel/admin/monitoringslapd.html) backend.
+
 ```toml
 [[inputs.openldap]]
   host = "localhost"
@@ -23,50 +25,62 @@ This plugin gathers metrics from OpenLDAP's cn=Monitor backend.
   # dn/password to bind with. If bind_dn is empty, an anonymous bind is performed.
   bind_dn = ""
   bind_password = ""
+  
+  # reverse metric names so they sort more naturally
+  # Defaults to false if unset, but is set to true when generating a new config
+  reverse_metric_names = true
 ```
 
 ### Measurements & Fields:
 
-All **monitorCounter**, **monitorOpInitiated**, and **monitorOpCompleted** attributes are gathered based on this LDAP query:
+All **monitorCounter**, **monitoredInfo**, **monitorOpInitiated**, and **monitorOpCompleted** attributes are gathered based on this LDAP query:
 
-```(|(objectClass=monitorCounterObject)(objectClass=monitorOperation))```
+```(|(objectClass=monitorCounterObject)(objectClass=monitorOperation)(objectClass=monitoredObject))```
 
-Metric names are based on their entry DN.
+Metric names are based on their entry DN with the cn=Monitor base removed. If `reverse_metric_names` is not set, metrics are based on their DN. If `reverse_metric_names` is set to `true`, the names are reversed. This is recommended as it allows the names to sort more naturally.
 
-Metrics for the **monitorOp*** attributes have **_initiated** and **_completed** added to the base name.
+Metrics for the **monitorOp*** attributes have **_initiated** and **_completed** added to the base name as appropriate.
 
 An OpenLDAP 2.4 server will provide these metrics:
 
 - openldap
-	- max_file_descriptors_connections
-	- current_connections
-	- total_connections
-	- abandon_operations_completed
-	- abandon_operations_initiated
-	- add_operations_completed
-	- add_operations_initiated
-	- bind_operations_completed
-	- bind_operations_initiated
-	- compare_operations_completed
-	- compare_operations_initiated
-	- delete_operations_completed
-	- delete_operations_initiated
-	- extended_operations_completed
-	- extended_operations_initiated
-	- modify_operations_completed
-	- modify_operations_initiated
-	- modrdn_operations_completed
-	- modrdn_operations_initiated
-	- search_operations_completed
-	- search_operations_initiated
-	- unbind_operations_completed
-	- unbind_operations_initiated
-	- bytes_statistics
-	- entries_statistics
-	- pdu_statistics
-	- referrals_statistics
-	- read_waiters
-	- write_waiters
+	- connections_current
+	- connections_max_file_descriptors
+	- connections_total
+	- operations_abandon_completed
+	- operations_abandon_initiated
+	- operations_add_completed
+	- operations_add_initiated
+	- operations_bind_completed
+	- operations_bind_initiated
+	- operations_compare_completed
+	- operations_compare_initiated
+	- operations_delete_completed
+	- operations_delete_initiated
+	- operations_extended_completed
+	- operations_extended_initiated
+	- operations_modify_completed
+	- operations_modify_initiated
+	- operations_modrdn_completed
+	- operations_modrdn_initiated
+	- operations_search_completed
+	- operations_search_initiated
+	- operations_unbind_completed
+	- operations_unbind_initiated
+	- statistics_bytes
+	- statistics_entries
+	- statistics_pdu
+	- statistics_referrals
+	- threads_active
+	- threads_backload
+	- threads_max
+	- threads_max_pending
+	- threads_open
+	- threads_pending
+	- threads_starting
+	- time_uptime
+	- waiters_read
+	- waiters_write
 
 ### Tags:
 
@@ -78,5 +92,5 @@ An OpenLDAP 2.4 server will provide these metrics:
 ```
 $ telegraf -config telegraf.conf -input-filter openldap -test --debug
 * Plugin: inputs.openldap, Collection 1
-> openldap,server=localhost,port=389,host=zirzla search_operations_completed=2i,delete_operations_completed=0i,read_waiters=1i,total_connections=1004i,bind_operations_completed=3i,unbind_operations_completed=3i,referrals_statistics=0i,current_connections=1i,bind_operations_initiated=3i,compare_operations_completed=0i,add_operations_completed=2i,delete_operations_initiated=0i,unbind_operations_initiated=3i,search_operations_initiated=3i,add_operations_initiated=2i,max_file_descriptors_connections=4096i,abandon_operations_initiated=0i,write_waiters=0i,modrdn_operations_completed=0i,abandon_operations_completed=0i,pdu_statistics=23i,modify_operations_initiated=0i,bytes_statistics=1660i,entries_statistics=17i,compare_operations_initiated=0i,modrdn_operations_initiated=0i,extended_operations_completed=0i,modify_operations_completed=0i,extended_operations_initiated=0i 1499990455000000000
+> openldap,server=localhost,port=389,host=niska.ait.psu.edu operations_bind_initiated=10i,operations_unbind_initiated=6i,operations_modrdn_completed=0i,operations_delete_initiated=0i,operations_add_completed=2i,operations_delete_completed=0i,operations_abandon_completed=0i,statistics_entries=1516i,threads_open=2i,threads_active=1i,waiters_read=1i,operations_modify_completed=0i,operations_extended_initiated=4i,threads_pending=0i,operations_search_initiated=36i,operations_compare_initiated=0i,connections_max_file_descriptors=4096i,operations_modify_initiated=0i,operations_modrdn_initiated=0i,threads_max=16i,time_uptime=6017i,connections_total=1037i,connections_current=1i,operations_add_initiated=2i,statistics_bytes=162071i,operations_unbind_completed=6i,operations_abandon_initiated=0i,statistics_pdu=1566i,threads_max_pending=0i,threads_backload=1i,waiters_write=0i,operations_bind_completed=10i,operations_search_completed=35i,operations_compare_completed=0i,operations_extended_completed=4i,statistics_referrals=0i,threads_starting=0i 1516912070000000000
 ```

plugins/inputs/openldap/openldap.go

@@ -20,6 +20,7 @@ type Openldap struct {
 	SslCa              string
 	BindDn             string
 	BindPassword       string
+	ReverseMetricNames bool
 }
 
 const sampleConfig string = `
@@ -40,13 +41,18 @@ const sampleConfig string = `
   # dn/password to bind with. If bind_dn is empty, an anonymous bind is performed.
   bind_dn = ""
   bind_password = ""
+
+  # Reverse metric names so they sort more naturally. Recommended.
+  # This defaults to false if unset, but is set to true when generating a new config
+  reverse_metric_names = true
 `
 
 var searchBase = "cn=Monitor"
-var searchFilter = "(|(objectClass=monitorCounterObject)(objectClass=monitorOperation))"
-var searchAttrs = []string{"monitorCounter", "monitorOpInitiated", "monitorOpCompleted"}
+var searchFilter = "(|(objectClass=monitorCounterObject)(objectClass=monitorOperation)(objectClass=monitoredObject))"
+var searchAttrs = []string{"monitorCounter", "monitorOpInitiated", "monitorOpCompleted", "monitoredInfo"}
 var attrTranslate = map[string]string{
 	"monitorCounter":     "",
+	"monitoredInfo":      "",
 	"monitorOpInitiated": "_initiated",
 	"monitorOpCompleted": "_completed",
 }
@@ -69,6 +75,7 @@ func NewOpenldap() *Openldap {
 		SslCa:              "",
 		BindDn:             "",
 		BindPassword:       "",
+		ReverseMetricNames: false,
 	}
 }
 
@@ -149,7 +156,7 @@ func gatherSearchResult(sr *ldap.SearchResult, o *Openldap, acc telegraf.Accumul
 		"port":   strconv.Itoa(o.Port),
 	}
 	for _, entry := range sr.Entries {
-		metricName := dnToMetric(entry.DN, searchBase)
+		metricName := dnToMetric(entry.DN, o)
 		for _, attr := range entry.Attributes {
 			if len(attr.Values[0]) >= 1 {
 				if v, err := strconv.ParseInt(attr.Values[0], 10, 64); err == nil {
@@ -162,15 +169,30 @@ func gatherSearchResult(sr *ldap.SearchResult, o *Openldap, acc telegraf.Accumul
 	return
 }
 
-// Convert a DN to metric name, eg cn=Read,cn=Waiters,cn=Monitor to read_waiters
-func dnToMetric(dn, searchBase string) string {
-	metricName := strings.Trim(dn, " ")
-	metricName = strings.Replace(metricName, " ", "_", -1)
-	metricName = strings.ToLower(metricName)
-	metricName = strings.TrimPrefix(metricName, "cn=")
-	metricName = strings.Replace(metricName, strings.ToLower(searchBase), "", -1)
-	metricName = strings.Replace(metricName, "cn=", "_", -1)
-	return strings.Replace(metricName, ",", "", -1)
+// Convert a DN to metric name, eg cn=Read,cn=Waiters,cn=Monitor becomes waiters_read
+// Assumes the last part of the DN is cn=Monitor and we want to drop it
+func dnToMetric(dn string, o *Openldap) string {
+	if o.ReverseMetricNames {
+		var metricParts []string
+
+		dn = strings.Trim(dn, " ")
+		dn = strings.Replace(dn, " ", "_", -1)
+		dn = strings.Replace(dn, "cn=", "", -1)
+		dn = strings.ToLower(dn)
+		metricParts = strings.Split(dn, ",")
+		for i, j := 0, len(metricParts)-1; i < j; i, j = i+1, j-1 {
+			metricParts[i], metricParts[j] = metricParts[j], metricParts[i]
+		}
+		return strings.Join(metricParts[1:], "_")
+	} else {
+		metricName := strings.Trim(dn, " ")
+		metricName = strings.Replace(metricName, " ", "_", -1)
+		metricName = strings.ToLower(metricName)
+		metricName = strings.TrimPrefix(metricName, "cn=")
+		metricName = strings.Replace(metricName, strings.ToLower("cn=Monitor"), "", -1)
+		metricName = strings.Replace(metricName, "cn=", "_", -1)
+		return strings.Replace(metricName, ",", "", -1)
+	}
 }
 
 func init() {

plugins/inputs/openldap/openldap_test.go

@@ -148,3 +148,24 @@ func commonTests(t *testing.T, o *Openldap, acc *testutil.Accumulator) {
 	assert.Equal(t, strconv.Itoa(o.Port), acc.TagValue("openldap", "port"), "Has a tag value of port=o.Port")
 	assert.True(t, acc.HasInt64Field("openldap", "total_connections"), "Has an integer field called total_connections")
 }
+
+func TestOpenldapReverseMetrics(t *testing.T) {
+	if testing.Short() {
+		t.Skip("Skipping integration test in short mode")
+	}
+
+	o := &Openldap{
+		Host:               testutil.GetLocalHost(),
+		Port:               389,
+		Ssl:                "",
+		InsecureSkipVerify: true,
+		BindDn:             "cn=manager,cn=config",
+		BindPassword:       "secret",
+		ReverseMetricNames: true,
+	}
+
+	var acc testutil.Accumulator
+	err := o.Gather(&acc)
+	require.NoError(t, err)
+	assert.True(t, acc.HasInt64Field("openldap", "connections_total"), "Has an integer field called connections_total")
+}

plugins/inputs/ping/README.md

@@ -17,7 +17,8 @@ urls = ["www.google.com"] # required
 # ping_interval = 1.0
 ## per-ping timeout, in s. 0 == no timeout (ping -W <TIMEOUT>)
 # timeout = 1.0
-## interface to send ping from (ping -I <INTERFACE>)
+## interface or source address to send ping from (ping -I <INTERFACE/SRC_ADDR>)
+## on Darwin and Freebsd only source address possible: (ping -S <SRC_ADDR>)
 # interface = ""
 ```
 

plugins/inputs/ping/ping.go

@@ -34,7 +34,7 @@ type Ping struct {
 	// Ping timeout, in seconds. 0 means no timeout (ping -W <TIMEOUT>)
 	Timeout float64
 
-	// Interface to send ping from (ping -I <INTERFACE>)
+	// Interface or source address to send ping from (ping -I/-S <INTERFACE/SRC_ADDR>)
 	Interface string
 
 	// URLs to ping
@@ -60,7 +60,8 @@ const sampleConfig = `
   # ping_interval = 1.0
   ## per-ping timeout, in s. 0 == no timeout (ping -W <TIMEOUT>)
   # timeout = 1.0
-  ## interface to send ping from (ping -I <INTERFACE>)
+  ## interface or source address to send ping from (ping -I <INTERFACE/SRC_ADDR>)
+  ## on Darwin and Freebsd only source address possible: (ping -S <SRC_ADDR>)
   # interface = ""
 `
 
@@ -179,7 +180,15 @@ func (p *Ping) args(url string) []string {
 		}
 	}
 	if p.Interface != "" {
-		args = append(args, "-I", p.Interface)
+		switch runtime.GOOS {
+		case "linux":
+			args = append(args, "-I", p.Interface)
+		case "freebsd", "darwin":
+			args = append(args, "-S", p.Interface)
+		default:
+			// Not sure the best option here, just assume GNU ping?
+			args = append(args, "-I", p.Interface)
+		}
 	}
 	args = append(args, url)
 	return args

plugins/inputs/postfix/README.md

@@ -13,6 +13,25 @@ For each of the active, hold, incoming, maildrop, and deferred queues (http://ww
   # queue_directory = "/var/spool/postfix"
 ```
 
+#### Permissions:
+
+Telegraf will need read access to the files in the queue directory.  You may
+need to alter the permissions of these directories to provide access to the
+telegraf user.
+
+Unix permissions:
+```sh
+$ sudo chgrp -R telegraf /var/spool/postfix/{active,hold,incoming,deferred}
+$ sudo chmod -R g+rXs /var/spool/postfix/{active,hold,incoming,deferred}
+$ sudo usermod -a -G postdrop telegraf
+$ sudo chmod g+r /var/spool/postfix/maildrop
+```
+
+Posix ACL:
+```sh
+$ sudo setfacl -Rdm u:telegraf:rX /var/spool/postfix/{active,hold,incoming,deferred,maildrop}
+```
+
 ### Measurements & Fields:
 
 - postfix_queue

plugins/inputs/postgresql/connect.go

@@ -1,77 +0,0 @@
-package postgresql
-
-import (
-	"fmt"
-	"net"
-	"net/url"
-	"sort"
-	"strings"
-)
-
-// pulled from lib/pq
-// ParseURL no longer needs to be used by clients of this library since supplying a URL as a
-// connection string to sql.Open() is now supported:
-//
-//	sql.Open("postgres", "postgres://bob:secret@1.2.3.4:5432/mydb?sslmode=verify-full")
-//
-// It remains exported here for backwards-compatibility.
-//
-// ParseURL converts a url to a connection string for driver.Open.
-// Example:
-//
-//	"postgres://bob:secret@1.2.3.4:5432/mydb?sslmode=verify-full"
-//
-// converts to:
-//
-//	"user=bob password=secret host=1.2.3.4 port=5432 dbname=mydb sslmode=verify-full"
-//
-// A minimal example:
-//
-//	"postgres://"
-//
-// This will be blank, causing driver.Open to use all of the defaults
-func ParseURL(uri string) (string, error) {
-	u, err := url.Parse(uri)
-	if err != nil {
-		return "", err
-	}
-
-	if u.Scheme != "postgres" && u.Scheme != "postgresql" {
-		return "", fmt.Errorf("invalid connection protocol: %s", u.Scheme)
-	}
-
-	var kvs []string
-	escaper := strings.NewReplacer(` `, `\ `, `'`, `\'`, `\`, `\\`)
-	accrue := func(k, v string) {
-		if v != "" {
-			kvs = append(kvs, k+"="+escaper.Replace(v))
-		}
-	}
-
-	if u.User != nil {
-		v := u.User.Username()
-		accrue("user", v)
-
-		v, _ = u.User.Password()
-		accrue("password", v)
-	}
-
-	if host, port, err := net.SplitHostPort(u.Host); err != nil {
-		accrue("host", u.Host)
-	} else {
-		accrue("host", host)
-		accrue("port", port)
-	}
-
-	if u.Path != "" {
-		accrue("dbname", u.Path[1:])
-	}
-
-	q := u.Query()
-	for k := range q {
-		accrue(k, q.Get(k))
-	}
-
-	sort.Strings(kvs) // Makes testing easier (not a performance concern)
-	return strings.Join(kvs, " "), nil
-}

plugins/inputs/postgresql/postgresql.go

@@ -2,26 +2,21 @@ package postgresql
 
 import (
 	"bytes"
-	"database/sql"
 	"fmt"
-	"regexp"
-	"sort"
 	"strings"
 
 	// register in driver.
 	_ "github.com/jackc/pgx/stdlib"
 
 	"github.com/influxdata/telegraf"
+	"github.com/influxdata/telegraf/internal"
 	"github.com/influxdata/telegraf/plugins/inputs"
 )
 
 type Postgresql struct {
-	Address          string
+	Service
 	Databases        []string
 	IgnoredDatabases []string
-	OrderedColumns   []string
-	AllColumns       []string
-	sanitizedAddress string
 }
 
 var ignoredColumns = map[string]bool{"stats_reset": true}
@@ -41,6 +36,15 @@ var sampleConfig = `
   ## to grab metrics for.
   ##
   address = "host=localhost user=postgres sslmode=disable"
+  ## A custom name for the database that will be used as the "server" tag in the
+  ## measurement output. If not specified, a default one generated from
+  ## the connection address is used.
+  # outputaddress = "db01"
+
+  ## connection configuration.
+  ## maxlifetime - specify the maximum lifetime of a connection.
+  ## default is forever (0s)
+  max_lifetime = "0s"
 
   ## A  list of databases to explicitly ignore.  If not specified, metrics for all
   ## databases are gathered.  Do NOT use with the 'databases' option.
@@ -63,24 +67,13 @@ func (p *Postgresql) IgnoredColumns() map[string]bool {
 	return ignoredColumns
 }
 
-var localhost = "host=localhost sslmode=disable"
-
 func (p *Postgresql) Gather(acc telegraf.Accumulator) error {
 	var (
-		err   error
-		db    *sql.DB
-		query string
+		err     error
+		query   string
+		columns []string
 	)
 
-	if p.Address == "" || p.Address == "localhost" {
-		p.Address = localhost
-	}
-
-	if db, err = sql.Open("pgx", p.Address); err != nil {
-		return err
-	}
-	defer db.Close()
-
 	if len(p.Databases) == 0 && len(p.IgnoredDatabases) == 0 {
 		query = `SELECT * FROM pg_stat_database`
 	} else if len(p.IgnoredDatabases) != 0 {
@@ -91,7 +84,7 @@ func (p *Postgresql) Gather(acc telegraf.Accumulator) error {
 			strings.Join(p.Databases, "','"))
 	}
 
-	rows, err := db.Query(query)
+	rows, err := p.DB.Query(query)
 	if err != nil {
 		return err
 	}
@@ -99,16 +92,12 @@ func (p *Postgresql) Gather(acc telegraf.Accumulator) error {
 	defer rows.Close()
 
 	// grab the column information from the result
-	p.OrderedColumns, err = rows.Columns()
-	if err != nil {
+	if columns, err = rows.Columns(); err != nil {
 		return err
-	} else {
-		p.AllColumns = make([]string, len(p.OrderedColumns))
-		copy(p.AllColumns, p.OrderedColumns)
 	}
 
 	for rows.Next() {
-		err = p.accRow(rows, acc)
+		err = p.accRow(rows, acc, columns)
 		if err != nil {
 			return err
 		}
@@ -116,7 +105,7 @@ func (p *Postgresql) Gather(acc telegraf.Accumulator) error {
 
 	query = `SELECT * FROM pg_stat_bgwriter`
 
-	bg_writer_row, err := db.Query(query)
+	bg_writer_row, err := p.DB.Query(query)
 	if err != nil {
 		return err
 	}
@@ -124,22 +113,17 @@ func (p *Postgresql) Gather(acc telegraf.Accumulator) error {
 	defer bg_writer_row.Close()
 
 	// grab the column information from the result
-	p.OrderedColumns, err = bg_writer_row.Columns()
-	if err != nil {
+	if columns, err = bg_writer_row.Columns(); err != nil {
 		return err
-	} else {
-		for _, v := range p.OrderedColumns {
-			p.AllColumns = append(p.AllColumns, v)
-		}
 	}
 
 	for bg_writer_row.Next() {
-		err = p.accRow(bg_writer_row, acc)
+		err = p.accRow(bg_writer_row, acc, columns)
 		if err != nil {
 			return err
 		}
 	}
-	sort.Strings(p.AllColumns)
+
 	return bg_writer_row.Err()
 }
 
@@ -147,37 +131,20 @@ type scanner interface {
 	Scan(dest ...interface{}) error
 }
 
-var passwordKVMatcher, _ = regexp.Compile("password=\\S+ ?")
-
-func (p *Postgresql) SanitizedAddress() (_ string, err error) {
-	var canonicalizedAddress string
-	if strings.HasPrefix(p.Address, "postgres://") || strings.HasPrefix(p.Address, "postgresql://") {
-		canonicalizedAddress, err = ParseURL(p.Address)
-		if err != nil {
-			return p.sanitizedAddress, err
-		}
-	} else {
-		canonicalizedAddress = p.Address
-	}
-	p.sanitizedAddress = passwordKVMatcher.ReplaceAllString(canonicalizedAddress, "")
-
-	return p.sanitizedAddress, err
-}
-
-func (p *Postgresql) accRow(row scanner, acc telegraf.Accumulator) error {
+func (p *Postgresql) accRow(row scanner, acc telegraf.Accumulator, columns []string) error {
 	var columnVars []interface{}
 	var dbname bytes.Buffer
 
 	// this is where we'll store the column name with its *interface{}
 	columnMap := make(map[string]*interface{})
 
-	for _, column := range p.OrderedColumns {
+	for _, column := range columns {
 		columnMap[column] = new(interface{})
 	}
 
 	// populate the array of interface{} with the pointers in the right order
 	for i := 0; i < len(columnMap); i++ {
-		columnVars = append(columnVars, columnMap[p.OrderedColumns[i]])
+		columnVars = append(columnVars, columnMap[columns[i]])
 	}
 
 	// deconstruct array of variables and send to Scan
@@ -215,6 +182,14 @@ func (p *Postgresql) accRow(row scanner, acc telegraf.Accumulator) error {
 
 func init() {
 	inputs.Add("postgresql", func() telegraf.Input {
-		return &Postgresql{}
+		return &Postgresql{
+			Service: Service{
+				MaxIdle: 1,
+				MaxOpen: 1,
+				MaxLifetime: internal.Duration{
+					Duration: 0,
+				},
+			},
+		}
 	})
 }

plugins/inputs/postgresql/postgresql_test.go

@@ -15,19 +15,18 @@ func TestPostgresqlGeneratesMetrics(t *testing.T) {
 	}
 
 	p := &Postgresql{
-		Address: fmt.Sprintf("host=%s user=postgres sslmode=disable",
-			testutil.GetLocalHost()),
+		Service: Service{
+			Address: fmt.Sprintf(
+				"host=%s user=postgres sslmode=disable",
+				testutil.GetLocalHost(),
+			),
+		},
 		Databases: []string{"postgres"},
 	}
 
 	var acc testutil.Accumulator
-	err := p.Gather(&acc)
-	require.NoError(t, err)
-
-	availableColumns := make(map[string]bool)
-	for _, col := range p.AllColumns {
-		availableColumns[col] = true
-	}
+	require.NoError(t, p.Start(&acc))
+	require.NoError(t, p.Gather(&acc))
 
 	intMetrics := []string{
 		"xact_commit",
@@ -71,39 +70,27 @@ func TestPostgresqlGeneratesMetrics(t *testing.T) {
 	metricsCounted := 0
 
 	for _, metric := range intMetrics {
-		_, ok := availableColumns[metric]
-		if ok {
-			assert.True(t, acc.HasInt64Field("postgresql", metric))
-			metricsCounted++
-		}
+		assert.True(t, acc.HasInt64Field("postgresql", metric))
+		metricsCounted++
 	}
 
 	for _, metric := range int32Metrics {
-		_, ok := availableColumns[metric]
-		if ok {
-			assert.True(t, acc.HasInt32Field("postgresql", metric))
-			metricsCounted++
-		}
+		assert.True(t, acc.HasInt32Field("postgresql", metric))
+		metricsCounted++
 	}
 
 	for _, metric := range floatMetrics {
-		_, ok := availableColumns[metric]
-		if ok {
-			assert.True(t, acc.HasFloatField("postgresql", metric))
-			metricsCounted++
-		}
+		assert.True(t, acc.HasFloatField("postgresql", metric))
+		metricsCounted++
 	}
 
 	for _, metric := range stringMetrics {
-		_, ok := availableColumns[metric]
-		if ok {
-			assert.True(t, acc.HasStringField("postgresql", metric))
-			metricsCounted++
-		}
+		assert.True(t, acc.HasStringField("postgresql", metric))
+		metricsCounted++
 	}
 
 	assert.True(t, metricsCounted > 0)
-	assert.Equal(t, len(availableColumns)-len(p.IgnoredColumns()), metricsCounted)
+	assert.Equal(t, len(floatMetrics)+len(intMetrics)+len(int32Metrics)+len(stringMetrics), metricsCounted)
 }
 
 func TestPostgresqlTagsMetricsWithDatabaseName(t *testing.T) {
@@ -112,15 +99,19 @@ func TestPostgresqlTagsMetricsWithDatabaseName(t *testing.T) {
 	}
 
 	p := &Postgresql{
-		Address: fmt.Sprintf("host=%s user=postgres sslmode=disable",
-			testutil.GetLocalHost()),
+		Service: Service{
+			Address: fmt.Sprintf(
+				"host=%s user=postgres sslmode=disable",
+				testutil.GetLocalHost(),
+			),
+		},
 		Databases: []string{"postgres"},
 	}
 
 	var acc testutil.Accumulator
 
-	err := p.Gather(&acc)
-	require.NoError(t, err)
+	require.NoError(t, p.Start(&acc))
+	require.NoError(t, p.Gather(&acc))
 
 	point, ok := acc.Get("postgresql")
 	require.True(t, ok)
@@ -134,14 +125,18 @@ func TestPostgresqlDefaultsToAllDatabases(t *testing.T) {
 	}
 
 	p := &Postgresql{
-		Address: fmt.Sprintf("host=%s user=postgres sslmode=disable",
-			testutil.GetLocalHost()),
+		Service: Service{
+			Address: fmt.Sprintf(
+				"host=%s user=postgres sslmode=disable",
+				testutil.GetLocalHost(),
+			),
+		},
 	}
 
 	var acc testutil.Accumulator
 
-	err := p.Gather(&acc)
-	require.NoError(t, err)
+	require.NoError(t, p.Start(&acc))
+	require.NoError(t, p.Gather(&acc))
 
 	var found bool
 
@@ -163,14 +158,17 @@ func TestPostgresqlIgnoresUnwantedColumns(t *testing.T) {
 	}
 
 	p := &Postgresql{
-		Address: fmt.Sprintf("host=%s user=postgres sslmode=disable",
-			testutil.GetLocalHost()),
+		Service: Service{
+			Address: fmt.Sprintf(
+				"host=%s user=postgres sslmode=disable",
+				testutil.GetLocalHost(),
+			),
+		},
 	}
 
 	var acc testutil.Accumulator
-
-	err := p.Gather(&acc)
-	require.NoError(t, err)
+	require.NoError(t, p.Start(&acc))
+	require.NoError(t, p.Gather(&acc))
 
 	for col := range p.IgnoredColumns() {
 		assert.False(t, acc.HasMeasurement(col))
@@ -183,15 +181,19 @@ func TestPostgresqlDatabaseWhitelistTest(t *testing.T) {
 	}
 
 	p := &Postgresql{
-		Address: fmt.Sprintf("host=%s user=postgres sslmode=disable",
-			testutil.GetLocalHost()),
+		Service: Service{
+			Address: fmt.Sprintf(
+				"host=%s user=postgres sslmode=disable",
+				testutil.GetLocalHost(),
+			),
+		},
 		Databases: []string{"template0"},
 	}
 
 	var acc testutil.Accumulator
 
-	err := p.Gather(&acc)
-	require.NoError(t, err)
+	require.NoError(t, p.Start(&acc))
+	require.NoError(t, p.Gather(&acc))
 
 	var foundTemplate0 = false
 	var foundTemplate1 = false
@@ -219,15 +221,18 @@ func TestPostgresqlDatabaseBlacklistTest(t *testing.T) {
 	}
 
 	p := &Postgresql{
-		Address: fmt.Sprintf("host=%s user=postgres sslmode=disable",
-			testutil.GetLocalHost()),
+		Service: Service{
+			Address: fmt.Sprintf(
+				"host=%s user=postgres sslmode=disable",
+				testutil.GetLocalHost(),
+			),
+		},
 		IgnoredDatabases: []string{"template0"},
 	}
 
 	var acc testutil.Accumulator
-
-	err := p.Gather(&acc)
-	require.NoError(t, err)
+	require.NoError(t, p.Start(&acc))
+	require.NoError(t, p.Gather(&acc))
 
 	var foundTemplate0 = false
 	var foundTemplate1 = false

plugins/inputs/postgresql/service.go

@@ -0,0 +1,142 @@
+package postgresql
+
+import (
+	"database/sql"
+	"fmt"
+	"net"
+	"net/url"
+	"regexp"
+	"sort"
+	"strings"
+
+	"github.com/influxdata/telegraf"
+	"github.com/influxdata/telegraf/internal"
+)
+
+// pulled from lib/pq
+// ParseURL no longer needs to be used by clients of this library since supplying a URL as a
+// connection string to sql.Open() is now supported:
+//
+//	sql.Open("postgres", "postgres://bob:secret@1.2.3.4:5432/mydb?sslmode=verify-full")
+//
+// It remains exported here for backwards-compatibility.
+//
+// ParseURL converts a url to a connection string for driver.Open.
+// Example:
+//
+//	"postgres://bob:secret@1.2.3.4:5432/mydb?sslmode=verify-full"
+//
+// converts to:
+//
+//	"user=bob password=secret host=1.2.3.4 port=5432 dbname=mydb sslmode=verify-full"
+//
+// A minimal example:
+//
+//	"postgres://"
+//
+// This will be blank, causing driver.Open to use all of the defaults
+func parseURL(uri string) (string, error) {
+	u, err := url.Parse(uri)
+	if err != nil {
+		return "", err
+	}
+
+	if u.Scheme != "postgres" && u.Scheme != "postgresql" {
+		return "", fmt.Errorf("invalid connection protocol: %s", u.Scheme)
+	}
+
+	var kvs []string
+	escaper := strings.NewReplacer(` `, `\ `, `'`, `\'`, `\`, `\\`)
+	accrue := func(k, v string) {
+		if v != "" {
+			kvs = append(kvs, k+"="+escaper.Replace(v))
+		}
+	}
+
+	if u.User != nil {
+		v := u.User.Username()
+		accrue("user", v)
+
+		v, _ = u.User.Password()
+		accrue("password", v)
+	}
+
+	if host, port, err := net.SplitHostPort(u.Host); err != nil {
+		accrue("host", u.Host)
+	} else {
+		accrue("host", host)
+		accrue("port", port)
+	}
+
+	if u.Path != "" {
+		accrue("dbname", u.Path[1:])
+	}
+
+	q := u.Query()
+	for k := range q {
+		accrue(k, q.Get(k))
+	}
+
+	sort.Strings(kvs) // Makes testing easier (not a performance concern)
+	return strings.Join(kvs, " "), nil
+}
+
+// Service common functionality shared between the postgresql and postgresql_extensible
+// packages.
+type Service struct {
+	Address       string
+	Outputaddress string
+	MaxIdle       int
+	MaxOpen       int
+	MaxLifetime   internal.Duration
+	DB            *sql.DB
+}
+
+// Start starts the ServiceInput's service, whatever that may be
+func (p *Service) Start(telegraf.Accumulator) (err error) {
+	const localhost = "host=localhost sslmode=disable"
+
+	if p.Address == "" || p.Address == "localhost" {
+		p.Address = localhost
+	}
+
+	if p.DB, err = sql.Open("pgx", p.Address); err != nil {
+		return err
+	}
+
+	p.DB.SetMaxOpenConns(p.MaxOpen)
+	p.DB.SetMaxIdleConns(p.MaxIdle)
+	p.DB.SetConnMaxLifetime(p.MaxLifetime.Duration)
+
+	return nil
+}
+
+// Stop stops the services and closes any necessary channels and connections
+func (p *Service) Stop() {
+	p.DB.Close()
+}
+
+var kvMatcher, _ = regexp.Compile("(password|sslcert|sslkey|sslmode|sslrootcert)=\\S+ ?")
+
+// SanitizedAddress utility function to strip sensitive information from the connection string.
+func (p *Service) SanitizedAddress() (sanitizedAddress string, err error) {
+	var (
+		canonicalizedAddress string
+	)
+
+	if p.Outputaddress != "" {
+		return p.Outputaddress, nil
+	}
+
+	if strings.HasPrefix(p.Address, "postgres://") || strings.HasPrefix(p.Address, "postgresql://") {
+		if canonicalizedAddress, err = parseURL(p.Address); err != nil {
+			return sanitizedAddress, err
+		}
+	} else {
+		canonicalizedAddress = p.Address
+	}
+
+	sanitizedAddress = kvMatcher.ReplaceAllString(canonicalizedAddress, "")
+
+	return sanitizedAddress, err
+}

plugins/inputs/postgresql_extensible/README.md

@@ -43,7 +43,11 @@ The example below has two queries are specified, with the following parameters:
   # withdbname was true.
   # Be careful that if the withdbname is set to false you don't have to define
   # the where clause (aka with the dbname)
-  # the tagvalue field is used to define custom tags (separated by comas)
+  #
+  # the tagvalue field is used to define custom tags (separated by comas).
+  # the query is expected to return columns which match the names of the
+  # defined tags. The values in these columns must be of a string-type,
+  # a number-type or a blob-type.
   #
   # Structure :
   # [[inputs.postgresql_extensible.query]]
@@ -109,6 +113,15 @@ using postgreql extensions ([pg_stat_statements](http://www.postgresql.org/docs/
   version=901
   withdbname=false
   tagvalue="db"
+[[inputs.postgresql_extensible.query]]
+  sqlquery="""
+    SELECT type, (enabled || '') AS enabled, COUNT(*)
+      FROM application_users
+      GROUP BY type, enabled
+  """
+  version=901
+  withdbname=false
+  tagvalue="type,enabled"
 ```
 
 # Postgresql Side

plugins/inputs/postgresql_extensible/postgresql_extensible.go

@@ -2,29 +2,24 @@ package postgresql_extensible
 
 import (
 	"bytes"
-	"database/sql"
 	"fmt"
 	"log"
-	"regexp"
 	"strings"
 
 	// register in driver.
 	_ "github.com/jackc/pgx/stdlib"
 
 	"github.com/influxdata/telegraf"
+	"github.com/influxdata/telegraf/internal"
 	"github.com/influxdata/telegraf/plugins/inputs"
 	"github.com/influxdata/telegraf/plugins/inputs/postgresql"
 )
 
 type Postgresql struct {
-	Address          string
-	Outputaddress    string
-	Databases        []string
-	OrderedColumns   []string
-	AllColumns       []string
-	AdditionalTags   []string
-	sanitizedAddress string
-	Query            []struct {
+	postgresql.Service
+	Databases      []string
+	AdditionalTags []string
+	Query          []struct {
 		Sqlquery    string
 		Version     int
 		Withdbname  bool
@@ -58,14 +53,20 @@ var sampleConfig = `
   ## to grab metrics for.
   #
   address = "host=localhost user=postgres sslmode=disable"
+
+  ## connection configuration.
+  ## maxlifetime - specify the maximum lifetime of a connection.
+  ## default is forever (0s)
+  max_lifetime = "0s"
+
   ## A list of databases to pull metrics about. If not specified, metrics for all
   ## databases are gathered.
   ## databases = ["app_production", "testing"]
   #
-  # outputaddress = "db01"
   ## A custom name for the database that will be used as the "server" tag in the
   ## measurement output. If not specified, a default one generated from
   ## the connection address is used.
+  # outputaddress = "db01"
   #
   ## Define the toml config where the sql queries are stored
   ## New queries can be added, if the withdbname is set to true and there is no
@@ -113,36 +114,25 @@ func (p *Postgresql) IgnoredColumns() map[string]bool {
 	return ignoredColumns
 }
 
-var localhost = "host=localhost sslmode=disable"
-
 func (p *Postgresql) Gather(acc telegraf.Accumulator) error {
 	var (
 		err         error
-		db          *sql.DB
 		sql_query   string
 		query_addon string
 		db_version  int
 		query       string
 		tag_value   string
 		meas_name   string
+		columns     []string
 	)
 
-	if p.Address == "" || p.Address == "localhost" {
-		p.Address = localhost
-	}
-
-	if db, err = sql.Open("pgx", p.Address); err != nil {
-		return err
-	}
-	defer db.Close()
-
 	// Retreiving the database version
 
 	query = `select substring(setting from 1 for 3) as version from pg_settings where name='server_version_num'`
-	err = db.QueryRow(query).Scan(&db_version)
-	if err != nil {
+	if err = p.DB.QueryRow(query).Scan(&db_version); err != nil {
 		db_version = 0
 	}
+
 	// We loop in order to process each query
 	// Query is not run if Database version does not match the query version.
 
@@ -168,7 +158,7 @@ func (p *Postgresql) Gather(acc telegraf.Accumulator) error {
 		sql_query += query_addon
 
 		if p.Query[i].Version <= db_version {
-			rows, err := db.Query(sql_query)
+			rows, err := p.DB.Query(sql_query)
 			if err != nil {
 				acc.AddError(err)
 				continue
@@ -177,15 +167,11 @@ func (p *Postgresql) Gather(acc telegraf.Accumulator) error {
 			defer rows.Close()
 
 			// grab the column information from the result
-			p.OrderedColumns, err = rows.Columns()
-			if err != nil {
+			if columns, err = rows.Columns(); err != nil {
 				acc.AddError(err)
 				continue
-			} else {
-				for _, v := range p.OrderedColumns {
-					p.AllColumns = append(p.AllColumns, v)
-				}
 			}
+
 			p.AdditionalTags = nil
 			if tag_value != "" {
 				tag_list := strings.Split(tag_value, ",")
@@ -195,7 +181,7 @@ func (p *Postgresql) Gather(acc telegraf.Accumulator) error {
 			}
 
 			for rows.Next() {
-				err = p.accRow(meas_name, rows, acc)
+				err = p.accRow(meas_name, rows, acc, columns)
 				if err != nil {
 					acc.AddError(err)
 					break
@@ -210,27 +196,7 @@ type scanner interface {
 	Scan(dest ...interface{}) error
 }
 
-var KVMatcher, _ = regexp.Compile("(password|sslcert|sslkey|sslmode|sslrootcert)=\\S+ ?")
-
-func (p *Postgresql) SanitizedAddress() (_ string, err error) {
-	if p.Outputaddress != "" {
-		return p.Outputaddress, nil
-	}
-	var canonicalizedAddress string
-	if strings.HasPrefix(p.Address, "postgres://") || strings.HasPrefix(p.Address, "postgresql://") {
-		canonicalizedAddress, err = postgresql.ParseURL(p.Address)
-		if err != nil {
-			return p.sanitizedAddress, err
-		}
-	} else {
-		canonicalizedAddress = p.Address
-	}
-	p.sanitizedAddress = KVMatcher.ReplaceAllString(canonicalizedAddress, "")
-
-	return p.sanitizedAddress, err
-}
-
-func (p *Postgresql) accRow(meas_name string, row scanner, acc telegraf.Accumulator) error {
+func (p *Postgresql) accRow(meas_name string, row scanner, acc telegraf.Accumulator, columns []string) error {
 	var (
 		err        error
 		columnVars []interface{}
@@ -241,13 +207,13 @@ func (p *Postgresql) accRow(meas_name string, row scanner, acc telegraf.Accumula
 	// this is where we'll store the column name with its *interface{}
 	columnMap := make(map[string]*interface{})
 
-	for _, column := range p.OrderedColumns {
+	for _, column := range columns {
 		columnMap[column] = new(interface{})
 	}
 
 	// populate the array of interface{} with the pointers in the right order
 	for i := 0; i < len(columnMap); i++ {
-		columnVars = append(columnVars, columnMap[p.OrderedColumns[i]])
+		columnVars = append(columnVars, columnMap[columns[i]])
 	}
 
 	// deconstruct array of variables and send to Scan
@@ -275,7 +241,7 @@ func (p *Postgresql) accRow(meas_name string, row scanner, acc telegraf.Accumula
 	fields := make(map[string]interface{})
 COLUMN:
 	for col, val := range columnMap {
-		log.Printf("D! postgresql_extensible: column: %s = %T: %s\n", col, *val, *val)
+		log.Printf("D! postgresql_extensible: column: %s = %T: %v\n", col, *val, *val)
 		_, ignore := ignoredColumns[col]
 		if ignore || *val == nil {
 			continue
@@ -310,6 +276,14 @@ COLUMN:
 
 func init() {
 	inputs.Add("postgresql_extensible", func() telegraf.Input {
-		return &Postgresql{}
+		return &Postgresql{
+			Service: postgresql.Service{
+				MaxIdle: 1,
+				MaxOpen: 1,
+				MaxLifetime: internal.Duration{
+					Duration: 0,
+				},
+			},
+		}
 	})
 }

plugins/inputs/postgresql_extensible/postgresql_extensible_test.go

@@ -4,22 +4,28 @@ import (
 	"fmt"
 	"testing"
 
+	"github.com/influxdata/telegraf/plugins/inputs/postgresql"
 	"github.com/influxdata/telegraf/testutil"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
-func queryRunner(t *testing.T, q query) (*Postgresql, *testutil.Accumulator) {
+func queryRunner(t *testing.T, q query) *testutil.Accumulator {
 	p := &Postgresql{
-		Address: fmt.Sprintf("host=%s user=postgres sslmode=disable",
-			testutil.GetLocalHost()),
+		Service: postgresql.Service{
+			Address: fmt.Sprintf(
+				"host=%s user=postgres sslmode=disable",
+				testutil.GetLocalHost(),
+			),
+		},
 		Databases: []string{"postgres"},
 		Query:     q,
 	}
 	var acc testutil.Accumulator
+	p.Start(&acc)
 
 	require.NoError(t, acc.GatherError(p.Gather))
-	return p, &acc
+	return &acc
 }
 
 func TestPostgresqlGeneratesMetrics(t *testing.T) {
@@ -27,18 +33,13 @@ func TestPostgresqlGeneratesMetrics(t *testing.T) {
 		t.Skip("Skipping integration test in short mode")
 	}
 
-	p, acc := queryRunner(t, query{{
+	acc := queryRunner(t, query{{
 		Sqlquery:   "select * from pg_stat_database",
 		Version:    901,
 		Withdbname: false,
 		Tagvalue:   "",
 	}})
 
-	availableColumns := make(map[string]bool)
-	for _, col := range p.AllColumns {
-		availableColumns[col] = true
-	}
-
 	intMetrics := []string{
 		"xact_commit",
 		"xact_rollback",
@@ -71,39 +72,27 @@ func TestPostgresqlGeneratesMetrics(t *testing.T) {
 	metricsCounted := 0
 
 	for _, metric := range intMetrics {
-		_, ok := availableColumns[metric]
-		if ok {
-			assert.True(t, acc.HasInt64Field("postgresql", metric))
-			metricsCounted++
-		}
+		assert.True(t, acc.HasInt64Field("postgresql", metric))
+		metricsCounted++
 	}
 
 	for _, metric := range int32Metrics {
-		_, ok := availableColumns[metric]
-		if ok {
-			assert.True(t, acc.HasInt32Field("postgresql", metric))
-			metricsCounted++
-		}
+		assert.True(t, acc.HasInt32Field("postgresql", metric))
+		metricsCounted++
 	}
 
 	for _, metric := range floatMetrics {
-		_, ok := availableColumns[metric]
-		if ok {
-			assert.True(t, acc.HasFloatField("postgresql", metric))
-			metricsCounted++
-		}
+		assert.True(t, acc.HasFloatField("postgresql", metric))
+		metricsCounted++
 	}
 
 	for _, metric := range stringMetrics {
-		_, ok := availableColumns[metric]
-		if ok {
-			assert.True(t, acc.HasStringField("postgresql", metric))
-			metricsCounted++
-		}
+		assert.True(t, acc.HasStringField("postgresql", metric))
+		metricsCounted++
 	}
 
 	assert.True(t, metricsCounted > 0)
-	assert.Equal(t, len(availableColumns)-len(p.IgnoredColumns()), metricsCounted)
+	assert.Equal(t, len(floatMetrics)+len(intMetrics)+len(int32Metrics)+len(stringMetrics), metricsCounted)
 }
 
 func TestPostgresqlQueryOutputTests(t *testing.T) {
@@ -137,7 +126,7 @@ func TestPostgresqlQueryOutputTests(t *testing.T) {
 	}
 
 	for q, assertions := range examples {
-		_, acc := queryRunner(t, query{{
+		acc := queryRunner(t, query{{
 			Sqlquery:   q,
 			Version:    901,
 			Withdbname: false,
@@ -153,7 +142,7 @@ func TestPostgresqlFieldOutput(t *testing.T) {
 		t.Skip("Skipping integration test in short mode")
 	}
 
-	_, acc := queryRunner(t, query{{
+	acc := queryRunner(t, query{{
 		Sqlquery:   "select * from pg_stat_database",
 		Version:    901,
 		Withdbname: false,
@@ -216,13 +205,18 @@ func TestPostgresqlIgnoresUnwantedColumns(t *testing.T) {
 	}
 
 	p := &Postgresql{
-		Address: fmt.Sprintf("host=%s user=postgres sslmode=disable",
-			testutil.GetLocalHost()),
+		Service: postgresql.Service{
+			Address: fmt.Sprintf(
+				"host=%s user=postgres sslmode=disable",
+				testutil.GetLocalHost(),
+			),
+		},
 	}
 
 	var acc testutil.Accumulator
-	require.NoError(t, acc.GatherError(p.Gather))
 
+	require.NoError(t, p.Start(&acc))
+	require.NoError(t, acc.GatherError(p.Gather))
 	assert.NotEmpty(t, p.IgnoredColumns())
 	for col := range p.IgnoredColumns() {
 		assert.False(t, acc.HasMeasurement(col))

plugins/inputs/procstat/README.md

@@ -1,121 +1,138 @@
-# Telegraf plugin: procstat
+# Procstat Input Plugin
 
-#### Description
+The procstat plugin can be used to monitor the system resource usage of one or more processes.
 
-The procstat plugin can be used to monitor system resource usage by an
-individual process using their /proc data.
+Processes can be selected for monitoring using one of several methods:
+- pidfile
+- exe
+- pattern
+- user
+- systemd_unit
+- cgroup
 
-Processes can be specified either by pid file, by executable name, by command
-line pattern matching, by username, by systemd unit name, or by cgroup name/path
-(in this order or priority). Procstat plugin will use `pgrep` when executable
-name is provided to obtain the pid. Procstat plugin will transmit IO, memory,
-cpu, file descriptor related measurements for every process specified. A prefix
-can be set to isolate individual process specific measurements.
+### Configuration:
 
-The plugin will tag processes according to how they are specified in the configuration. If a pid file is used, a "pidfile" tag will be generated.
-On the other hand, if an executable is used an "exe" tag will be generated. Possible tag names:
-
-* pidfile
-* exe
-* pattern
-* user
-* systemd_unit
-* cgroup
-
-Additionally the plugin will tag processes by their PID (pid_tag = true in the config) and their process name:
-
-* pid
-* process_name
-
-Example:
-
-```
+```toml
+# Monitor process cpu and memory usage
 [[inputs.procstat]]
-  exe = "influxd"
-  prefix = "influxd"
+  ## PID file to monitor process
+  pid_file = "/var/run/nginx.pid"
+  ## executable name (ie, pgrep <exe>)
+  # exe = "nginx"
+  ## pattern as argument for pgrep (ie, pgrep -f <pattern>)
+  # pattern = "nginx"
+  ## user as argument for pgrep (ie, pgrep -u <user>)
+  # user = "nginx"
+  ## Systemd unit name
+  # systemd_unit = "nginx.service"
+  ## CGroup name or path
+  # cgroup = "systemd/system.slice/nginx.service"
 
+  ## override for process_name
+  ## This is optional; default is sourced from /proc/<pid>/status
+  # process_name = "bar"
+
+  ## Field name prefix
+  # prefix = ""
+
+  ## Add PID as a tag instead of a field; useful to differentiate between
+  ## processes whose tags are otherwise the same.  Can create a large number
+  ## of series, use judiciously.
+  # pid_tag = false
+
+  ## Method to use when finding process IDs.  Can be one of 'pgrep', or
+  ## 'native'.  The pgrep finder calls the pgrep executable in the PATH while
+  ## the native finder performs the search directly in a manor dependent on the
+  ## platform.  Default is 'pgrep'
+  # pid_finder = "pgrep"
+```
+
+#### Windows support
+
+Preliminary support for Windows has been added, however you may prefer using
+the `win_perf_counters` input plugin as a more mature alternative.
+
+When using the `pid_finder = "native"` in Windows, the pattern lookup method is
+implemented as a WMI query.  The pattern allows fuzzy matching using only
+[WMI query patterns](https://msdn.microsoft.com/en-us/library/aa392263(v=vs.85).aspx):
+```toml
 [[inputs.procstat]]
-  pid_file = "/var/run/lxc/dnsmasq.pid"
+  pattern = "%influx%"
+  pid_finder = "native"
 ```
 
-The above configuration would result in output like:
+### Metrics:
+
+- procstat
+  - tags:
+    - pid (when `pid_tag` is true)
+    - process_name
+    - pidfile (when defined)
+    - exe (when defined)
+    - pattern (when defined)
+    - user (when selected)
+    - systemd_unit (when defined)
+    - cgroup (when defined)
+  - fields:
+    - cpu_time (int)
+    - cpu_time_guest (float)
+    - cpu_time_guest_nice (float)
+    - cpu_time_idle (float)
+    - cpu_time_iowait (float)
+    - cpu_time_irq (float)
+    - cpu_time_nice (float)
+    - cpu_time_soft_irq (float)
+    - cpu_time_steal (float)
+    - cpu_time_stolen (float)
+    - cpu_time_system (float)
+    - cpu_time_user (float)
+    - cpu_usage (float)
+    - involuntary_context_switches (int)
+    - memory_data (int)
+    - memory_locked (int)
+    - memory_rss (int)
+    - memory_stack (int)
+    - memory_swap (int)
+    - memory_vms (int)
+    - nice_priority (int)
+    - num_fds (int, *telegraf* may need to be ran as **root**)
+    - num_threads (int)
+    - pid (int)
+    - read_bytes (int, *telegraf* may need to be ran as **root**)
+    - read_count (int, *telegraf* may need to be ran as **root**)
+    - realtime_priority (int)
+    - rlimit_cpu_time_hard (int)
+    - rlimit_cpu_time_soft (int)
+    - rlimit_file_locks_hard (int)
+    - rlimit_file_locks_soft (int)
+    - rlimit_memory_data_hard (int)
+    - rlimit_memory_data_soft (int)
+    - rlimit_memory_locked_hard (int)
+    - rlimit_memory_locked_soft (int)
+    - rlimit_memory_rss_hard (int)
+    - rlimit_memory_rss_soft (int)
+    - rlimit_memory_stack_hard (int)
+    - rlimit_memory_stack_soft (int)
+    - rlimit_memory_vms_hard (int)
+    - rlimit_memory_vms_soft (int)
+    - rlimit_nice_priority_hard (int)
+    - rlimit_nice_priority_soft (int)
+    - rlimit_num_fds_hard (int)
+    - rlimit_num_fds_soft (int)
+    - rlimit_realtime_priority_hard (int)
+    - rlimit_realtime_priority_soft (int)
+    - rlimit_signals_pending_hard (int)
+    - rlimit_signals_pending_soft (int)
+    - signals_pending (int)
+    - voluntary_context_switches (int)
+    - write_bytes (int, *telegraf* may need to be ran as **root**)
+    - write_count (int, *telegraf* may need to be ran as **root**)
+
+*NOTE: Resource limit > 2147483647 will be reported as 2147483647.*
+
+### Example Output:
 
 ```
-> procstat,pidfile=/var/run/lxc/dnsmasq.pid,process_name=dnsmasq rlimit_file_locks_soft=2147483647i,rlimit_signals_pending_hard=1758i,voluntary_context_switches=478i,read_bytes=307200i,cpu_time_user=0.01,cpu_time_guest=0,memory_swap=0i,memory_locked=0i,rlimit_num_fds_hard=4096i,rlimit_nice_priority_hard=0i,num_fds=11i,involuntary_context_switches=20i,read_count=23i,memory_rss=1388544i,rlimit_memory_rss_soft=2147483647i,rlimit_memory_rss_hard=2147483647i,nice_priority=20i,rlimit_cpu_time_hard=2147483647i,cpu_time=0i,write_bytes=0i,cpu_time_idle=0,cpu_time_nice=0,memory_data=229376i,memory_stack=135168i,rlimit_cpu_time_soft=2147483647i,rlimit_memory_data_hard=2147483647i,rlimit_memory_locked_hard=65536i,rlimit_signals_pending_soft=1758i,write_count=11i,cpu_time_iowait=0,cpu_time_steal=0,cpu_time_stolen=0,rlimit_memory_stack_soft=8388608i,cpu_time_system=0.02,cpu_time_guest_nice=0,rlimit_memory_locked_soft=65536i,rlimit_memory_vms_soft=2147483647i,rlimit_file_locks_hard=2147483647i,rlimit_realtime_priority_hard=0i,pid=828i,num_threads=1i,cpu_time_soft_irq=0,rlimit_memory_vms_hard=2147483647i,rlimit_realtime_priority_soft=0i,memory_vms=15884288i,rlimit_memory_stack_hard=2147483647i,cpu_time_irq=0,rlimit_memory_data_soft=2147483647i,rlimit_num_fds_soft=1024i,signals_pending=0i,rlimit_nice_priority_soft=0i,realtime_priority=0i
-> procstat,exe=influxd,process_name=influxd rlimit_num_fds_hard=16384i,rlimit_signals_pending_hard=1758i,realtime_priority=0i,rlimit_memory_vms_hard=2147483647i,rlimit_signals_pending_soft=1758i,cpu_time_stolen=0,rlimit_memory_stack_hard=2147483647i,rlimit_realtime_priority_hard=0i,cpu_time=0i,pid=500i,voluntary_context_switches=975i,cpu_time_idle=0,memory_rss=3072000i,memory_locked=0i,rlimit_nice_priority_soft=0i,signals_pending=0i,nice_priority=20i,read_bytes=823296i,cpu_time_soft_irq=0,rlimit_memory_data_hard=2147483647i,rlimit_memory_locked_soft=65536i,write_count=8i,cpu_time_irq=0,memory_vms=33501184i,rlimit_memory_stack_soft=8388608i,cpu_time_iowait=0,rlimit_memory_vms_soft=2147483647i,rlimit_nice_priority_hard=0i,num_fds=29i,memory_data=229376i,rlimit_cpu_time_soft=2147483647i,rlimit_file_locks_soft=2147483647i,num_threads=1i,write_bytes=0i,cpu_time_steal=0,rlimit_memory_rss_hard=2147483647i,cpu_time_guest=0,cpu_time_guest_nice=0,cpu_usage=0,rlimit_memory_locked_hard=65536i,rlimit_file_locks_hard=2147483647i,involuntary_context_switches=38i,read_count=16851i,memory_swap=0i,rlimit_memory_data_soft=2147483647i,cpu_time_user=0.11,rlimit_cpu_time_hard=2147483647i,rlimit_num_fds_soft=16384i,rlimit_realtime_priority_soft=0i,cpu_time_system=0.27,cpu_time_nice=0,memory_stack=135168i,rlimit_memory_rss_soft=2147483647i
+procstat,pidfile=/var/run/lxc/dnsmasq.pid,process_name=dnsmasq rlimit_file_locks_soft=2147483647i,rlimit_signals_pending_hard=1758i,voluntary_context_switches=478i,read_bytes=307200i,cpu_time_user=0.01,cpu_time_guest=0,memory_swap=0i,memory_locked=0i,rlimit_num_fds_hard=4096i,rlimit_nice_priority_hard=0i,num_fds=11i,involuntary_context_switches=20i,read_count=23i,memory_rss=1388544i,rlimit_memory_rss_soft=2147483647i,rlimit_memory_rss_hard=2147483647i,nice_priority=20i,rlimit_cpu_time_hard=2147483647i,cpu_time=0i,write_bytes=0i,cpu_time_idle=0,cpu_time_nice=0,memory_data=229376i,memory_stack=135168i,rlimit_cpu_time_soft=2147483647i,rlimit_memory_data_hard=2147483647i,rlimit_memory_locked_hard=65536i,rlimit_signals_pending_soft=1758i,write_count=11i,cpu_time_iowait=0,cpu_time_steal=0,cpu_time_stolen=0,rlimit_memory_stack_soft=8388608i,cpu_time_system=0.02,cpu_time_guest_nice=0,rlimit_memory_locked_soft=65536i,rlimit_memory_vms_soft=2147483647i,rlimit_file_locks_hard=2147483647i,rlimit_realtime_priority_hard=0i,pid=828i,num_threads=1i,cpu_time_soft_irq=0,rlimit_memory_vms_hard=2147483647i,rlimit_realtime_priority_soft=0i,memory_vms=15884288i,rlimit_memory_stack_hard=2147483647i,cpu_time_irq=0,rlimit_memory_data_soft=2147483647i,rlimit_num_fds_soft=1024i,signals_pending=0i,rlimit_nice_priority_soft=0i,realtime_priority=0i
+procstat,exe=influxd,process_name=influxd rlimit_num_fds_hard=16384i,rlimit_signals_pending_hard=1758i,realtime_priority=0i,rlimit_memory_vms_hard=2147483647i,rlimit_signals_pending_soft=1758i,cpu_time_stolen=0,rlimit_memory_stack_hard=2147483647i,rlimit_realtime_priority_hard=0i,cpu_time=0i,pid=500i,voluntary_context_switches=975i,cpu_time_idle=0,memory_rss=3072000i,memory_locked=0i,rlimit_nice_priority_soft=0i,signals_pending=0i,nice_priority=20i,read_bytes=823296i,cpu_time_soft_irq=0,rlimit_memory_data_hard=2147483647i,rlimit_memory_locked_soft=65536i,write_count=8i,cpu_time_irq=0,memory_vms=33501184i,rlimit_memory_stack_soft=8388608i,cpu_time_iowait=0,rlimit_memory_vms_soft=2147483647i,rlimit_nice_priority_hard=0i,num_fds=29i,memory_data=229376i,rlimit_cpu_time_soft=2147483647i,rlimit_file_locks_soft=2147483647i,num_threads=1i,write_bytes=0i,cpu_time_steal=0,rlimit_memory_rss_hard=2147483647i,cpu_time_guest=0,cpu_time_guest_nice=0,cpu_usage=0,rlimit_memory_locked_hard=65536i,rlimit_file_locks_hard=2147483647i,involuntary_context_switches=38i,read_count=16851i,memory_swap=0i,rlimit_memory_data_soft=2147483647i,cpu_time_user=0.11,rlimit_cpu_time_hard=2147483647i,rlimit_num_fds_soft=16384i,rlimit_realtime_priority_soft=0i,cpu_time_system=0.27,cpu_time_nice=0,memory_stack=135168i,rlimit_memory_rss_soft=2147483647i
 ```
-
-# Measurements
-Note: prefix can be set by the user, per process.
-
-
-Threads related measurement names:
-- procstat_[prefix_]num_threads value=5
-
-File descriptor related measurement names (*telegraf* needs to run as **root**):
-- procstat_[prefix_]num_fds value=4
-
-Priority related measurement names:
-- procstat_[prefix_]realtime_priority value=0
-- procstat_[prefix_]nice_priority value=20
-
-Signals related measurement names:
-- procstat_[prefix_]signals_pending value=0
-
-Context switch related measurement names:
-- procstat_[prefix_]voluntary_context_switches value=250
-- procstat_[prefix_]involuntary_context_switches value=0
-
-I/O related measurement names (*telegraf* needs to run as **root**):
-- procstat_[prefix_]read_count value=396
-- procstat_[prefix_]write_count value=1
-- procstat_[prefix_]read_bytes value=1019904
-- procstat_[prefix_]write_bytes value=1
-
-CPU related measurement names:
-- procstat_[prefix_]cpu_time value=0.01
-- procstat_[prefix_]cpu_time_user value=0
-- procstat_[prefix_]cpu_time_system value=0.01
-- procstat_[prefix_]cpu_time_idle value=0
-- procstat_[prefix_]cpu_time_nice value=0
-- procstat_[prefix_]cpu_time_iowait value=0
-- procstat_[prefix_]cpu_time_irq value=0
-- procstat_[prefix_]cpu_time_soft_irq value=0
-- procstat_[prefix_]cpu_time_steal value=0
-- procstat_[prefix_]cpu_time_stolen value=0
-- procstat_[prefix_]cpu_time_guest value=0
-- procstat_[prefix_]cpu_time_guest_nice value=0
-
-Memory related measurement names:
-- procstat_[prefix_]memory_rss value=1777664
-- procstat_[prefix_]memory_vms value=24227840
-- procstat_[prefix_]memory_swap value=282624
-- procstat_[prefix_]memory_data value=229376
-- procstat_[prefix_]memory_stack value=135168
-- procstat_[prefix_]memory_locked value=0
-
-Resource limits:
-- procstat_[prefix_]rlimit_cpu_time_hard value=2147483647
-- procstat_[prefix_]rlimit_cpu_time_soft value=2147483647
-- procstat_[prefix_]rlimit_file_locks_hard value=2147483647
-- procstat_[prefix_]rlimit_file_locks_soft value=2147483647
-- procstat_[prefix_]rlimit_memory_data_hard value=2147483647
-- procstat_[prefix_]rlimit_memory_data_soft value=2147483647
-- procstat_[prefix_]rlimit_memory_locked_hard value=65536
-- procstat_[prefix_]rlimit_memory_locked_soft value=65536
-- procstat_[prefix_]rlimit_memory_rss_hard value=2147483647
-- procstat_[prefix_]rlimit_memory_rss_soft value=2147483647
-- procstat_[prefix_]rlimit_memory_stack_hard value=2147483647
-- procstat_[prefix_]rlimit_memory_stack_soft value=8388608
-- procstat_[prefix_]rlimit_memory_vms_hard value=2147483647
-- procstat_[prefix_]rlimit_memory_vms_soft value=2147483647
-- procstat_[prefix_]rlimit_nice_priority_hard value=0
-- procstat_[prefix_]rlimit_nice_priority_soft value=0
-- procstat_[prefix_]rlimit_num_fds_hard value=16384
-- procstat_[prefix_]rlimit_num_fds_soft value=16384
-- procstat_[prefix_]rlimit_realtime_priority_hard value=0
-- procstat_[prefix_]rlimit_realtime_priority_soft value=0
-- procstat_[prefix_]rlimit_signals_pending_hard value=1758
-- procstat_[prefix_]rlimit_signals_pending_soft value=1758
-
-*NOTE: Due to a limitation in an underlying library Telegraf uses, any resource limit > 2147483647 will be misreported as 2147483647.*

plugins/inputs/procstat/native_finder.go

@@ -0,0 +1,57 @@
+package procstat
+
+import (
+	"fmt"
+	"io/ioutil"
+	"strconv"
+	"strings"
+
+	"github.com/shirou/gopsutil/process"
+)
+
+//NativeFinder uses gopsutil to find processes
+type NativeFinder struct {
+}
+
+//NewNativeFinder ...
+func NewNativeFinder() (PIDFinder, error) {
+	return &NativeFinder{}, nil
+}
+
+//Uid will return all pids for the given user
+func (pg *NativeFinder) Uid(user string) ([]PID, error) {
+	var dst []PID
+	procs, err := process.Processes()
+	if err != nil {
+		return dst, err
+	}
+	for _, p := range procs {
+		username, err := p.Username()
+		if err != nil {
+			//skip, this can happen if we don't have permissions or
+			//the pid no longer exists
+			continue
+		}
+		if username == user {
+			dst = append(dst, PID(p.Pid))
+		}
+	}
+	return dst, nil
+}
+
+//PidFile returns the pid from the pid file given.
+func (pg *NativeFinder) PidFile(path string) ([]PID, error) {
+	var pids []PID
+	pidString, err := ioutil.ReadFile(path)
+	if err != nil {
+		return pids, fmt.Errorf("Failed to read pidfile '%s'. Error: '%s'",
+			path, err)
+	}
+	pid, err := strconv.Atoi(strings.TrimSpace(string(pidString)))
+	if err != nil {
+		return pids, err
+	}
+	pids = append(pids, PID(pid))
+	return pids, nil
+
+}

plugins/inputs/procstat/native_finder_notwindows.go

@@ -0,0 +1,59 @@
+// +build !windows
+
+package procstat
+
+import (
+	"regexp"
+
+	"github.com/shirou/gopsutil/process"
+)
+
+//Pattern matches on the process name
+func (pg *NativeFinder) Pattern(pattern string) ([]PID, error) {
+	var pids []PID
+	regxPattern, err := regexp.Compile(pattern)
+	if err != nil {
+		return pids, err
+	}
+	procs, err := process.Processes()
+	if err != nil {
+		return pids, err
+	}
+	for _, p := range procs {
+		name, err := p.Exe()
+		if err != nil {
+			//skip, this can be caused by the pid no longer existing
+			//or you having no permissions to access it
+			continue
+		}
+		if regxPattern.MatchString(name) {
+			pids = append(pids, PID(p.Pid))
+		}
+	}
+	return pids, err
+}
+
+//FullPattern matches on the command line when the proccess was executed
+func (pg *NativeFinder) FullPattern(pattern string) ([]PID, error) {
+	var pids []PID
+	regxPattern, err := regexp.Compile(pattern)
+	if err != nil {
+		return pids, err
+	}
+	procs, err := process.Processes()
+	if err != nil {
+		return pids, err
+	}
+	for _, p := range procs {
+		cmd, err := p.Cmdline()
+		if err != nil {
+			//skip, this can be caused by the pid no longer existing
+			//or you having no permissions to access it
+			continue
+		}
+		if regxPattern.MatchString(cmd) {
+			pids = append(pids, PID(p.Pid))
+		}
+	}
+	return pids, err
+}

plugins/inputs/procstat/native_finder_windows.go

@@ -0,0 +1,91 @@
+package procstat
+
+import (
+	"context"
+	"fmt"
+	"regexp"
+	"time"
+
+	"github.com/StackExchange/wmi"
+	"github.com/shirou/gopsutil/process"
+)
+
+//Timeout is the timeout used when making wmi calls
+var Timeout = 5 * time.Second
+
+type queryType string
+
+const (
+	like     = queryType("LIKE")
+	equals   = queryType("=")
+	notEqual = queryType("!=")
+)
+
+//Pattern matches on the process name
+func (pg *NativeFinder) Pattern(pattern string) ([]PID, error) {
+	var pids []PID
+	regxPattern, err := regexp.Compile(pattern)
+	if err != nil {
+		return pids, err
+	}
+	procs, err := process.Processes()
+	if err != nil {
+		return pids, err
+	}
+	for _, p := range procs {
+		name, err := p.Name()
+		if err != nil {
+			//skip, this can be caused by the pid no longer existing
+			//or you having no permissions to access it
+			continue
+		}
+		if regxPattern.MatchString(name) {
+			pids = append(pids, PID(p.Pid))
+		}
+	}
+	return pids, err
+}
+
+//FullPattern matches the cmdLine on windows and will find a pattern using a WMI like query
+func (pg *NativeFinder) FullPattern(pattern string) ([]PID, error) {
+	var pids []PID
+	procs, err := getWin32ProcsByVariable("CommandLine", like, pattern, Timeout)
+	if err != nil {
+		return pids, err
+	}
+	for _, p := range procs {
+		pids = append(pids, PID(p.ProcessID))
+	}
+	return pids, nil
+}
+
+//GetWin32ProcsByVariable allows you to query any variable with a like query
+func getWin32ProcsByVariable(variable string, qType queryType, value string, timeout time.Duration) ([]process.Win32_Process, error) {
+	var dst []process.Win32_Process
+	var query string
+	// should look like "WHERE CommandLine LIKE "procstat"
+	query = fmt.Sprintf("WHERE %s %s %q", variable, qType, value)
+	q := wmi.CreateQuery(&dst, query)
+	ctx, cancel := context.WithTimeout(context.Background(), timeout)
+	defer cancel()
+	err := WMIQueryWithContext(ctx, q, &dst)
+	if err != nil {
+		return []process.Win32_Process{}, fmt.Errorf("could not get win32Proc: %s", err)
+	}
+	return dst, nil
+}
+
+// WMIQueryWithContext - wraps wmi.Query with a timed-out context to avoid hanging
+func WMIQueryWithContext(ctx context.Context, query string, dst interface{}, connectServerArgs ...interface{}) error {
+	errChan := make(chan error, 1)
+	go func() {
+		errChan <- wmi.Query(query, dst, connectServerArgs...)
+	}()
+
+	select {
+	case <-ctx.Done():
+		return ctx.Err()
+	case err := <-errChan:
+		return err
+	}
+}

plugins/inputs/procstat/native_finder_windows_test.go

@@ -0,0 +1,40 @@
+package procstat
+
+import (
+	"fmt"
+	"testing"
+
+	"os/user"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestGather_RealPattern(t *testing.T) {
+	pg, err := NewNativeFinder()
+	require.NoError(t, err)
+	pids, err := pg.Pattern(`procstat`)
+	require.NoError(t, err)
+	fmt.Println(pids)
+	assert.Equal(t, len(pids) > 0, true)
+}
+
+func TestGather_RealFullPattern(t *testing.T) {
+	pg, err := NewNativeFinder()
+	require.NoError(t, err)
+	pids, err := pg.FullPattern(`%procstat%`)
+	require.NoError(t, err)
+	fmt.Println(pids)
+	assert.Equal(t, len(pids) > 0, true)
+}
+
+func TestGather_RealUser(t *testing.T) {
+	user, err := user.Current()
+	require.NoError(t, err)
+	pg, err := NewNativeFinder()
+	require.NoError(t, err)
+	pids, err := pg.Uid(user.Username)
+	require.NoError(t, err)
+	fmt.Println(pids)
+	assert.Equal(t, len(pids) > 0, true)
+}

plugins/inputs/procstat/pgrep.go

@@ -8,13 +8,6 @@ import (
 	"strings"
 )
 
-type PIDFinder interface {
-	PidFile(path string) ([]PID, error)
-	Pattern(pattern string) ([]PID, error)
-	Uid(user string) ([]PID, error)
-	FullPattern(path string) ([]PID, error)
-}
-
 // Implemention of PIDGatherer that execs pgrep to find processes
 type Pgrep struct {
 	path string

plugins/inputs/procstat/process.go

@@ -23,6 +23,13 @@ type Process interface {
 	RlimitUsage(bool) ([]process.RlimitStat, error)
 }
 
+type PIDFinder interface {
+	PidFile(path string) ([]PID, error)
+	Pattern(pattern string) ([]PID, error)
+	Uid(user string) ([]PID, error)
+	FullPattern(path string) ([]PID, error)
+}
+
 type Proc struct {
 	hasCPUTimes bool
 	tags        map[string]string

plugins/inputs/procstat/procstat.go

@@ -22,6 +22,7 @@ var (
 type PID int32
 
 type Procstat struct {
+	PidFinder   string `toml:"pid_finder"`
 	PidFile     string `toml:"pid_file"`
 	Exe         string
 	Pattern     string
@@ -32,14 +33,14 @@ type Procstat struct {
 	CGroup      string `toml:"cgroup"`
 	PidTag      bool
 
-	pidFinder       PIDFinder
+	finder PIDFinder
+
 	createPIDFinder func() (PIDFinder, error)
 	procs           map[PID]Process
 	createProcess   func(PID) (Process, error)
 }
 
 var sampleConfig = `
-  ## Must specify one of: pid_file, exe, or pattern
   ## PID file to monitor process
   pid_file = "/var/run/nginx.pid"
   ## executable name (ie, pgrep <exe>)
@@ -56,12 +57,20 @@ var sampleConfig = `
   ## override for process_name
   ## This is optional; default is sourced from /proc/<pid>/status
   # process_name = "bar"
+
   ## Field name prefix
-  prefix = ""
-  ## comment this out if you want raw cpu_time stats
-  fielddrop = ["cpu_time_*"]
-  ## This is optional; moves pid into a tag instead of a field
-  pid_tag = false
+  # prefix = ""
+
+  ## Add PID as a tag instead of a field; useful to differentiate between
+  ## processes whose tags are otherwise the same.  Can create a large number
+  ## of series, use judiciously.
+  # pid_tag = false
+
+  ## Method to use when finding process IDs.  Can be one of 'pgrep', or
+  ## 'native'.  The pgrep finder calls the pgrep executable in the PATH while
+  ## the native finder performs the search directly in a manor dependent on the
+  ## platform.  Default is 'pgrep'
+  # pid_finder = "pgrep"
 `
 
 func (_ *Procstat) SampleConfig() string {
@@ -74,7 +83,15 @@ func (_ *Procstat) Description() string {
 
 func (p *Procstat) Gather(acc telegraf.Accumulator) error {
 	if p.createPIDFinder == nil {
-		p.createPIDFinder = defaultPIDFinder
+		switch p.PidFinder {
+		case "native":
+			p.createPIDFinder = NewNativeFinder
+		case "pgrep":
+			p.createPIDFinder = NewPgrep
+		default:
+			p.createPIDFinder = defaultPIDFinder
+		}
+
 	}
 	if p.createProcess == nil {
 		p.createProcess = defaultProcess
@@ -252,14 +269,15 @@ func (p *Procstat) updateProcesses(prevInfo map[PID]Process) (map[PID]Process, e
 
 // Create and return PIDGatherer lazily
 func (p *Procstat) getPIDFinder() (PIDFinder, error) {
-	if p.pidFinder == nil {
+
+	if p.finder == nil {
 		f, err := p.createPIDFinder()
 		if err != nil {
 			return nil, err
 		}
-		p.pidFinder = f
+		p.finder = f
 	}
-	return p.pidFinder, nil
+	return p.finder, nil
 }
 
 // Get matching PIDs and their initial tags
@@ -292,7 +310,7 @@ func (p *Procstat) findPids() ([]PID, map[string]string, error) {
 		pids, err = p.cgroupPIDs()
 		tags = map[string]string{"cgroup": p.CGroup}
 	} else {
-		err = fmt.Errorf("Either exe, pid_file, user, or pattern has to be specified")
+		err = fmt.Errorf("Either exe, pid_file, user, pattern, systemd_unit, or cgroup must be specified")
 	}
 
 	return pids, tags, err

plugins/inputs/procstat/procstat_test.go

@@ -6,6 +6,7 @@ import (
 	"os"
 	"os/exec"
 	"path/filepath"
+	"runtime"
 	"strings"
 	"testing"
 	"time"
@@ -349,6 +350,10 @@ func TestGather_systemdUnitPIDs(t *testing.T) {
 }
 
 func TestGather_cgroupPIDs(t *testing.T) {
+	//no cgroups in windows
+	if runtime.GOOS == "windows" {
+		t.Skip("no cgroups in windows")
+	}
 	td, err := ioutil.TempDir("", "")
 	require.NoError(t, err)
 	defer os.RemoveAll(td)

plugins/inputs/prometheus/README.md

@@ -67,7 +67,7 @@ Measurement names are based on the Metric Family and tags are created for each
 label.  The value is added to a field named based on the metric type.
 
 All metrics receive the `url` tag indicating the related URL specified in the
-Telegraf configuration.  If using Kubernetes service discovery the `address`
+Telegraf configuration. If using Kubernetes service discovery the `address`
 tag is also added indicating the discovered ip address.
 
 ### Example Output:

plugins/inputs/prometheus/prometheus.go

@@ -20,7 +20,7 @@ const acceptHeader = `application/vnd.google.protobuf;proto=io.prometheus.client
 
 type Prometheus struct {
 	// An array of urls to scrape metrics from.
-	Urls []string
+	URLs []string `toml:"urls"`
 
 	// An array of Kubernetes services to scrape metrics from.
 	KubernetesServices []string
@@ -73,12 +73,12 @@ func (p *Prometheus) Description() string {
 
 var ErrProtocolError = errors.New("prometheus protocol error")
 
-func (p *Prometheus) AddressToURL(u *url.URL, address string) string {
+func (p *Prometheus) AddressToURL(u *url.URL, address string) *url.URL {
 	host := address
 	if u.Port() != "" {
 		host = address + ":" + u.Port()
 	}
-	reconstructedUrl := url.URL{
+	reconstructedURL := &url.URL{
 		Scheme:     u.Scheme,
 		Opaque:     u.Opaque,
 		User:       u.User,
@@ -89,36 +89,42 @@ func (p *Prometheus) AddressToURL(u *url.URL, address string) string {
 		Fragment:   u.Fragment,
 		Host:       host,
 	}
-	return reconstructedUrl.String()
+	return reconstructedURL
 }
 
-type UrlAndAddress struct {
-	OriginalUrl string
-	Url         string
+type URLAndAddress struct {
+	OriginalURL *url.URL
+	URL         *url.URL
 	Address     string
 }
 
-func (p *Prometheus) GetAllURLs() ([]UrlAndAddress, error) {
-	allUrls := make([]UrlAndAddress, 0)
-	for _, url := range p.Urls {
-		allUrls = append(allUrls, UrlAndAddress{Url: url, OriginalUrl: url})
+func (p *Prometheus) GetAllURLs() ([]URLAndAddress, error) {
+	allURLs := make([]URLAndAddress, 0)
+	for _, u := range p.URLs {
+		URL, err := url.Parse(u)
+		if err != nil {
+			log.Printf("prometheus: Could not parse %s, skipping it. Error: %s", u, err)
+			continue
+		}
+
+		allURLs = append(allURLs, URLAndAddress{URL: URL, OriginalURL: URL})
 	}
 	for _, service := range p.KubernetesServices {
-		u, err := url.Parse(service)
+		URL, err := url.Parse(service)
 		if err != nil {
 			return nil, err
 		}
-		resolvedAddresses, err := net.LookupHost(u.Hostname())
+		resolvedAddresses, err := net.LookupHost(URL.Hostname())
 		if err != nil {
-			log.Printf("prometheus: Could not resolve %s, skipping it. Error: %s", u.Host, err)
+			log.Printf("prometheus: Could not resolve %s, skipping it. Error: %s", URL.Host, err)
 			continue
 		}
 		for _, resolved := range resolvedAddresses {
-			serviceUrl := p.AddressToURL(u, resolved)
-			allUrls = append(allUrls, UrlAndAddress{Url: serviceUrl, Address: resolved, OriginalUrl: service})
+			serviceURL := p.AddressToURL(URL, resolved)
+			allURLs = append(allURLs, URLAndAddress{URL: serviceURL, Address: resolved, OriginalURL: URL})
 		}
 	}
-	return allUrls, nil
+	return allURLs, nil
 }
 
 // Reads stats from all configured servers accumulates stats.
@@ -134,16 +140,16 @@ func (p *Prometheus) Gather(acc telegraf.Accumulator) error {
 
 	var wg sync.WaitGroup
 
-	allUrls, err := p.GetAllURLs()
+	allURLs, err := p.GetAllURLs()
 	if err != nil {
 		return err
 	}
-	for _, url := range allUrls {
+	for _, URL := range allURLs {
 		wg.Add(1)
-		go func(serviceUrl UrlAndAddress) {
+		go func(serviceURL URLAndAddress) {
 			defer wg.Done()
-			acc.AddError(p.gatherURL(serviceUrl, acc))
-		}(url)
+			acc.AddError(p.gatherURL(serviceURL, acc))
+		}(URL)
 	}
 
 	wg.Wait()
@@ -178,8 +184,8 @@ func (p *Prometheus) createHttpClient() (*http.Client, error) {
 	return client, nil
 }
 
-func (p *Prometheus) gatherURL(url UrlAndAddress, acc telegraf.Accumulator) error {
-	var req, err = http.NewRequest("GET", url.Url, nil)
+func (p *Prometheus) gatherURL(u URLAndAddress, acc telegraf.Accumulator) error {
+	var req, err = http.NewRequest("GET", u.URL.String(), nil)
 	req.Header.Add("Accept", acceptHeader)
 	var token []byte
 	var resp *http.Response
@@ -194,11 +200,11 @@ func (p *Prometheus) gatherURL(url UrlAndAddress, acc telegraf.Accumulator) erro
 
 	resp, err = p.client.Do(req)
 	if err != nil {
-		return fmt.Errorf("error making HTTP request to %s: %s", url.Url, err)
+		return fmt.Errorf("error making HTTP request to %s: %s", u.URL, err)
 	}
 	defer resp.Body.Close()
 	if resp.StatusCode != http.StatusOK {
-		return fmt.Errorf("%s returned HTTP status %s", url.Url, resp.Status)
+		return fmt.Errorf("%s returned HTTP status %s", u.URL, resp.Status)
 	}
 
 	body, err := ioutil.ReadAll(resp.Body)
@@ -209,14 +215,16 @@ func (p *Prometheus) gatherURL(url UrlAndAddress, acc telegraf.Accumulator) erro
 	metrics, err := Parse(body, resp.Header)
 	if err != nil {
 		return fmt.Errorf("error reading metrics for %s: %s",
-			url.Url, err)
+			u.URL, err)
 	}
 	// Add (or not) collected metrics
 	for _, metric := range metrics {
 		tags := metric.Tags()
-		tags["url"] = url.OriginalUrl
-		if url.Address != "" {
-			tags["address"] = url.Address
+		// strip user and password from URL
+		u.OriginalURL.User = nil
+		tags["url"] = u.OriginalURL.String()
+		if u.Address != "" {
+			tags["address"] = u.Address
 		}
 
 		switch metric.Type() {

plugins/inputs/prometheus/prometheus_test.go

@@ -37,7 +37,7 @@ func TestPrometheusGeneratesMetrics(t *testing.T) {
 	defer ts.Close()
 
 	p := &Prometheus{
-		Urls: []string{ts.URL},
+		URLs: []string{ts.URL},
 	}
 
 	var acc testutil.Accumulator
@@ -89,7 +89,7 @@ func TestPrometheusGeneratesMetricsAlthoughFirstDNSFails(t *testing.T) {
 	defer ts.Close()
 
 	p := &Prometheus{
-		Urls:               []string{ts.URL},
+		URLs:               []string{ts.URL},
 		KubernetesServices: []string{"http://random.telegraf.local:88/metrics"},
 	}
 

plugins/inputs/rabbitmq/README.md

@@ -40,6 +40,15 @@ For additional details reference the [RabbitMQ Management HTTP Stats](https://cd
   ## A list of queues to gather as the rabbitmq_queue measurement. If not
   ## specified, metrics for all queues are gathered.
   # queues = ["telegraf"]
+
+  ## A list of exchanges to gather as the rabbitmq_exchange measurement. If not
+  ## specified, metrics for all exchanges are gathered.
+  # exchanges = ["telegraf"]
+
+  ## Queues to include and exclude. Globs accepted.
+  ## Note that an empty array for both will include all queues
+  # queue_name_include = []
+  # queue_name_exclude = []
 ```
 
 ### Measurements & Fields:
@@ -52,10 +61,13 @@ For additional details reference the [RabbitMQ Management HTTP Stats](https://cd
   - messages (int, messages)
   - messages_acked (int, messages)
   - messages_delivered (int, messages)
+  - messages_delivered_get (int, messages)
   - messages_published (int, messages)
   - messages_ready (int, messages)
   - messages_unacked (int, messages)
   - queues (int, queues)
+  - clustering_listeners (int, cluster nodes)
+  - amqp_listeners (int, amqp nodes up)
 
 - rabbitmq_node
   - disk_free (int, bytes)
@@ -69,6 +81,7 @@ For additional details reference the [RabbitMQ Management HTTP Stats](https://cd
   - run_queue (int, erlang processes)
   - sockets_total (int, sockets)
   - sockets_used (int, sockets)
+  - running (int, node up)
 
 - rabbitmq_queue
   - consumer_utilisation (float, percent)
@@ -94,6 +107,10 @@ For additional details reference the [RabbitMQ Management HTTP Stats](https://cd
   - messages_redeliver_rate (float, messages per second)
   - messages_unack (integer, count)
 
+- rabbitmq_exchange
+  - messages_publish_in (int, count)
+  - messages_publish_out (int, count)
+
 ### Tags:
 
 - All measurements have the following tags:
@@ -113,13 +130,29 @@ For additional details reference the [RabbitMQ Management HTTP Stats](https://cd
   - durable
   - auto_delete
 
+- rabbitmq_exchange
+  - url
+  - exchange
+  - type
+  - vhost
+  - internal
+  - durable
+  - auto_delete
+
 ### Sample Queries:
 
+Message rates for the entire node can be calculated from total message counts. For instance, to get the rate of messages published per minute, use this query:
+
+```
+SELECT NON_NEGATIVE_DERIVATIVE(LAST("messages_published"), 1m) AS messages_published_rate
+FROM rabbitmq_overview WHERE time > now() - 10m GROUP BY time(1m)
+```
 
 ### Example Output:
 
 ```
 rabbitmq_queue,url=http://amqp.example.org:15672,queue=telegraf,vhost=influxdb,node=rabbit@amqp.example.org,durable=true,auto_delete=false,host=amqp.example.org messages_deliver_get=0i,messages_publish=329i,messages_publish_rate=0.2,messages_redeliver_rate=0,message_bytes_ready=0i,message_bytes_unacked=0i,messages_deliver=329i,messages_unack=0i,consumers=1i,idle_since="",messages=0i,messages_deliver_rate=0.2,messages_deliver_get_rate=0.2,messages_redeliver=0i,memory=43032i,message_bytes_ram=0i,messages_ack=329i,messages_ready=0i,messages_ack_rate=0.2,consumer_utilisation=1,message_bytes=0i,message_bytes_persist=0i 1493684035000000000
-rabbitmq_overview,url=http://amqp.example.org:15672,host=amqp.example.org channels=2i,consumers=1i,exchanges=17i,messages_acked=329i,messages=0i,messages_ready=0i,messages_unacked=0i,connections=2i,queues=1i,messages_delivered=329i,messages_published=329i 1493684035000000000
-rabbitmq_node,url=http://amqp.example.org:15672,node=rabbit@amqp.example.org,host=amqp.example.org fd_total=1024i,fd_used=32i,mem_limit=8363329126i,sockets_total=829i,disk_free=8175935488i,disk_free_limit=50000000i,mem_used=58771080i,proc_total=1048576i,proc_used=267i,run_queue=0i,sockets_used=2i 149368403500000000
+rabbitmq_overview,url=http://amqp.example.org:15672,host=amqp.example.org channels=2i,consumers=1i,exchanges=17i,messages_acked=329i,messages=0i,messages_ready=0i,messages_unacked=0i,connections=2i,queues=1i,messages_delivered=329i,messages_published=329i,clustering_listeners=2i,amqp_listeners=1i 1493684035000000000
+rabbitmq_node,url=http://amqp.example.org:15672,node=rabbit@amqp.example.org,host=amqp.example.org fd_total=1024i,fd_used=32i,mem_limit=8363329126i,sockets_total=829i,disk_free=8175935488i,disk_free_limit=50000000i,mem_used=58771080i,proc_total=1048576i,proc_used=267i,run_queue=0i,sockets_used=2i,running=1i 149368403500000000
+rabbitmq_exchange,url=http://amqp.example.org:15672,exchange=telegraf,type=fanout,vhost=influxdb,internal=false,durable=true,auto_delete=false,host=amqp.example.org messages_publish_in=2i,messages_publish_out=1i 149368403500000000
 ```

plugins/inputs/rabbitmq/rabbitmq.go

@@ -9,6 +9,7 @@ import (
 	"time"
 
 	"github.com/influxdata/telegraf"
+	"github.com/influxdata/telegraf/filter"
 	"github.com/influxdata/telegraf/internal"
 	"github.com/influxdata/telegraf/plugins/inputs"
 )
@@ -48,10 +49,18 @@ type RabbitMQ struct {
 	ResponseHeaderTimeout internal.Duration `toml:"header_timeout"`
 	ClientTimeout         internal.Duration `toml:"client_timeout"`
 
-	Nodes  []string
-	Queues []string
+	Nodes     []string
+	Queues    []string
+	Exchanges []string
+
+	QueueInclude []string `toml:"queue_name_include"`
+	QueueExclude []string `toml:"queue_name_exclude"`
 
 	Client *http.Client
+
+	filterCreated     bool
+	excludeEveryQueue bool
+	queueFilter       filter.Filter
 }
 
 // OverviewResponse ...
@@ -59,6 +68,12 @@ type OverviewResponse struct {
 	MessageStats *MessageStats `json:"message_stats"`
 	ObjectTotals *ObjectTotals `json:"object_totals"`
 	QueueTotals  *QueueTotals  `json:"queue_totals"`
+	Listeners    []Listeners   `json:"listeners"`
+}
+
+// Listeners ...
+type Listeners struct {
+	Protocol string `json:"protocol"`
 }
 
 // Details ...
@@ -72,12 +87,14 @@ type MessageStats struct {
 	AckDetails        Details `json:"ack_details"`
 	Deliver           int64
 	DeliverDetails    Details `json:"deliver_details"`
-	DeliverGet        int64
+	DeliverGet        int64   `json:"deliver_get"`
 	DeliverGetDetails Details `json:"deliver_get_details"`
 	Publish           int64
 	PublishDetails    Details `json:"publish_details"`
 	Redeliver         int64
 	RedeliverDetails  Details `json:"redeliver_details"`
+	PublishIn         int64   `json:"publish_in"`
+	PublishOut        int64   `json:"publish_out"`
 }
 
 // ObjectTotals ...
@@ -131,12 +148,23 @@ type Node struct {
 	RunQueue      int64 `json:"run_queue"`
 	SocketsTotal  int64 `json:"sockets_total"`
 	SocketsUsed   int64 `json:"sockets_used"`
+	Running       bool  `json:"running"`
+}
+
+type Exchange struct {
+	Name         string
+	MessageStats `json:"message_stats"`
+	Type         string
+	Internal     bool
+	Vhost        string
+	Durable      bool
+	AutoDelete   bool `json:"auto_delete"`
 }
 
 // gatherFunc ...
 type gatherFunc func(r *RabbitMQ, acc telegraf.Accumulator)
 
-var gatherFunctions = []gatherFunc{gatherOverview, gatherNodes, gatherQueues}
+var gatherFunctions = []gatherFunc{gatherOverview, gatherNodes, gatherQueues, gatherExchanges}
 
 var sampleConfig = `
   ## Management Plugin url. (default: http://localhost:15672)
@@ -171,6 +199,15 @@ var sampleConfig = `
   ## A list of queues to gather as the rabbitmq_queue measurement. If not
   ## specified, metrics for all queues are gathered.
   # queues = ["telegraf"]
+
+  ## A list of exchanges to gather as the rabbitmq_exchange measurement. If not
+  ## specified, metrics for all exchanges are gathered.
+  # exchanges = ["telegraf"]
+
+  ## Queues to include and exclude. Globs accepted.
+  ## Note that an empty array for both will include all queues
+  queue_name_include = []
+  queue_name_exclude = []
 `
 
 // SampleConfig ...
@@ -201,6 +238,15 @@ func (r *RabbitMQ) Gather(acc telegraf.Accumulator) error {
 		}
 	}
 
+	// Create queue filter if not already created
+	if !r.filterCreated {
+		err := r.createQueueFilter()
+		if err != nil {
+			return err
+		}
+		r.filterCreated = true
+	}
+
 	var wg sync.WaitGroup
 	wg.Add(len(gatherFunctions))
 	for _, f := range gatherFunctions {
@@ -258,27 +304,39 @@ func gatherOverview(r *RabbitMQ, acc telegraf.Accumulator) {
 		return
 	}
 
-	if overview.QueueTotals == nil || overview.ObjectTotals == nil || overview.MessageStats == nil {
+	if overview.QueueTotals == nil || overview.ObjectTotals == nil || overview.MessageStats == nil || overview.Listeners == nil {
 		acc.AddError(fmt.Errorf("Wrong answer from rabbitmq. Probably auth issue"))
 		return
 	}
 
+	var clustering_listeners, amqp_listeners int64 = 0, 0
+	for _, listener := range overview.Listeners {
+		if listener.Protocol == "clustering" {
+			clustering_listeners++
+		} else if listener.Protocol == "amqp" {
+			amqp_listeners++
+		}
+	}
+
 	tags := map[string]string{"url": r.URL}
 	if r.Name != "" {
 		tags["name"] = r.Name
 	}
 	fields := map[string]interface{}{
-		"messages":           overview.QueueTotals.Messages,
-		"messages_ready":     overview.QueueTotals.MessagesReady,
-		"messages_unacked":   overview.QueueTotals.MessagesUnacknowledged,
-		"channels":           overview.ObjectTotals.Channels,
-		"connections":        overview.ObjectTotals.Connections,
-		"consumers":          overview.ObjectTotals.Consumers,
-		"exchanges":          overview.ObjectTotals.Exchanges,
-		"queues":             overview.ObjectTotals.Queues,
-		"messages_acked":     overview.MessageStats.Ack,
-		"messages_delivered": overview.MessageStats.Deliver,
-		"messages_published": overview.MessageStats.Publish,
+		"messages":               overview.QueueTotals.Messages,
+		"messages_ready":         overview.QueueTotals.MessagesReady,
+		"messages_unacked":       overview.QueueTotals.MessagesUnacknowledged,
+		"channels":               overview.ObjectTotals.Channels,
+		"connections":            overview.ObjectTotals.Connections,
+		"consumers":              overview.ObjectTotals.Consumers,
+		"exchanges":              overview.ObjectTotals.Exchanges,
+		"queues":                 overview.ObjectTotals.Queues,
+		"messages_acked":         overview.MessageStats.Ack,
+		"messages_delivered":     overview.MessageStats.Deliver,
+		"messages_delivered_get": overview.MessageStats.DeliverGet,
+		"messages_published":     overview.MessageStats.Publish,
+		"clustering_listeners":   clustering_listeners,
+		"amqp_listeners":         amqp_listeners,
 	}
 	acc.AddFields("rabbitmq_overview", fields, tags)
 }
@@ -301,6 +359,11 @@ func gatherNodes(r *RabbitMQ, acc telegraf.Accumulator) {
 		tags := map[string]string{"url": r.URL}
 		tags["node"] = node.Name
 
+		var running int64 = 0
+		if node.Running {
+			running = 1
+		}
+
 		fields := map[string]interface{}{
 			"disk_free":       node.DiskFree,
 			"disk_free_limit": node.DiskFreeLimit,
@@ -313,12 +376,16 @@ func gatherNodes(r *RabbitMQ, acc telegraf.Accumulator) {
 			"run_queue":       node.RunQueue,
 			"sockets_total":   node.SocketsTotal,
 			"sockets_used":    node.SocketsUsed,
+			"running":         running,
 		}
 		acc.AddFields("rabbitmq_node", fields, tags, now)
 	}
 }
 
 func gatherQueues(r *RabbitMQ, acc telegraf.Accumulator) {
+	if r.excludeEveryQueue {
+		return
+	}
 	// Gather information about queues
 	queues := make([]Queue, 0)
 	err := r.requestJSON("/api/queues", &queues)
@@ -328,7 +395,7 @@ func gatherQueues(r *RabbitMQ, acc telegraf.Accumulator) {
 	}
 
 	for _, queue := range queues {
-		if !r.shouldGatherQueue(queue) {
+		if !r.queueFilter.Match(queue.Name) {
 			continue
 		}
 		tags := map[string]string{
@@ -373,6 +440,40 @@ func gatherQueues(r *RabbitMQ, acc telegraf.Accumulator) {
 	}
 }
 
+func gatherExchanges(r *RabbitMQ, acc telegraf.Accumulator) {
+	// Gather information about exchanges
+	exchanges := make([]Exchange, 0)
+	err := r.requestJSON("/api/exchanges", &exchanges)
+	if err != nil {
+		acc.AddError(err)
+		return
+	}
+
+	for _, exchange := range exchanges {
+		if !r.shouldGatherExchange(exchange) {
+			continue
+		}
+		tags := map[string]string{
+			"url":         r.URL,
+			"exchange":    exchange.Name,
+			"type":        exchange.Type,
+			"vhost":       exchange.Vhost,
+			"internal":    strconv.FormatBool(exchange.Internal),
+			"durable":     strconv.FormatBool(exchange.Durable),
+			"auto_delete": strconv.FormatBool(exchange.AutoDelete),
+		}
+
+		acc.AddFields(
+			"rabbitmq_exchange",
+			map[string]interface{}{
+				"messages_publish_in":  exchange.MessageStats.PublishIn,
+				"messages_publish_out": exchange.MessageStats.PublishOut,
+			},
+			tags,
+		)
+	}
+}
+
 func (r *RabbitMQ) shouldGatherNode(node Node) bool {
 	if len(r.Nodes) == 0 {
 		return true
@@ -387,13 +488,34 @@ func (r *RabbitMQ) shouldGatherNode(node Node) bool {
 	return false
 }
 
-func (r *RabbitMQ) shouldGatherQueue(queue Queue) bool {
-	if len(r.Queues) == 0 {
+func (r *RabbitMQ) createQueueFilter() error {
+	// Backwards compatibility for deprecated `queues` parameter.
+	if len(r.Queues) > 0 {
+		r.QueueInclude = append(r.QueueInclude, r.Queues...)
+	}
+
+	filter, err := filter.NewIncludeExcludeFilter(r.QueueInclude, r.QueueExclude)
+	if err != nil {
+		return err
+	}
+	r.queueFilter = filter
+
+	for _, q := range r.QueueExclude {
+		if q == "*" {
+			r.excludeEveryQueue = true
+		}
+	}
+
+	return nil
+}
+
+func (r *RabbitMQ) shouldGatherExchange(exchange Exchange) bool {
+	if len(r.Exchanges) == 0 {
 		return true
 	}
 
-	for _, name := range r.Queues {
-		if name == queue.Name {
+	for _, name := range r.Exchanges {
+		if name == exchange.Name {
 			return true
 		}
 	}

plugins/inputs/rabbitmq/rabbitmq_test.go

@@ -51,7 +51,25 @@ const sampleOverviewResponse = `
         "messages_unacknowledged_details": {
             "rate": 0.0
         }
-    }
+    },
+    "listeners": [
+        {
+            "name": "rabbit@node-a",
+            "protocol": "amqp"
+        },
+        {
+            "name": "rabbit@node-b",
+            "protocol": "amqp"
+        },
+        {
+            "name": "rabbit@node-a",
+            "protocol": "clustering"
+        },
+        {
+            "name": "rabbit@node-b",
+            "protocol": "clustering"
+        }
+    ]
 }
 `
 
@@ -374,6 +392,102 @@ const sampleQueuesResponse = `
 ]
 `
 
+const sampleExchangesResponse = `
+[
+  {
+    "arguments": { },
+    "internal": false,
+    "auto_delete": false,
+    "durable": true,
+    "type": "direct",
+    "vhost": "\/",
+    "name": ""
+  },
+  {
+    "message_stats": {
+      "publish_in_details": {
+        "rate": 0
+      },
+      "publish_in": 2,
+      "publish_out_details": {
+        "rate": 0
+      },
+      "publish_out": 1
+    },
+    "arguments": { },
+    "internal": false,
+    "auto_delete": false,
+    "durable": true,
+    "type": "fanout",
+    "vhost": "\/",
+    "name": "telegraf"
+  },
+  {
+    "arguments": { },
+    "internal": false,
+    "auto_delete": false,
+    "durable": true,
+    "type": "direct",
+    "vhost": "\/",
+    "name": "amq.direct"
+  },
+  {
+    "arguments": { },
+    "internal": false,
+    "auto_delete": false,
+    "durable": true,
+    "type": "fanout",
+    "vhost": "\/",
+    "name": "amq.fanout"
+  },
+  {
+    "arguments": { },
+    "internal": false,
+    "auto_delete": false,
+    "durable": true,
+    "type": "headers",
+    "vhost": "\/",
+    "name": "amq.headers"
+  },
+  {
+    "arguments": { },
+    "internal": false,
+    "auto_delete": false,
+    "durable": true,
+    "type": "headers",
+    "vhost": "\/",
+    "name": "amq.match"
+  },
+  {
+    "arguments": { },
+    "internal": true,
+    "auto_delete": false,
+    "durable": true,
+    "type": "topic",
+    "vhost": "\/",
+    "name": "amq.rabbitmq.log"
+  },
+  {
+    "arguments": { },
+    "internal": true,
+    "auto_delete": false,
+    "durable": true,
+    "type": "topic",
+    "vhost": "\/",
+    "name": "amq.rabbitmq.trace"
+  },
+  {
+    "arguments": { },
+    "internal": false,
+    "auto_delete": false,
+    "durable": true,
+    "type": "topic",
+    "vhost": "\/",
+    "name": "amq.topic"
+  }
+]
+`
+
 func TestRabbitMQGeneratesMetrics(t *testing.T) {
 	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		var rsp string
@@ -385,6 +499,8 @@ func TestRabbitMQGeneratesMetrics(t *testing.T) {
 			rsp = sampleNodesResponse
 		case "/api/queues":
 			rsp = sampleQueuesResponse
+		case "/api/exchanges":
+			rsp = sampleExchangesResponse
 		default:
 			panic("Cannot handle request")
 		}
@@ -416,6 +532,8 @@ func TestRabbitMQGeneratesMetrics(t *testing.T) {
 		"consumers",
 		"exchanges",
 		"queues",
+		"clustering_listeners",
+		"amqp_listeners",
 	}
 
 	for _, metric := range intMetrics {
@@ -434,6 +552,7 @@ func TestRabbitMQGeneratesMetrics(t *testing.T) {
 		"run_queue",
 		"sockets_total",
 		"sockets_used",
+		"running",
 	}
 
 	for _, metric := range nodeIntMetrics {
@@ -441,4 +560,13 @@ func TestRabbitMQGeneratesMetrics(t *testing.T) {
 	}
 
 	assert.True(t, acc.HasMeasurement("rabbitmq_queue"))
+
+	exchangeIntMetrics := []string{
+		"messages_publish_in",
+		"messages_publish_out",
+	}
+
+	for _, metric := range exchangeIntMetrics {
+		assert.True(t, acc.HasInt64Field("rabbitmq_exchange", metric))
+	}
 }

plugins/inputs/redis/README.md

@@ -87,7 +87,10 @@ Additionally the plugin also calculates the hit/miss ratio (keyspace\_hitrate) a
 
     **Replication**
     - connected_slaves(int, number)
+    - master_link_down_since_seconds(int, number)
+    - master_link_status(string)
     - master_repl_offset(int, number)
+    - second_repl_offset(int, number)
     - repl_backlog_active(int, number)
     - repl_backlog_size(int, bytes)
     - repl_backlog_first_byte_offset(int, number)

plugins/inputs/redis/redis.go

@@ -2,21 +2,47 @@ package redis
 
 import (
 	"bufio"
-	"errors"
 	"fmt"
-	"net"
+	"io"
+	"log"
 	"net/url"
 	"strconv"
 	"strings"
 	"sync"
 	"time"
 
+	"github.com/go-redis/redis"
 	"github.com/influxdata/telegraf"
 	"github.com/influxdata/telegraf/plugins/inputs"
 )
 
 type Redis struct {
 	Servers []string
+
+	clients     []Client
+	initialized bool
+}
+
+type Client interface {
+	Info() *redis.StringCmd
+	BaseTags() map[string]string
+}
+
+type RedisClient struct {
+	client *redis.Client
+	tags   map[string]string
+}
+
+func (r *RedisClient) Info() *redis.StringCmd {
+	return r.client.Info()
+}
+
+func (r *RedisClient) BaseTags() map[string]string {
+	tags := make(map[string]string)
+	for k, v := range r.tags {
+		tags[k] = v
+	}
+	return tags
 }
 
 var sampleConfig = `
@@ -32,8 +58,6 @@ var sampleConfig = `
   servers = ["tcp://localhost:6379"]
 `
 
-var defaultTimeout = 5 * time.Second
-
 func (r *Redis) SampleConfig() string {
 	return sampleConfig
 }
@@ -48,111 +72,107 @@ var Tracking = map[string]string{
 	"role":              "replication_role",
 }
 
-var ErrProtocolError = errors.New("redis protocol error")
-
-const defaultPort = "6379"
-
-// Reads stats from all configured servers accumulates stats.
-// Returns one of the errors encountered while gather stats (if any).
-func (r *Redis) Gather(acc telegraf.Accumulator) error {
-	if len(r.Servers) == 0 {
-		url := &url.URL{
-			Scheme: "tcp",
-			Host:   ":6379",
-		}
-		r.gatherServer(url, acc)
+func (r *Redis) init(acc telegraf.Accumulator) error {
+	if r.initialized {
 		return nil
 	}
 
-	var wg sync.WaitGroup
-	for _, serv := range r.Servers {
+	if len(r.Servers) == 0 {
+		r.Servers = []string{"tcp://localhost:6379"}
+	}
+
+	r.clients = make([]Client, len(r.Servers))
+
+	for i, serv := range r.Servers {
 		if !strings.HasPrefix(serv, "tcp://") && !strings.HasPrefix(serv, "unix://") {
+			log.Printf("W! [inputs.redis]: server URL found without scheme; please update your configuration file")
 			serv = "tcp://" + serv
 		}
 
 		u, err := url.Parse(serv)
 		if err != nil {
-			acc.AddError(fmt.Errorf("Unable to parse to address '%s': %s", serv, err))
-			continue
-		} else if u.Scheme == "" {
-			// fallback to simple string based address (i.e. "10.0.0.1:10000")
-			u.Scheme = "tcp"
-			u.Host = serv
-			u.Path = ""
+			return fmt.Errorf("Unable to parse to address %q: %v", serv, err)
 		}
-		if u.Scheme == "tcp" {
-			_, _, err := net.SplitHostPort(u.Host)
-			if err != nil {
-				u.Host = u.Host + ":" + defaultPort
+
+		password := ""
+		if u.User != nil {
+			pw, ok := u.User.Password()
+			if ok {
+				password = pw
 			}
 		}
 
+		var address string
+		if u.Scheme == "unix" {
+			address = u.Path
+		} else {
+			address = u.Host
+		}
+
+		client := redis.NewClient(
+			&redis.Options{
+				Addr:     address,
+				Password: password,
+				Network:  u.Scheme,
+				PoolSize: 1,
+			},
+		)
+
+		tags := map[string]string{}
+		if u.Scheme == "unix" {
+			tags["socket"] = u.Path
+		} else {
+			tags["server"] = u.Hostname()
+			tags["port"] = u.Port()
+		}
+
+		r.clients[i] = &RedisClient{
+			client: client,
+			tags:   tags,
+		}
+	}
+
+	r.initialized = true
+	return nil
+}
+
+// Reads stats from all configured servers accumulates stats.
+// Returns one of the errors encountered while gather stats (if any).
+func (r *Redis) Gather(acc telegraf.Accumulator) error {
+	if !r.initialized {
+		err := r.init(acc)
+		if err != nil {
+			return err
+		}
+	}
+
+	var wg sync.WaitGroup
+
+	for _, client := range r.clients {
 		wg.Add(1)
-		go func(serv string) {
+		go func(client Client) {
 			defer wg.Done()
-			acc.AddError(r.gatherServer(u, acc))
-		}(serv)
+			acc.AddError(r.gatherServer(client, acc))
+		}(client)
 	}
 
 	wg.Wait()
 	return nil
 }
 
-func (r *Redis) gatherServer(addr *url.URL, acc telegraf.Accumulator) error {
-	var address string
-
-	if addr.Scheme == "unix" {
-		address = addr.Path
-	} else {
-		address = addr.Host
-	}
-	c, err := net.DialTimeout(addr.Scheme, address, defaultTimeout)
+func (r *Redis) gatherServer(client Client, acc telegraf.Accumulator) error {
+	info, err := client.Info().Result()
 	if err != nil {
-		return fmt.Errorf("Unable to connect to redis server '%s': %s", address, err)
-	}
-	defer c.Close()
-
-	// Extend connection
-	c.SetDeadline(time.Now().Add(defaultTimeout))
-
-	if addr.User != nil {
-		pwd, set := addr.User.Password()
-		if set && pwd != "" {
-			c.Write([]byte(fmt.Sprintf("AUTH %s\r\n", pwd)))
-
-			rdr := bufio.NewReader(c)
-
-			line, err := rdr.ReadString('\n')
-			if err != nil {
-				return err
-			}
-			if line[0] != '+' {
-				return fmt.Errorf("%s", strings.TrimSpace(line)[1:])
-			}
-		}
+		return err
 	}
 
-	c.Write([]byte("INFO\r\n"))
-	c.Write([]byte("EOF\r\n"))
-	rdr := bufio.NewReader(c)
-
-	var tags map[string]string
-
-	if addr.Scheme == "unix" {
-		tags = map[string]string{"socket": addr.Path}
-	} else {
-		// Setup tags for all redis metrics
-		host, port := "unknown", "unknown"
-		// If there's an error, ignore and use 'unknown' tags
-		host, port, _ = net.SplitHostPort(addr.Host)
-		tags = map[string]string{"server": host, "port": port}
-	}
-	return gatherInfoOutput(rdr, acc, tags)
+	rdr := strings.NewReader(info)
+	return gatherInfoOutput(rdr, acc, client.BaseTags())
 }
 
 // gatherInfoOutput gathers
 func gatherInfoOutput(
-	rdr *bufio.Reader,
+	rdr io.Reader,
 	acc telegraf.Accumulator,
 	tags map[string]string,
 ) error {
@@ -163,13 +183,11 @@ func gatherInfoOutput(
 	fields := make(map[string]interface{})
 	for scanner.Scan() {
 		line := scanner.Text()
-		if strings.Contains(line, "ERR") {
-			break
-		}
 
 		if len(line) == 0 {
 			continue
 		}
+
 		if line[0] == '#' {
 			if len(line) > 2 {
 				section = line[2:]
@@ -189,6 +207,10 @@ func gatherInfoOutput(
 			}
 		}
 
+		if strings.HasPrefix(name, "master_replid") {
+			continue
+		}
+
 		if name == "mem_allocator" {
 			continue
 		}

plugins/inputs/redis/redis_test.go

@@ -86,6 +86,7 @@ func TestRedis_ParseMetrics(t *testing.T) {
 		"repl_backlog_size":              int64(1048576),
 		"repl_backlog_first_byte_offset": int64(0),
 		"repl_backlog_histlen":           int64(0),
+		"second_repl_offset":             int64(-1),
 		"used_cpu_sys":                   float64(0.14),
 		"used_cpu_user":                  float64(0.05),
 		"used_cpu_sys_children":          float64(0.00),
@@ -189,7 +190,10 @@ latest_fork_usec:0
 # Replication
 role:master
 connected_slaves:0
+master_replid:8c4d7b768b26826825ceb20ff4a2c7c54616350b
+master_replid2:0000000000000000000000000000000000000000
 master_repl_offset:0
+second_repl_offset:-1
 repl_backlog_active:0
 repl_backlog_size:1048576
 repl_backlog_first_byte_offset:0

plugins/inputs/smart/README.md

@@ -1,4 +1,4 @@
-# Telegraf S.M.A.R.T. plugin
+# S.M.A.R.T. Input Plugin
 
 Get metrics using the command line utility `smartctl` for S.M.A.R.T. (Self-Monitoring, Analysis and Reporting Technology) storage devices. SMART is a monitoring system included in computer hard disk drives (HDDs) and solid-state drives (SSDs)[1] that detects and reports on various indicators of drive reliability, with the intent of enabling the anticipation of hardware failures.
 See smartmontools (https://www.smartmontools.org/).
@@ -24,68 +24,7 @@ To enable SMART on a storage device run:
 smartctl -s on <device>
 ```
 
-## Measurements
-
-- smart_device:
-
-    * Tags:
-      - `capacity`
-      - `device`
-      - `device_model`
-      - `enabled`
-      - `health`
-      - `serial_no`
-      - `wwn`
-    * Fields:
-      - `exit_status`
-      - `health_ok`
-      - `read_error_rate`
-      - `seek_error`
-      - `temp_c`
-      - `udma_crc_errors`
-
-- smart_attribute:
-
-    * Tags:
-      - `device`
-      - `fail`
-      - `flags`
-      - `id`
-      - `name`
-      - `serial_no`
-      - `wwn`
-    * Fields:
-      - `exit_status`
-      - `raw_value`
-      - `threshold`
-      - `value`
-      - `worst`
-
-### Flags
-
-The interpretation of the tag `flags` is:
- - *K* auto-keep
- - *C* event count
- - *R* error rate
- - *S* speed/performance
- - *O* updated online
- - *P* prefailure warning
-
-### Exit Status
-
-The `exit_status` field captures the exit status of the smartctl command which
-is defined by a bitmask. For the interpretation of the bitmask see the man page for
-smartctl.
-
-### Device Names
-
-Device names, e.g., `/dev/sda`, are *not persistent*, and may be
-subject to change across reboots or system changes. Instead, you can the
-*World Wide Name* (WWN) or serial number to identify devices. On Linux block
-devices can be referenced by the WWN in the following location:
-`/dev/disk/by-id/`.
-
-## Configuration
+### Configuration:
 
 ```toml
 # Read metrics from storage devices supporting S.M.A.R.T.
@@ -122,14 +61,72 @@ devices can be referenced by the WWN in the following location:
   # devices = [ "/dev/ada0 -d atacam" ]
 ```
 
+### Metrics:
+
+- smart_device:
+  - tags:
+    - capacity
+    - device
+    - device_model
+    - enabled
+    - health
+    - serial_no
+    - wwn
+  - fields:
+    - exit_status
+    - health_ok
+    - read_error_rate
+    - seek_error
+    - temp_c
+    - udma_crc_errors
+
+- smart_attribute:
+  - tags:
+    - device
+    - fail
+    - flags
+    - id
+    - name
+    - serial_no
+    - wwn
+  - fields:
+    - exit_status
+    - raw_value
+    - threshold
+    - value
+    - worst
+
+#### Flags
+
+The interpretation of the tag `flags` is:
+ - `K` auto-keep
+ - `C` event count
+ - `R` error rate
+ - `S` speed/performance
+ - `O` updated online
+ - `P` prefailure warning
+
+#### Exit Status
+
+The `exit_status` field captures the exit status of the smartctl command which
+is defined by a bitmask. For the interpretation of the bitmask see the man page for
+smartctl.
+
+#### Device Names
+
+Device names, e.g., `/dev/sda`, are *not persistent*, and may be
+subject to change across reboots or system changes. Instead, you can the
+*World Wide Name* (WWN) or serial number to identify devices. On Linux block
+devices can be referenced by the WWN in the following location:
+`/dev/disk/by-id/`.
+
 To run `smartctl` with `sudo` create a wrapper script and use `path` in
 the configuration to execute that.
 
-## Output
+### Output
 
-Example output from an _Apple SSD_:
 ```
-> smart_attribute,serial_no=S1K5NYCD964433,wwn=5002538655584d30,id=199,name=UDMA_CRC_Error_Count,flags=-O-RC-,fail=-,host=mbpro.local,device=/dev/rdisk0 threshold=0i,raw_value=0i,exit_status=0i,value=200i,worst=200i 1502536854000000000
-> smart_attribute,device=/dev/rdisk0,serial_no=S1K5NYCD964433,wwn=5002538655584d30,id=240,name=Unknown_SSD_Attribute,flags=-O---K,fail=-,host=mbpro.local exit_status=0i,value=100i,worst=100i,threshold=0i,raw_value=0i 1502536854000000000
-> smart_device,enabled=Enabled,host=mbpro.local,device=/dev/rdisk0,model=APPLE\ SSD\ SM0512F,serial_no=S1K5NYCD964433,wwn=5002538655584d30,capacity=500277790720 udma_crc_errors=0i,exit_status=0i,health_ok=true,read_error_rate=0i,temp_c=40i 1502536854000000000
+smart_device,enabled=Enabled,host=mbpro.local,device=rdisk0,model=APPLE\ SSD\ SM0512F,serial_no=S1K5NYCD964433,wwn=5002538655584d30,capacity=500277790720 udma_crc_errors=0i,exit_status=0i,health_ok=true,read_error_rate=0i,temp_c=40i 1502536854000000000
+smart_attribute,serial_no=S1K5NYCD964433,wwn=5002538655584d30,id=199,name=UDMA_CRC_Error_Count,flags=-O-RC-,fail=-,host=mbpro.local,device=rdisk0 threshold=0i,raw_value=0i,exit_status=0i,value=200i,worst=200i 1502536854000000000
+smart_attribute,device=rdisk0,serial_no=S1K5NYCD964433,wwn=5002538655584d30,id=240,name=Unknown_SSD_Attribute,flags=-O---K,fail=-,host=mbpro.local exit_status=0i,value=100i,worst=100i,threshold=0i,raw_value=0i 1502536854000000000
 ```

plugins/inputs/smart/smart.go

@@ -3,6 +3,7 @@ package smart
 import (
 	"fmt"
 	"os/exec"
+	"path"
 	"regexp"
 	"strconv"
 	"strings"
@@ -134,7 +135,7 @@ func (m *Smart) scan() ([]string, error) {
 
 	devices := []string{}
 	for _, line := range strings.Split(string(out), "\n") {
-		dev := strings.Split(line, "#")
+		dev := strings.Split(line, " ")
 		if len(dev) > 1 && !excludedDev(m.Excludes, strings.TrimSpace(dev[0])) {
 			devices = append(devices, strings.TrimSpace(dev[0]))
 		}
@@ -178,13 +179,13 @@ func exitStatus(err error) (int, error) {
 	return 0, err
 }
 
-func gatherDisk(acc telegraf.Accumulator, usesudo, attributes bool, path, nockeck, device string, wg *sync.WaitGroup) {
+func gatherDisk(acc telegraf.Accumulator, usesudo, attributes bool, smartctl, nockeck, device string, wg *sync.WaitGroup) {
 
 	defer wg.Done()
 	// smartctl 5.41 & 5.42 have are broken regarding handling of --nocheck/-n
 	args := []string{"--info", "--health", "--attributes", "--tolerance=verypermissive", "-n", nockeck, "--format=brief"}
 	args = append(args, strings.Split(device, " ")...)
-	cmd := sudo(usesudo, path, args...)
+	cmd := sudo(usesudo, smartctl, args...)
 	out, e := internal.CombinedOutputTimeout(cmd, time.Second*5)
 	outStr := string(out)
 
@@ -196,7 +197,8 @@ func gatherDisk(acc telegraf.Accumulator, usesudo, attributes bool, path, nockec
 	}
 
 	device_tags := map[string]string{}
-	device_tags["device"] = strings.Split(device, " ")[0]
+	device_node := strings.Split(device, " ")[0]
+	device_tags["device"] = path.Base(device_node)
 	device_fields := make(map[string]interface{})
 	device_fields["exit_status"] = exitStatus
 
@@ -240,7 +242,8 @@ func gatherDisk(acc telegraf.Accumulator, usesudo, attributes bool, path, nockec
 				tags := map[string]string{}
 				fields := make(map[string]interface{})
 
-				tags["device"] = strings.Split(device, " ")[0]
+				device_node := strings.Split(device, " ")[0]
+				tags["device"] = path.Base(device_node)
 
 				if serial, ok := device_tags["serial_no"]; ok {
 					tags["serial_no"] = serial

plugins/inputs/smart/smart_test.go

@@ -89,7 +89,7 @@ func TestGatherAttributes(t *testing.T) {
 				"exit_status": int(0),
 			},
 			map[string]string{
-				"device":    "/dev/ada0",
+				"device":    "ada0",
 				"serial_no": "S0X5NZBC422720",
 				"wwn":       "5002538043584d30",
 				"id":        "1",
@@ -107,7 +107,7 @@ func TestGatherAttributes(t *testing.T) {
 				"exit_status": int(0),
 			},
 			map[string]string{
-				"device":    "/dev/ada0",
+				"device":    "ada0",
 				"serial_no": "S0X5NZBC422720",
 				"wwn":       "5002538043584d30",
 				"id":        "5",
@@ -125,7 +125,7 @@ func TestGatherAttributes(t *testing.T) {
 				"exit_status": int(0),
 			},
 			map[string]string{
-				"device":    "/dev/ada0",
+				"device":    "ada0",
 				"serial_no": "S0X5NZBC422720",
 				"wwn":       "5002538043584d30",
 				"id":        "9",
@@ -143,7 +143,7 @@ func TestGatherAttributes(t *testing.T) {
 				"exit_status": int(0),
 			},
 			map[string]string{
-				"device":    "/dev/ada0",
+				"device":    "ada0",
 				"serial_no": "S0X5NZBC422720",
 				"wwn":       "5002538043584d30",
 				"id":        "12",
@@ -161,7 +161,7 @@ func TestGatherAttributes(t *testing.T) {
 				"exit_status": int(0),
 			},
 			map[string]string{
-				"device":    "/dev/ada0",
+				"device":    "ada0",
 				"serial_no": "S0X5NZBC422720",
 				"wwn":       "5002538043584d30",
 				"id":        "169",
@@ -179,7 +179,7 @@ func TestGatherAttributes(t *testing.T) {
 				"exit_status": int(0),
 			},
 			map[string]string{
-				"device":    "/dev/ada0",
+				"device":    "ada0",
 				"serial_no": "S0X5NZBC422720",
 				"wwn":       "5002538043584d30",
 				"id":        "173",
@@ -197,7 +197,7 @@ func TestGatherAttributes(t *testing.T) {
 				"exit_status": int(0),
 			},
 			map[string]string{
-				"device":    "/dev/ada0",
+				"device":    "ada0",
 				"serial_no": "S0X5NZBC422720",
 				"wwn":       "5002538043584d30",
 				"id":        "190",
@@ -215,7 +215,7 @@ func TestGatherAttributes(t *testing.T) {
 				"exit_status": int(0),
 			},
 			map[string]string{
-				"device":    "/dev/ada0",
+				"device":    "ada0",
 				"serial_no": "S0X5NZBC422720",
 				"wwn":       "5002538043584d30",
 				"id":        "192",
@@ -233,7 +233,7 @@ func TestGatherAttributes(t *testing.T) {
 				"exit_status": int(0),
 			},
 			map[string]string{
-				"device":    "/dev/ada0",
+				"device":    "ada0",
 				"serial_no": "S0X5NZBC422720",
 				"wwn":       "5002538043584d30",
 				"id":        "194",
@@ -251,7 +251,7 @@ func TestGatherAttributes(t *testing.T) {
 				"exit_status": int(0),
 			},
 			map[string]string{
-				"device":    "/dev/ada0",
+				"device":    "ada0",
 				"serial_no": "S0X5NZBC422720",
 				"wwn":       "5002538043584d30",
 				"id":        "197",
@@ -269,7 +269,7 @@ func TestGatherAttributes(t *testing.T) {
 				"exit_status": int(0),
 			},
 			map[string]string{
-				"device":    "/dev/ada0",
+				"device":    "ada0",
 				"serial_no": "S0X5NZBC422720",
 				"wwn":       "5002538043584d30",
 				"id":        "199",
@@ -287,7 +287,7 @@ func TestGatherAttributes(t *testing.T) {
 				"exit_status": int(0),
 			},
 			map[string]string{
-				"device":    "/dev/ada0",
+				"device":    "ada0",
 				"serial_no": "S0X5NZBC422720",
 				"wwn":       "5002538043584d30",
 				"id":        "240",
@@ -317,7 +317,7 @@ func TestGatherAttributes(t *testing.T) {
 				"udma_crc_errors": int64(0),
 			},
 			map[string]string{
-				"device":    "/dev/ada0",
+				"device":    "ada0",
 				"model":     "APPLE SSD SM256E",
 				"serial_no": "S0X5NZBC422720",
 				"wwn":       "5002538043584d30",
@@ -363,7 +363,7 @@ func TestGatherNoAttributes(t *testing.T) {
 				"udma_crc_errors": int64(0),
 			},
 			map[string]string{
-				"device":    "/dev/ada0",
+				"device":    "ada0",
 				"model":     "APPLE SSD SM256E",
 				"serial_no": "S0X5NZBC422720",
 				"wwn":       "5002538043584d30",

plugins/inputs/snmp/snmp_test.go

@@ -333,6 +333,9 @@ func TestGetSNMPConnection_caching(t *testing.T) {
 }
 
 func TestGosnmpWrapper_walk_retry(t *testing.T) {
+	if testing.Short() {
+		t.Skip("Skipping test due to random failures.")
+	}
 	srvr, err := net.ListenUDP("udp4", &net.UDPAddr{})
 	defer srvr.Close()
 	require.NoError(t, err)
@@ -379,6 +382,8 @@ func TestGosnmpWrapper_walk_retry(t *testing.T) {
 }
 
 func TestGosnmpWrapper_get_retry(t *testing.T) {
+	// TODO: Fix this test
+	t.Skip("Test failing too often, skip for now and revisit later.")
 	srvr, err := net.ListenUDP("udp4", &net.UDPAddr{})
 	defer srvr.Close()
 	require.NoError(t, err)

plugins/inputs/solr/solr.go

@@ -246,6 +246,9 @@ func addAdminCoresStatusToAcc(acc telegraf.Accumulator, adminCoreStatus *AdminCo
 // Add core metrics section to accumulator
 func addCoreMetricsToAcc(acc telegraf.Accumulator, core string, mBeansData *MBeansData, time time.Time) error {
 	var coreMetrics map[string]Core
+	if len(mBeansData.SolrMbeans) < 2 {
+		return fmt.Errorf("no core metric data to unmarshall")
+	}
 	if err := json.Unmarshal(mBeansData.SolrMbeans[1], &coreMetrics); err != nil {
 		return err
 	}
@@ -274,9 +277,14 @@ func addCoreMetricsToAcc(acc telegraf.Accumulator, core string, mBeansData *MBea
 func addQueryHandlerMetricsToAcc(acc telegraf.Accumulator, core string, mBeansData *MBeansData, time time.Time) error {
 	var queryMetrics map[string]QueryHandler
 
+	if len(mBeansData.SolrMbeans) < 4 {
+		return fmt.Errorf("no query handler metric data to unmarshall")
+	}
+
 	if err := json.Unmarshal(mBeansData.SolrMbeans[3], &queryMetrics); err != nil {
 		return err
 	}
+
 	for name, metrics := range queryMetrics {
 		coreFields := map[string]interface{}{
 			"15min_rate_reqs_per_second": metrics.Stats.One5minRateReqsPerSecond,
@@ -310,6 +318,9 @@ func addQueryHandlerMetricsToAcc(acc telegraf.Accumulator, core string, mBeansDa
 func addUpdateHandlerMetricsToAcc(acc telegraf.Accumulator, core string, mBeansData *MBeansData, time time.Time) error {
 	var updateMetrics map[string]UpdateHandler
 
+	if len(mBeansData.SolrMbeans) < 6 {
+		return fmt.Errorf("no update handler metric data to unmarshall")
+	}
 	if err := json.Unmarshal(mBeansData.SolrMbeans[5], &updateMetrics); err != nil {
 		return err
 	}
@@ -364,6 +375,9 @@ func getFloat(unk interface{}) float64 {
 
 // Add cache metrics section to accumulator
 func addCacheMetricsToAcc(acc telegraf.Accumulator, core string, mBeansData *MBeansData, time time.Time) error {
+	if len(mBeansData.SolrMbeans) < 8 {
+		return fmt.Errorf("no cache metric data to unmarshall")
+	}
 	var cacheMetrics map[string]Cache
 	if err := json.Unmarshal(mBeansData.SolrMbeans[7], &cacheMetrics); err != nil {
 		return err