From 2150510bd49ac73bbe971dff11792207f8bac0e0 Mon Sep 17 00:00:00 2001 From: Cameron Sparr Date: Wed, 26 Oct 2016 16:38:56 +0100 Subject: [PATCH 01/14] nats_consumer: buffer incoming messages fixes #1956 --- plugins/inputs/nats_consumer/nats_consumer.go | 52 ++++++++++++++----- .../nats_consumer/nats_consumer_test.go | 5 ++ 2 files changed, 43 insertions(+), 14 deletions(-) diff --git a/plugins/inputs/nats_consumer/nats_consumer.go b/plugins/inputs/nats_consumer/nats_consumer.go index 6d59cce28..cbb85e016 100644 --- a/plugins/inputs/nats_consumer/nats_consumer.go +++ b/plugins/inputs/nats_consumer/nats_consumer.go @@ -28,12 +28,17 @@ type natsConsumer struct { Servers []string Secure bool + // Client pending limits: + PendingMessageLimit int + PendingBytesLimit int + // Legacy metric buffer support MetricBuffer int parser parsers.Parser sync.Mutex + wg sync.WaitGroup Conn *nats.Conn Subs []*nats.Subscription @@ -47,13 +52,18 @@ type natsConsumer struct { var sampleConfig = ` ## urls of NATS servers - servers = ["nats://localhost:4222"] + # servers = ["nats://localhost:4222"] ## Use Transport Layer Security - secure = false + # secure = false ## subject(s) to consume - subjects = ["telegraf"] + # subjects = ["telegraf"] ## name a queue group - queue_group = "telegraf_consumers" + # queue_group = "telegraf_consumers" + + ## Sets the limits for pending msgs and bytes for each subscription + ## These shouldn't need to be adjusted except in very high throughput scenarios + # pending_message_limit = 65536 + # pending_bytes_limit = 67108864 ## Data format to consume. ## Each data format has it's own unique set of configuration options, read @@ -112,12 +122,22 @@ func (n *natsConsumer) Start(acc telegraf.Accumulator) error { n.errs = make(chan error) n.Conn.SetErrorHandler(n.natsErrHandler) - n.in = make(chan *nats.Msg) + n.in = make(chan *nats.Msg, 1000) for _, subj := range n.Subjects { - sub, err := n.Conn.ChanQueueSubscribe(subj, n.QueueGroup, n.in) + sub, err := n.Conn.QueueSubscribe(subj, n.QueueGroup, func(m *nats.Msg) { + n.in <- m + }) if err != nil { return err } + // ensure that the subscription has been processed by the server + if err = n.Conn.Flush(); err != nil { + return err + } + // set the subscription pending limits + if err = sub.SetPendingLimits(n.PendingMessageLimit, n.PendingBytesLimit); err != nil { + return err + } n.Subs = append(n.Subs, sub) } } @@ -125,6 +145,7 @@ func (n *natsConsumer) Start(acc telegraf.Accumulator) error { n.done = make(chan struct{}) // Start the message reader + n.wg.Add(1) go n.receiver() log.Printf("I! Started the NATS consumer service, nats: %v, subjects: %v, queue: %v\n", n.Conn.ConnectedUrl(), n.Subjects, n.QueueGroup) @@ -135,7 +156,7 @@ func (n *natsConsumer) Start(acc telegraf.Accumulator) error { // receiver() reads all incoming messages from NATS, and parses them into // telegraf metrics. func (n *natsConsumer) receiver() { - defer n.clean() + defer n.wg.Done() for { select { case <-n.done: @@ -151,17 +172,11 @@ func (n *natsConsumer) receiver() { for _, metric := range metrics { n.acc.AddFields(metric.Name(), metric.Fields(), metric.Tags(), metric.Time()) } - } } } func (n *natsConsumer) clean() { - n.Lock() - defer n.Unlock() - close(n.in) - close(n.errs) - for _, sub := range n.Subs { if err := sub.Unsubscribe(); err != nil { log.Printf("E! Error unsubscribing from subject %s in queue %s: %s\n", @@ -177,6 +192,8 @@ func (n *natsConsumer) clean() { func (n *natsConsumer) Stop() { n.Lock() close(n.done) + n.wg.Wait() + n.clean() n.Unlock() } @@ -186,6 +203,13 @@ func (n *natsConsumer) Gather(acc telegraf.Accumulator) error { func init() { inputs.Add("nats_consumer", func() telegraf.Input { - return &natsConsumer{} + return &natsConsumer{ + Servers: []string{"nats://localhost:4222"}, + Secure: false, + Subjects: []string{"telegraf"}, + QueueGroup: "telegraf_consumers", + PendingBytesLimit: nats.DefaultSubPendingBytesLimit, + PendingMessageLimit: nats.DefaultSubPendingMsgsLimit, + } }) } diff --git a/plugins/inputs/nats_consumer/nats_consumer_test.go b/plugins/inputs/nats_consumer/nats_consumer_test.go index 75fde66a6..206714b1a 100644 --- a/plugins/inputs/nats_consumer/nats_consumer_test.go +++ b/plugins/inputs/nats_consumer/nats_consumer_test.go @@ -39,6 +39,7 @@ func TestRunParser(t *testing.T) { defer close(n.done) n.parser, _ = parsers.NewInfluxParser() + n.wg.Add(1) go n.receiver() in <- natsMsg(testMsg) time.Sleep(time.Millisecond * 25) @@ -56,6 +57,7 @@ func TestRunParserInvalidMsg(t *testing.T) { defer close(n.done) n.parser, _ = parsers.NewInfluxParser() + n.wg.Add(1) go n.receiver() in <- natsMsg(invalidMsg) time.Sleep(time.Millisecond * 25) @@ -73,6 +75,7 @@ func TestRunParserAndGather(t *testing.T) { defer close(n.done) n.parser, _ = parsers.NewInfluxParser() + n.wg.Add(1) go n.receiver() in <- natsMsg(testMsg) time.Sleep(time.Millisecond * 25) @@ -91,6 +94,7 @@ func TestRunParserAndGatherGraphite(t *testing.T) { defer close(n.done) n.parser, _ = parsers.NewGraphiteParser("_", []string{}, nil) + n.wg.Add(1) go n.receiver() in <- natsMsg(testMsgGraphite) time.Sleep(time.Millisecond * 25) @@ -109,6 +113,7 @@ func TestRunParserAndGatherJSON(t *testing.T) { defer close(n.done) n.parser, _ = parsers.NewJSONParser("nats_json_test", []string{}, nil) + n.wg.Add(1) go n.receiver() in <- natsMsg(testMsgJSON) time.Sleep(time.Millisecond * 25) From 137272afea67f28a56f7c898bb43e2ca32386b4f Mon Sep 17 00:00:00 2001 From: Prunar Date: Wed, 2 Nov 2016 15:25:09 +0100 Subject: [PATCH 02/14] Update README.md (#1963) Typo --- plugins/inputs/win_perf_counters/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/inputs/win_perf_counters/README.md b/plugins/inputs/win_perf_counters/README.md index 967714b48..fe35d97f9 100644 --- a/plugins/inputs/win_perf_counters/README.md +++ b/plugins/inputs/win_perf_counters/README.md @@ -64,7 +64,7 @@ Instances (this is an array) is the instances of a counter you would like return it can be one or more values. Example, `Instances = ["C:","D:","E:"]` will return only for the instances -C:, D: and E: where relevant. To get all instnaces of a Counter, use ["*"] only. +C:, D: and E: where relevant. To get all instances of a Counter, use ["*"] only. By default any results containing _Total are stripped, unless this is specified as the wanted instance. Alternatively see the option IncludeTotal below. From e43cfc2fcee7fdf56ce23da3d05db1bd3c4ac34b Mon Sep 17 00:00:00 2001 From: Matteo Cerutti Date: Thu, 3 Nov 2016 10:46:54 +0000 Subject: [PATCH 03/14] fix leap_status value in chrony input plugin (#1983) --- plugins/inputs/chrony/chrony.go | 6 +++++- plugins/inputs/chrony/chrony_test.go | 4 ++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/plugins/inputs/chrony/chrony.go b/plugins/inputs/chrony/chrony.go index 51081b87b..f39108921 100644 --- a/plugins/inputs/chrony/chrony.go +++ b/plugins/inputs/chrony/chrony.go @@ -103,9 +103,13 @@ func processChronycOutput(out string) (map[string]interface{}, map[string]string tags["stratum"] = valueFields[0] continue } + if strings.Contains(strings.ToLower(name), "reference_id") { + tags["reference_id"] = valueFields[0] + continue + } value, err := strconv.ParseFloat(valueFields[0], 64) if err != nil { - tags[name] = strings.ToLower(valueFields[0]) + tags[name] = strings.ToLower(strings.Join(valueFields, " ")) continue } if strings.Contains(stats[1], "slow") { diff --git a/plugins/inputs/chrony/chrony_test.go b/plugins/inputs/chrony/chrony_test.go index 143d7e9d9..496986c40 100644 --- a/plugins/inputs/chrony/chrony_test.go +++ b/plugins/inputs/chrony/chrony_test.go @@ -27,7 +27,7 @@ func TestGather(t *testing.T) { tags := map[string]string{ "reference_id": "192.168.1.22", - "leap_status": "normal", + "leap_status": "not synchronized", "stratum": "3", } fields := map[string]interface{}{ @@ -85,7 +85,7 @@ Skew : 0.006 ppm Root delay : 0.001655 seconds Root dispersion : 0.003307 seconds Update interval : 507.2 seconds -Leap status : Normal +Leap status : Not synchronized ` args := os.Args From 16081b2d1af6f69093e6a86300d8aea0002629dd Mon Sep 17 00:00:00 2001 From: Cameron Sparr Date: Thu, 3 Nov 2016 14:31:55 +0000 Subject: [PATCH 04/14] Update etc/telegraf.conf --- etc/telegraf.conf | 118 ++++++++++++++++++++++++++++++++++------------ 1 file changed, 89 insertions(+), 29 deletions(-) diff --git a/etc/telegraf.conf b/etc/telegraf.conf index 2ad0bcbae..9f95a9ca5 100644 --- a/etc/telegraf.conf +++ b/etc/telegraf.conf @@ -66,7 +66,7 @@ debug = false ## Run telegraf in quiet mode (error log messages only). quiet = false - ## Specify the log file name. The empty string means to log to stdout. + ## Specify the log file name. The empty string means to log to stderr. logfile = "" ## Override default hostname, if empty use os.Hostname() @@ -441,6 +441,30 @@ +############################################################################### +# PROCESSOR PLUGINS # +############################################################################### + +# # Print all metrics that pass through this filter. +# [[processors.printer]] + + + +############################################################################### +# AGGREGATOR PLUGINS # +############################################################################### + +# # Keep the aggregate min/max of each metric passing through. +# [[aggregators.minmax]] +# ## General Aggregator Arguments: +# ## The period on which to flush & clear the aggregator. +# period = "30s" +# ## If true, the original metric will be dropped by the +# ## aggregator and will not get sent to the output plugins. +# drop_original = false + + + ############################################################################### # INPUT PLUGINS # ############################################################################### @@ -582,15 +606,18 @@ # # Read specific statistics per cgroup # [[inputs.cgroup]] -# ## Directories in which to look for files, globs are supported. -# # paths = [ -# # "/cgroup/memory", -# # "/cgroup/memory/child1", -# # "/cgroup/memory/child2/*", -# # ] -# ## cgroup stat fields, as file names, globs are supported. -# ## these file names are appended to each path from above. -# # files = ["memory.*usage*", "memory.limit_in_bytes"] +# ## Directories in which to look for files, globs are supported. +# ## Consider restricting paths to the set of cgroups you really +# ## want to monitor if you have a large number of cgroups, to avoid +# ## any cardinality issues. +# # paths = [ +# # "/cgroup/memory", +# # "/cgroup/memory/child1", +# # "/cgroup/memory/child2/*", +# # ] +# ## cgroup stat fields, as file names, globs are supported. +# ## these file names are appended to each path from above. +# # files = ["memory.*usage*", "memory.limit_in_bytes"] # # Pull Metric Statistics from Amazon CloudWatch @@ -850,12 +877,15 @@ # ## An array of address to gather stats about. Specify an ip on hostname # ## with optional port. ie localhost, 10.10.3.33:1936, etc. # ## Make sure you specify the complete path to the stats endpoint -# ## ie 10.10.3.33:1936/haproxy?stats +# ## including the protocol, ie http://10.10.3.33:1936/haproxy?stats # # # ## If no servers are specified, then default to 127.0.0.1:1936/haproxy?stats # servers = ["http://myhaproxy.com:1936/haproxy?stats"] -# ## Or you can also use local socket -# ## servers = ["socket:/run/haproxy/admin.sock"] +# ## +# ## You can also use local socket with standard wildcard globbing. +# ## Server address not starting with 'http' will be treated as a possible +# ## socket, so both examples below are valid. +# ## servers = ["socket:/run/haproxy/admin.sock", "/run/haproxy/*.sock"] # # HTTP/HTTPS request given an address a method and a timeout @@ -1000,6 +1030,22 @@ # attribute = "LoadedClassCount,UnloadedClassCount,TotalLoadedClassCount" +# # Read metrics from the kubernetes kubelet api +# [[inputs.kubernetes]] +# ## URL for the kubelet +# url = "http://1.1.1.1:10255" +# +# ## Use bearer token for authorization +# # bearer_token = /path/to/bearer/token +# +# ## Optional SSL Config +# # ssl_ca = /path/to/cafile +# # ssl_cert = /path/to/certfile +# # ssl_key = /path/to/keyfile +# ## Use SSL but skip chain & host verification +# # insecure_skip_verify = false + + # # Read metrics from a LeoFS Server via SNMP # [[inputs.leofs]] # ## An array of URI to gather stats about LeoFS. @@ -1119,13 +1165,13 @@ # ## gather metrics from SHOW BINARY LOGS command output # gather_binary_logs = false # # -# ## gather metrics from PERFORMANCE_SCHEMA.TABLE_IO_WAITS_SUMMART_BY_TABLE +# ## gather metrics from PERFORMANCE_SCHEMA.TABLE_IO_WAITS_SUMMARY_BY_TABLE # gather_table_io_waits = false # # # ## gather metrics from PERFORMANCE_SCHEMA.TABLE_LOCK_WAITS # gather_table_lock_waits = false # # -# ## gather metrics from PERFORMANCE_SCHEMA.TABLE_IO_WAITS_SUMMART_BY_INDEX_USAGE +# ## gather metrics from PERFORMANCE_SCHEMA.TABLE_IO_WAITS_SUMMARY_BY_INDEX_USAGE # gather_index_io_waits = false # # # ## gather metrics from PERFORMANCE_SCHEMA.EVENT_WAITS @@ -1247,13 +1293,13 @@ # ## urls to ping # urls = ["www.google.com"] # required # ## number of pings to send per collection (ping -c ) -# count = 1 # required +# # count = 1 # ## interval, in s, at which to ping. 0 == default (ping -i ) -# ping_interval = 0.0 +# # ping_interval = 1.0 # ## per-ping timeout, in s. 0 == no timeout (ping -W ) -# timeout = 1.0 +# # timeout = 1.0 # ## interface to send ping from (ping -I ) -# interface = "" +# # interface = "" # # Read metrics from one or many postgresql servers @@ -1681,9 +1727,18 @@ # ## Address and port to host HTTP listener on # service_address = ":8186" # -# ## timeouts +# ## maximum duration before timing out read of the request # read_timeout = "10s" +# ## maximum duration before timing out write of the response # write_timeout = "10s" +# +# ## Maximum allowed http request body size in bytes. +# ## 0 means to use the default of 536,870,912 bytes (500 mebibytes) +# max_body_size = 0 +# +# ## Maximum line size allowed to be sent in bytes. +# ## 0 means to use the default of 65536 bytes (64 kibibytes) +# max_line_size = 0 # # Read metrics from Kafka topic(s) @@ -1778,13 +1833,18 @@ # # Read metrics from NATS subject(s) # [[inputs.nats_consumer]] # ## urls of NATS servers -# servers = ["nats://localhost:4222"] +# # servers = ["nats://localhost:4222"] # ## Use Transport Layer Security -# secure = false +# # secure = false # ## subject(s) to consume -# subjects = ["telegraf"] +# # subjects = ["telegraf"] # ## name a queue group -# queue_group = "telegraf_consumers" +# # queue_group = "telegraf_consumers" +# +# ## Sets the limits for pending msgs and bytes for each subscription +# ## These shouldn't need to be adjusted except in very high throughput scenarios +# # pending_message_limit = 65536 +# # pending_bytes_limit = 67108864 # # ## Data format to consume. # ## Each data format has it's own unique set of configuration options, read @@ -1871,14 +1931,14 @@ # # Generic TCP listener # [[inputs.tcp_listener]] # ## Address and port to host TCP listener on -# service_address = ":8094" +# # service_address = ":8094" # # ## Number of TCP messages allowed to queue up. Once filled, the # ## TCP listener will start dropping packets. -# allowed_pending_messages = 10000 +# # allowed_pending_messages = 10000 # # ## Maximum number of concurrent TCP connections to allow -# max_tcp_connections = 250 +# # max_tcp_connections = 250 # # ## Data format to consume. # ## Each data format has it's own unique set of configuration options, read @@ -1890,11 +1950,11 @@ # # Generic UDP listener # [[inputs.udp_listener]] # ## Address and port to host UDP listener on -# service_address = ":8092" +# # service_address = ":8092" # # ## Number of UDP messages allowed to queue up. Once filled, the # ## UDP listener will start dropping packets. -# allowed_pending_messages = 10000 +# # allowed_pending_messages = 10000 # # ## Data format to consume. # ## Each data format has it's own unique set of configuration options, read From 34182d9c9f63070c52c8d0a76b54b2cebaae31bb Mon Sep 17 00:00:00 2001 From: Cameron Sparr Date: Thu, 3 Nov 2016 14:34:09 +0000 Subject: [PATCH 05/14] Add release 1.2 section to changelog --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 83aecc135..e2df116d1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,11 @@ +## v1.2 [unreleased] + +### Release Notes + +### Features + +### Bugfixes + ## v1.1 [unreleased] ### Release Notes From fe6162b2a1d176fc6ec54f888c21aa10667b72fe Mon Sep 17 00:00:00 2001 From: Cameron Sparr Date: Thu, 3 Nov 2016 16:37:52 +0000 Subject: [PATCH 06/14] Use short commit in Makefile build --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 3b4ecfa1d..6c75b9295 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ VERSION := $(shell sh -c 'git describe --always --tags') BRANCH := $(shell sh -c 'git rev-parse --abbrev-ref HEAD') -COMMIT := $(shell sh -c 'git rev-parse HEAD') +COMMIT := $(shell sh -c 'git rev-parse --short HEAD') ifdef GOBIN PATH := $(GOBIN):$(PATH) else From a6ed4d4c3a052071adccf225d327513f354c884d Mon Sep 17 00:00:00 2001 From: Cameron Sparr Date: Thu, 3 Nov 2016 17:07:25 +0000 Subject: [PATCH 07/14] CircleCI script, do not explicitly set version tag --- scripts/circle-test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/circle-test.sh b/scripts/circle-test.sh index 6e7a125c1..8f65a706b 100755 --- a/scripts/circle-test.sh +++ b/scripts/circle-test.sh @@ -82,6 +82,6 @@ if [ $? -eq 0 ]; then unset GOGC tag=$(git describe --exact-match HEAD) echo $tag - exit_if_fail ./scripts/build.py --release --package --version=$tag --platform=all --arch=all --upload --bucket=dl.influxdata.com/telegraf/releases + exit_if_fail ./scripts/build.py --release --package --platform=all --arch=all --upload --bucket=dl.influxdata.com/telegraf/releases mv build $CIRCLE_ARTIFACTS fi From dc28875437aa6cc8ad701c875d8294a76fd4859f Mon Sep 17 00:00:00 2001 From: Cameron Sparr Date: Wed, 26 Oct 2016 17:17:20 +0100 Subject: [PATCH 08/14] Update gopsutil dependency primarily for a fix in Windows network counter getting code closes #1949 --- CHANGELOG.md | 2 ++ Godeps | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e2df116d1..43dfe1deb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,8 @@ ### Bugfixes +- [#1949](https://github.com/influxdata/telegraf/issues/1949): Fix windows `net` plugin. + ## v1.1 [unreleased] ### Release Notes diff --git a/Godeps b/Godeps index 6dc0cec2d..6521c2955 100644 --- a/Godeps +++ b/Godeps @@ -47,7 +47,7 @@ github.com/prometheus/client_model fa8ad6fec33561be4280a8f0514318c79d7f6cb6 github.com/prometheus/common e8eabff8812b05acf522b45fdcd725a785188e37 github.com/prometheus/procfs 406e5b7bfd8201a36e2bb5f7bdae0b03380c2ce8 github.com/samuel/go-zookeeper 218e9c81c0dd8b3b18172b2bbfad92cc7d6db55f -github.com/shirou/gopsutil 4d0c402af66c78735c5ccf820dc2ca7de5e4ff08 +github.com/shirou/gopsutil 1516eb9ddc5e61ba58874047a98f8b44b5e585e8 github.com/soniah/gosnmp 3fe3beb30fa9700988893c56a63b1df8e1b68c26 github.com/streadway/amqp b4f3ceab0337f013208d31348b578d83c0064744 github.com/stretchr/testify 1f4a1643a57e798696635ea4c126e9127adb7d3c From 7a9fef80f5c75ce2ecb6a3b30ec34a7eb85f339a Mon Sep 17 00:00:00 2001 From: Johannes Rudolph Date: Thu, 3 Nov 2016 19:28:24 +0100 Subject: [PATCH 09/14] Update README.md (#1868) I think this is a copy paste bug? ;-) --- plugins/inputs/mysql/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/inputs/mysql/README.md b/plugins/inputs/mysql/README.md index 20dd73e02..3599b2fb8 100644 --- a/plugins/inputs/mysql/README.md +++ b/plugins/inputs/mysql/README.md @@ -133,7 +133,7 @@ The unit of fields varies by the tags. * file_events_total(float,number) * file_events_seconds_total(float, milliseconds) * file_events_bytes_total(float, bytes) -* Perf file events statements - gathers attributes of each event +* Perf events statements - gathers attributes of each event * events_statements_total(float, number) * events_statements_seconds_total(float, millieconds) * events_statements_errors_total(float, number) From 47de43abf3864b99465b5b033f420ce3d6e25e29 Mon Sep 17 00:00:00 2001 From: Cameron Sparr Date: Tue, 11 Oct 2016 13:06:17 +0100 Subject: [PATCH 10/14] Use rfc3339 timestamps in telegraf log output closes #1564 also add unit and benchmark tests --- CHANGELOG.md | 2 ++ logger/logger.go | 6 +++-- logger/logger_test.go | 62 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 68 insertions(+), 2 deletions(-) create mode 100644 logger/logger_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 43dfe1deb..280a51876 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,8 @@ ### Features +- [#1564](https://github.com/influxdata/telegraf/issues/1564): Use RFC3339 timestamps in log output. + ### Bugfixes - [#1949](https://github.com/influxdata/telegraf/issues/1949): Fix windows `net` plugin. diff --git a/logger/logger.go b/logger/logger.go index 21db2d411..49613c27d 100644 --- a/logger/logger.go +++ b/logger/logger.go @@ -4,6 +4,7 @@ import ( "io" "log" "os" + "time" "github.com/influxdata/wlog" ) @@ -19,8 +20,8 @@ type telegrafLog struct { writer io.Writer } -func (t *telegrafLog) Write(p []byte) (n int, err error) { - return t.writer.Write(p) +func (t *telegrafLog) Write(b []byte) (n int, err error) { + return t.writer.Write(append([]byte(time.Now().UTC().Format(time.RFC3339)+" "), b...)) } // SetupLogging configures the logging output. @@ -30,6 +31,7 @@ func (t *telegrafLog) Write(p []byte) (n int, err error) { // interpreted as stderr. If there is an error opening the file the // logger will fallback to stderr. func SetupLogging(debug, quiet bool, logfile string) { + log.SetFlags(0) if debug { wlog.SetLevel(wlog.DEBUG) } diff --git a/logger/logger_test.go b/logger/logger_test.go new file mode 100644 index 000000000..8c0826e65 --- /dev/null +++ b/logger/logger_test.go @@ -0,0 +1,62 @@ +package logger + +import ( + "bytes" + "io/ioutil" + "log" + "os" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestWriteLogToFile(t *testing.T) { + tmpfile, err := ioutil.TempFile("", "") + assert.NoError(t, err) + defer func() { os.Remove(tmpfile.Name()) }() + + SetupLogging(false, false, tmpfile.Name()) + log.Printf("I! TEST") + log.Printf("D! TEST") // <- should be ignored + + f, err := ioutil.ReadFile(tmpfile.Name()) + assert.NoError(t, err) + assert.Equal(t, f[19:], []byte("Z I! TEST\n")) +} + +func TestDebugWriteLogToFile(t *testing.T) { + tmpfile, err := ioutil.TempFile("", "") + assert.NoError(t, err) + defer func() { os.Remove(tmpfile.Name()) }() + + SetupLogging(true, false, tmpfile.Name()) + log.Printf("D! TEST") + + f, err := ioutil.ReadFile(tmpfile.Name()) + assert.NoError(t, err) + assert.Equal(t, f[19:], []byte("Z D! TEST\n")) +} + +func TestErrorWriteLogToFile(t *testing.T) { + tmpfile, err := ioutil.TempFile("", "") + assert.NoError(t, err) + defer func() { os.Remove(tmpfile.Name()) }() + + SetupLogging(false, true, tmpfile.Name()) + log.Printf("E! TEST") + log.Printf("I! TEST") // <- should be ignored + + f, err := ioutil.ReadFile(tmpfile.Name()) + assert.NoError(t, err) + assert.Equal(t, f[19:], []byte("Z E! TEST\n")) +} + +func BenchmarkTelegrafLogWrite(b *testing.B) { + var msg = []byte("test") + var buf bytes.Buffer + w := newTelegrafWriter(&buf) + for i := 0; i < b.N; i++ { + buf.Reset() + w.Write(msg) + } +} From b349800f7a049a87c2c044ecd8615275fe32d2f1 Mon Sep 17 00:00:00 2001 From: John Engelman Date: Fri, 4 Nov 2016 08:16:41 -0500 Subject: [PATCH 11/14] Fix up AWS plugin docs so they don't use single quotes. (#1991) Also don't use named returns in fetchNamespaceMetrics since it's non-standard for the rest of the codebase. --- plugins/inputs/cloudwatch/README.md | 20 ++++++++--------- plugins/inputs/cloudwatch/cloudwatch.go | 28 +++++++++++------------- plugins/outputs/cloudwatch/cloudwatch.go | 4 ++-- 3 files changed, 25 insertions(+), 27 deletions(-) diff --git a/plugins/inputs/cloudwatch/README.md b/plugins/inputs/cloudwatch/README.md index 4430e48fd..ca7145c30 100644 --- a/plugins/inputs/cloudwatch/README.md +++ b/plugins/inputs/cloudwatch/README.md @@ -18,21 +18,21 @@ API endpoint. In the following order the plugin will attempt to authenticate. ```toml [[inputs.cloudwatch]] ## Amazon Region (required) - region = 'us-east-1' + region = "us-east-1" ## Requested CloudWatch aggregation Period (required - must be a multiple of 60s) - period = '1m' + period = "1m" ## Collection Delay (required - must account for metrics availability via CloudWatch API) - delay = '1m' + delay = "1m" ## Override global run interval (optional - defaults to global interval) ## Recomended: use metric 'interval' that is a multiple of 'period' to avoid ## gaps or overlap in pulled data - interval = '1m' + interval = "1m" ## Metric Statistic Namespace (required) - namespace = 'AWS/ELB' + namespace = "AWS/ELB" ## Maximum requests per second. Note that the global default AWS rate limit is ## 10 reqs/sec, so if you define multiple namespaces, these should add up to a @@ -43,16 +43,16 @@ API endpoint. In the following order the plugin will attempt to authenticate. ## Defaults to all Metrics in Namespace if nothing is provided ## Refreshes Namespace available metrics every 1h [[inputs.cloudwatch.metrics]] - names = ['Latency', 'RequestCount'] + names = ["Latency", "RequestCount"] ## Dimension filters for Metric (optional) [[inputs.cloudwatch.metrics.dimensions]] - name = 'LoadBalancerName' - value = 'p-example' + name = "LoadBalancerName" + value = "p-example" [[inputs.cloudwatch.metrics.dimensions]] - name = 'AvailabilityZone' - value = '*' + name = "AvailabilityZone" + value = "*" ``` #### Requirements and Terminology diff --git a/plugins/inputs/cloudwatch/cloudwatch.go b/plugins/inputs/cloudwatch/cloudwatch.go index ebc4147d8..c3dbda05b 100644 --- a/plugins/inputs/cloudwatch/cloudwatch.go +++ b/plugins/inputs/cloudwatch/cloudwatch.go @@ -63,7 +63,7 @@ type ( func (c *CloudWatch) SampleConfig() string { return ` ## Amazon Region - region = 'us-east-1' + region = "us-east-1" ## Amazon Credentials ## Credentials are loaded in the following order @@ -81,21 +81,21 @@ func (c *CloudWatch) SampleConfig() string { #shared_credential_file = "" ## Requested CloudWatch aggregation Period (required - must be a multiple of 60s) - period = '1m' + period = "1m" ## Collection Delay (required - must account for metrics availability via CloudWatch API) - delay = '1m' + delay = "1m" ## Recomended: use metric 'interval' that is a multiple of 'period' to avoid ## gaps or overlap in pulled data - interval = '1m' + interval = "1m" ## Configure the TTL for the internal cache of metrics. ## Defaults to 1 hr if not specified - #cache_ttl = '10m' + #cache_ttl = "10m" ## Metric Statistic Namespace (required) - namespace = 'AWS/ELB' + namespace = "AWS/ELB" ## Maximum requests per second. Note that the global default AWS rate limit is ## 10 reqs/sec, so if you define multiple namespaces, these should add up to a @@ -106,12 +106,12 @@ func (c *CloudWatch) SampleConfig() string { ## Defaults to all Metrics in Namespace if nothing is provided ## Refreshes Namespace available metrics every 1h #[[inputs.cloudwatch.metrics]] - # names = ['Latency', 'RequestCount'] + # names = ["Latency", "RequestCount"] # # ## Dimension filters for Metric (optional) # [[inputs.cloudwatch.metrics.dimensions]] - # name = 'LoadBalancerName' - # value = 'p-example' + # name = "LoadBalancerName" + # value = "p-example" ` } @@ -133,7 +133,6 @@ func (c *CloudWatch) Gather(acc telegraf.Accumulator) error { if !hasWilcard(m.Dimensions) { dimensions := make([]*cloudwatch.Dimension, len(m.Dimensions)) for k, d := range m.Dimensions { - fmt.Printf("Dimension [%s]:[%s]\n", d.Name, d.Value) dimensions[k] = &cloudwatch.Dimension{ Name: aws.String(d.Name), Value: aws.String(d.Value), @@ -229,13 +228,12 @@ func (c *CloudWatch) initializeCloudWatch() error { /* * Fetch available metrics for given CloudWatch Namespace */ -func (c *CloudWatch) fetchNamespaceMetrics() (metrics []*cloudwatch.Metric, err error) { +func (c *CloudWatch) fetchNamespaceMetrics() ([]*cloudwatch.Metric, error) { if c.metricCache != nil && c.metricCache.IsValid() { - metrics = c.metricCache.Metrics - return + return c.metricCache.Metrics, nil } - metrics = []*cloudwatch.Metric{} + metrics := []*cloudwatch.Metric{} var token *string for more := true; more; { @@ -263,7 +261,7 @@ func (c *CloudWatch) fetchNamespaceMetrics() (metrics []*cloudwatch.Metric, err TTL: c.CacheTTL.Duration, } - return + return metrics, nil } /* diff --git a/plugins/outputs/cloudwatch/cloudwatch.go b/plugins/outputs/cloudwatch/cloudwatch.go index 045dae462..4eb2706a3 100644 --- a/plugins/outputs/cloudwatch/cloudwatch.go +++ b/plugins/outputs/cloudwatch/cloudwatch.go @@ -30,7 +30,7 @@ type CloudWatch struct { var sampleConfig = ` ## Amazon REGION - region = 'us-east-1' + region = "us-east-1" ## Amazon Credentials ## Credentials are loaded in the following order @@ -48,7 +48,7 @@ var sampleConfig = ` #shared_credential_file = "" ## Namespace for the CloudWatch MetricDatums - namespace = 'InfluxData/Telegraf' + namespace = "InfluxData/Telegraf" ` func (c *CloudWatch) SampleConfig() string { From e84b356a1223c27f349e23f5683fd93b4fb15edf Mon Sep 17 00:00:00 2001 From: Cameron Sparr Date: Fri, 4 Nov 2016 13:18:44 +0000 Subject: [PATCH 12/14] Update etc/telegraf.conf --- etc/telegraf.conf | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/etc/telegraf.conf b/etc/telegraf.conf index 9f95a9ca5..b999713a7 100644 --- a/etc/telegraf.conf +++ b/etc/telegraf.conf @@ -160,7 +160,7 @@ # # Configuration for AWS CloudWatch output. # [[outputs.cloudwatch]] # ## Amazon REGION -# region = 'us-east-1' +# region = "us-east-1" # # ## Amazon Credentials # ## Credentials are loaded in the following order @@ -178,7 +178,7 @@ # #shared_credential_file = "" # # ## Namespace for the CloudWatch MetricDatums -# namespace = 'InfluxData/Telegraf' +# namespace = "InfluxData/Telegraf" # # Configuration for DataDog API to send metrics to. @@ -623,7 +623,7 @@ # # Pull Metric Statistics from Amazon CloudWatch # [[inputs.cloudwatch]] # ## Amazon Region -# region = 'us-east-1' +# region = "us-east-1" # # ## Amazon Credentials # ## Credentials are loaded in the following order @@ -641,21 +641,21 @@ # #shared_credential_file = "" # # ## Requested CloudWatch aggregation Period (required - must be a multiple of 60s) -# period = '1m' +# period = "1m" # # ## Collection Delay (required - must account for metrics availability via CloudWatch API) -# delay = '1m' +# delay = "1m" # # ## Recomended: use metric 'interval' that is a multiple of 'period' to avoid # ## gaps or overlap in pulled data -# interval = '1m' +# interval = "1m" # # ## Configure the TTL for the internal cache of metrics. # ## Defaults to 1 hr if not specified -# #cache_ttl = '10m' +# #cache_ttl = "10m" # # ## Metric Statistic Namespace (required) -# namespace = 'AWS/ELB' +# namespace = "AWS/ELB" # # ## Maximum requests per second. Note that the global default AWS rate limit is # ## 10 reqs/sec, so if you define multiple namespaces, these should add up to a @@ -666,12 +666,12 @@ # ## Defaults to all Metrics in Namespace if nothing is provided # ## Refreshes Namespace available metrics every 1h # #[[inputs.cloudwatch.metrics]] -# # names = ['Latency', 'RequestCount'] +# # names = ["Latency", "RequestCount"] # # # # ## Dimension filters for Metric (optional) # # [[inputs.cloudwatch.metrics.dimensions]] -# # name = 'LoadBalancerName' -# # value = 'p-example' +# # name = "LoadBalancerName" +# # value = "p-example" # # Gather health check statuses from services registered in Consul From 5d3850c44e048f5619a04c14373c3324c2c9c407 Mon Sep 17 00:00:00 2001 From: John Engelman Date: Mon, 7 Nov 2016 06:14:04 -0600 Subject: [PATCH 13/14] Update docs on Cloudwatch. Set default period to 5m. (#2000) --- etc/telegraf.conf | 7 +++---- plugins/inputs/cloudwatch/README.md | 23 +++++++++++++++-------- plugins/inputs/cloudwatch/cloudwatch.go | 13 ++++++++++--- 3 files changed, 28 insertions(+), 15 deletions(-) diff --git a/etc/telegraf.conf b/etc/telegraf.conf index b999713a7..8ebf0a7a7 100644 --- a/etc/telegraf.conf +++ b/etc/telegraf.conf @@ -641,14 +641,14 @@ # #shared_credential_file = "" # # ## Requested CloudWatch aggregation Period (required - must be a multiple of 60s) -# period = "1m" +# period = "5m" # # ## Collection Delay (required - must account for metrics availability via CloudWatch API) -# delay = "1m" +# delay = "5m" # # ## Recomended: use metric 'interval' that is a multiple of 'period' to avoid # ## gaps or overlap in pulled data -# interval = "1m" +# interval = "5m" # # ## Configure the TTL for the internal cache of metrics. # ## Defaults to 1 hr if not specified @@ -1979,4 +1979,3 @@ # # [inputs.webhooks.rollbar] # path = "/rollbar" - diff --git a/plugins/inputs/cloudwatch/README.md b/plugins/inputs/cloudwatch/README.md index ca7145c30..643e18c3b 100644 --- a/plugins/inputs/cloudwatch/README.md +++ b/plugins/inputs/cloudwatch/README.md @@ -20,16 +20,23 @@ API endpoint. In the following order the plugin will attempt to authenticate. ## Amazon Region (required) region = "us-east-1" + # The minimum period for Cloudwatch metrics is 1 minute (60s). However not all + # metrics are made available to the 1 minute period. Some are collected at + # 3 minute and 5 minutes intervals. See https://aws.amazon.com/cloudwatch/faqs/#monitoring. + # Note that if a period is configured that is smaller than the minimum for a + # particular metric, that metric will not be returned by the Cloudwatch API + # and will not be collected by Telegraf. + # ## Requested CloudWatch aggregation Period (required - must be a multiple of 60s) - period = "1m" + period = "5m" ## Collection Delay (required - must account for metrics availability via CloudWatch API) - delay = "1m" + delay = "5m" ## Override global run interval (optional - defaults to global interval) ## Recomended: use metric 'interval' that is a multiple of 'period' to avoid ## gaps or overlap in pulled data - interval = "1m" + interval = "5m" ## Metric Statistic Namespace (required) namespace = "AWS/ELB" @@ -71,16 +78,16 @@ wildcard dimension is ignored. Example: ``` [[inputs.cloudwatch.metrics]] - names = ['Latency'] + names = ["Latency"] ## Dimension filters for Metric (optional) [[inputs.cloudwatch.metrics.dimensions]] - name = 'LoadBalancerName' - value = 'p-example' + name = "LoadBalancerName" + value = "p-example" [[inputs.cloudwatch.metrics.dimensions]] - name = 'AvailabilityZone' - value = '*' + name = "AvailabilityZone" + value = "*" ``` If the following ELBs are available: diff --git a/plugins/inputs/cloudwatch/cloudwatch.go b/plugins/inputs/cloudwatch/cloudwatch.go index c3dbda05b..bc8de313e 100644 --- a/plugins/inputs/cloudwatch/cloudwatch.go +++ b/plugins/inputs/cloudwatch/cloudwatch.go @@ -80,15 +80,22 @@ func (c *CloudWatch) SampleConfig() string { #profile = "" #shared_credential_file = "" + # The minimum period for Cloudwatch metrics is 1 minute (60s). However not all + # metrics are made available to the 1 minute period. Some are collected at + # 3 minute and 5 minutes intervals. See https://aws.amazon.com/cloudwatch/faqs/#monitoring. + # Note that if a period is configured that is smaller than the minimum for a + # particular metric, that metric will not be returned by the Cloudwatch API + # and will not be collected by Telegraf. + # ## Requested CloudWatch aggregation Period (required - must be a multiple of 60s) - period = "1m" + period = "5m" ## Collection Delay (required - must account for metrics availability via CloudWatch API) - delay = "1m" + delay = "5m" ## Recomended: use metric 'interval' that is a multiple of 'period' to avoid ## gaps or overlap in pulled data - interval = "1m" + interval = "5m" ## Configure the TTL for the internal cache of metrics. ## Defaults to 1 hr if not specified From 1c7715780ebc82bd97c2f58292381f421c99ed0f Mon Sep 17 00:00:00 2001 From: Cameron Sparr Date: Fri, 4 Nov 2016 14:12:22 +0000 Subject: [PATCH 14/14] Documentation improvements - fully document aggregator and processor plugins - improve readme.md closes #1989 --- README.md | 229 ++++++++++++++------------- docs/AGGREGATORS_AND_PROCESSORS.md | 59 +++++++ plugins/aggregators/minmax/README.md | 42 +++++ plugins/processors/printer/README.md | 14 ++ 4 files changed, 233 insertions(+), 111 deletions(-) create mode 100644 docs/AGGREGATORS_AND_PROCESSORS.md create mode 100644 plugins/aggregators/minmax/README.md create mode 100644 plugins/processors/printer/README.md diff --git a/README.md b/README.md index efba9e6c7..92ebf19ee 100644 --- a/README.md +++ b/README.md @@ -1,15 +1,23 @@ # Telegraf [![Circle CI](https://circleci.com/gh/influxdata/telegraf.svg?style=svg)](https://circleci.com/gh/influxdata/telegraf) [![Docker pulls](https://img.shields.io/docker/pulls/library/telegraf.svg)](https://hub.docker.com/_/telegraf/) -Telegraf is an agent written in Go for collecting metrics from the system it's -running on, or from other services, and writing them into InfluxDB or other -[outputs](https://github.com/influxdata/telegraf#supported-output-plugins). +Telegraf is an agent written in Go for collecting, processing, aggregating, +and writing metrics. Design goals are to have a minimal memory footprint with a plugin system so that developers in the community can easily add support for collecting metrics from well known services (like Hadoop, Postgres, or Redis) and third party APIs (like Mailchimp, AWS CloudWatch, or Google Analytics). -New input and output plugins are designed to be easy to contribute, +Telegraf is plugin-driven and has the concept of 4 distinct plugins: + +1. [Input Plugins](#input-plugins) collect metrics from the system, services, or 3rd party APIs +2. [Processor Plugins](#processor-plugins) transform, decorate, and/or filter metrics +3. [Aggregator Plugins](#aggregator-plugins) create aggregate metrics (e.g. mean, min, max, quantiles, etc.) +4. [Output Plugins](#output-plugins) write metrics to various destinations + +For more information on Processor and Aggregator plugins please [read this](./docs/AGGREGATORS_AND_PROCESSORS.md). + +New plugins are designed to be easy to contribute, we'll eagerly accept pull requests and will manage the set of plugins that Telegraf supports. See the [contributing guide](CONTRIBUTING.md) for instructions on writing @@ -39,7 +47,7 @@ controlled via `systemctl [action] telegraf` ### yum/apt Repositories: There is a yum/apt repo available for the whole InfluxData stack, see -[here](https://docs.influxdata.com/influxdb/v0.10/introduction/installation/#installation) +[here](https://docs.influxdata.com/influxdb/latest/introduction/installation/#installation) for instructions on setting up the repo. Once it is configured, you will be able to use this repo to install & update telegraf. @@ -127,77 +135,71 @@ telegraf --config telegraf.conf -input-filter cpu:mem -output-filter influxdb See the [configuration guide](docs/CONFIGURATION.md) for a rundown of the more advanced configuration options. -## Supported Input Plugins +## Input Plugins -Telegraf currently has support for collecting metrics from many sources. For -more information on each, please look at the directory of the same name in -`plugins/inputs`. - -Currently implemented sources: - -* [aws cloudwatch](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/cloudwatch) -* [aerospike](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/aerospike) -* [apache](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/apache) -* [bcache](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/bcache) -* [cassandra](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/cassandra) -* [ceph](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/ceph) -* [chrony](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/chrony) -* [consul](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/consul) -* [conntrack](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/conntrack) -* [couchbase](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/couchbase) -* [couchdb](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/couchdb) -* [disque](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/disque) -* [dns query time](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/dns_query) -* [docker](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/docker) -* [dovecot](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/dovecot) -* [elasticsearch](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/elasticsearch) -* [exec](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/exec) (generic executable plugin, support JSON, influx, graphite and nagios) -* [filestat](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/filestat) -* [haproxy](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/haproxy) -* [hddtemp](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/hddtemp) -* [http_response](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/http_response) -* [httpjson](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/httpjson) (generic JSON-emitting http service plugin) -* [influxdb](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/influxdb) -* [ipmi_sensor](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/ipmi_sensor) -* [iptables](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/iptables) -* [jolokia](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/jolokia) -* [leofs](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/leofs) -* [lustre2](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/lustre2) -* [mailchimp](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/mailchimp) -* [memcached](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/memcached) -* [mesos](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/mesos) -* [mongodb](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/mongodb) -* [mysql](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/mysql) -* [net_response](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/net_response) -* [nginx](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/nginx) -* [nsq](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/nsq) -* [nstat](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/nstat) -* [ntpq](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/ntpq) -* [phpfpm](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/phpfpm) -* [phusion passenger](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/passenger) -* [ping](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/ping) -* [postgresql](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/postgresql) -* [postgresql_extensible](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/postgresql_extensible) -* [powerdns](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/powerdns) -* [procstat](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/procstat) -* [prometheus](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/prometheus) -* [puppetagent](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/puppetagent) -* [rabbitmq](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/rabbitmq) -* [raindrops](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/raindrops) -* [redis](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/redis) -* [rethinkdb](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/rethinkdb) -* [riak](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/riak) -* [sensors](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/sensors) -* [snmp](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/snmp) -* [snmp_legacy](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/snmp_legacy) -* [sql server](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/sqlserver) (microsoft) -* [twemproxy](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/twemproxy) -* [varnish](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/varnish) -* [zfs](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/zfs) -* [zookeeper](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/zookeeper) -* [win_perf_counters ](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/win_perf_counters) (windows performance counters) -* [sysstat](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/sysstat) -* [system](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/system) +* [aws cloudwatch](./plugins/inputs/cloudwatch) +* [aerospike](./plugins/inputs/aerospike) +* [apache](./plugins/inputs/apache) +* [bcache](./plugins/inputs/bcache) +* [cassandra](./plugins/inputs/cassandra) +* [ceph](./plugins/inputs/ceph) +* [chrony](./plugins/inputs/chrony) +* [consul](./plugins/inputs/consul) +* [conntrack](./plugins/inputs/conntrack) +* [couchbase](./plugins/inputs/couchbase) +* [couchdb](./plugins/inputs/couchdb) +* [disque](./plugins/inputs/disque) +* [dns query time](./plugins/inputs/dns_query) +* [docker](./plugins/inputs/docker) +* [dovecot](./plugins/inputs/dovecot) +* [elasticsearch](./plugins/inputs/elasticsearch) +* [exec](./plugins/inputs/exec) (generic executable plugin, support JSON, influx, graphite and nagios) +* [filestat](./plugins/inputs/filestat) +* [haproxy](./plugins/inputs/haproxy) +* [hddtemp](./plugins/inputs/hddtemp) +* [http_response](./plugins/inputs/http_response) +* [httpjson](./plugins/inputs/httpjson) (generic JSON-emitting http service plugin) +* [influxdb](./plugins/inputs/influxdb) +* [ipmi_sensor](./plugins/inputs/ipmi_sensor) +* [iptables](./plugins/inputs/iptables) +* [jolokia](./plugins/inputs/jolokia) +* [leofs](./plugins/inputs/leofs) +* [lustre2](./plugins/inputs/lustre2) +* [mailchimp](./plugins/inputs/mailchimp) +* [memcached](./plugins/inputs/memcached) +* [mesos](./plugins/inputs/mesos) +* [mongodb](./plugins/inputs/mongodb) +* [mysql](./plugins/inputs/mysql) +* [net_response](./plugins/inputs/net_response) +* [nginx](./plugins/inputs/nginx) +* [nsq](./plugins/inputs/nsq) +* [nstat](./plugins/inputs/nstat) +* [ntpq](./plugins/inputs/ntpq) +* [phpfpm](./plugins/inputs/phpfpm) +* [phusion passenger](./plugins/inputs/passenger) +* [ping](./plugins/inputs/ping) +* [postgresql](./plugins/inputs/postgresql) +* [postgresql_extensible](./plugins/inputs/postgresql_extensible) +* [powerdns](./plugins/inputs/powerdns) +* [procstat](./plugins/inputs/procstat) +* [prometheus](./plugins/inputs/prometheus) +* [puppetagent](./plugins/inputs/puppetagent) +* [rabbitmq](./plugins/inputs/rabbitmq) +* [raindrops](./plugins/inputs/raindrops) +* [redis](./plugins/inputs/redis) +* [rethinkdb](./plugins/inputs/rethinkdb) +* [riak](./plugins/inputs/riak) +* [sensors](./plugins/inputs/sensors) +* [snmp](./plugins/inputs/snmp) +* [snmp_legacy](./plugins/inputs/snmp_legacy) +* [sql server](./plugins/inputs/sqlserver) (microsoft) +* [twemproxy](./plugins/inputs/twemproxy) +* [varnish](./plugins/inputs/varnish) +* [zfs](./plugins/inputs/zfs) +* [zookeeper](./plugins/inputs/zookeeper) +* [win_perf_counters ](./plugins/inputs/win_perf_counters) (windows performance counters) +* [sysstat](./plugins/inputs/sysstat) +* [system](./plugins/inputs/system) * cpu * mem * net @@ -211,45 +213,50 @@ Currently implemented sources: Telegraf can also collect metrics via the following service plugins: -* [http_listener](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/http_listener) -* [kafka_consumer](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/kafka_consumer) -* [mqtt_consumer](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/mqtt_consumer) -* [nats_consumer](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/nats_consumer) -* [nsq_consumer](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/nsq_consumer) -* [logparser](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/logparser) -* [statsd](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/statsd) -* [tail](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/tail) -* [tcp_listener](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/tcp_listener) -* [udp_listener](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/udp_listener) -* [webhooks](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/webhooks) - * [filestack](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/webhooks/filestack) - * [github](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/webhooks/github) - * [mandrill](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/webhooks/mandrill) - * [rollbar](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/webhooks/rollbar) +* [http_listener](./plugins/inputs/http_listener) +* [kafka_consumer](./plugins/inputs/kafka_consumer) +* [mqtt_consumer](./plugins/inputs/mqtt_consumer) +* [nats_consumer](./plugins/inputs/nats_consumer) +* [nsq_consumer](./plugins/inputs/nsq_consumer) +* [logparser](./plugins/inputs/logparser) +* [statsd](./plugins/inputs/statsd) +* [tail](./plugins/inputs/tail) +* [tcp_listener](./plugins/inputs/tcp_listener) +* [udp_listener](./plugins/inputs/udp_listener) +* [webhooks](./plugins/inputs/webhooks) + * [filestack](./plugins/inputs/webhooks/filestack) + * [github](./plugins/inputs/webhooks/github) + * [mandrill](./plugins/inputs/webhooks/mandrill) + * [rollbar](./plugins/inputs/webhooks/rollbar) -We'll be adding support for many more over the coming months. Read on if you -want to add support for another service or third-party API. +## Processor Plugins -## Supported Output Plugins +* [printer](./plugins/processors/printer) -* [influxdb](https://github.com/influxdata/telegraf/tree/master/plugins/outputs/influxdb) -* [amon](https://github.com/influxdata/telegraf/tree/master/plugins/outputs/amon) -* [amqp](https://github.com/influxdata/telegraf/tree/master/plugins/outputs/amqp) -* [aws kinesis](https://github.com/influxdata/telegraf/tree/master/plugins/outputs/kinesis) -* [aws cloudwatch](https://github.com/influxdata/telegraf/tree/master/plugins/outputs/cloudwatch) -* [datadog](https://github.com/influxdata/telegraf/tree/master/plugins/outputs/datadog) -* [file](https://github.com/influxdata/telegraf/tree/master/plugins/outputs/file) -* [graphite](https://github.com/influxdata/telegraf/tree/master/plugins/outputs/graphite) -* [graylog](https://github.com/influxdata/telegraf/tree/master/plugins/outputs/graylog) -* [instrumental](https://github.com/influxdata/telegraf/tree/master/plugins/outputs/instrumental) -* [kafka](https://github.com/influxdata/telegraf/tree/master/plugins/outputs/kafka) -* [librato](https://github.com/influxdata/telegraf/tree/master/plugins/outputs/librato) -* [mqtt](https://github.com/influxdata/telegraf/tree/master/plugins/outputs/mqtt) -* [nats](https://github.com/influxdata/telegraf/tree/master/plugins/outputs/nats) -* [nsq](https://github.com/influxdata/telegraf/tree/master/plugins/outputs/nsq) -* [opentsdb](https://github.com/influxdata/telegraf/tree/master/plugins/outputs/opentsdb) -* [prometheus](https://github.com/influxdata/telegraf/tree/master/plugins/outputs/prometheus_client) -* [riemann](https://github.com/influxdata/telegraf/tree/master/plugins/outputs/riemann) +## Aggregator Plugins + +* [minmax](./plugins/aggregators/minmax) + +## Output Plugins + +* [influxdb](./plugins/outputs/influxdb) +* [amon](./plugins/outputs/amon) +* [amqp](./plugins/outputs/amqp) +* [aws kinesis](./plugins/outputs/kinesis) +* [aws cloudwatch](./plugins/outputs/cloudwatch) +* [datadog](./plugins/outputs/datadog) +* [file](./plugins/outputs/file) +* [graphite](./plugins/outputs/graphite) +* [graylog](./plugins/outputs/graylog) +* [instrumental](./plugins/outputs/instrumental) +* [kafka](./plugins/outputs/kafka) +* [librato](./plugins/outputs/librato) +* [mqtt](./plugins/outputs/mqtt) +* [nats](./plugins/outputs/nats) +* [nsq](./plugins/outputs/nsq) +* [opentsdb](./plugins/outputs/opentsdb) +* [prometheus](./plugins/outputs/prometheus_client) +* [riemann](./plugins/outputs/riemann) ## Contributing diff --git a/docs/AGGREGATORS_AND_PROCESSORS.md b/docs/AGGREGATORS_AND_PROCESSORS.md new file mode 100644 index 000000000..af4a0e6d0 --- /dev/null +++ b/docs/AGGREGATORS_AND_PROCESSORS.md @@ -0,0 +1,59 @@ +# Telegraf Aggregator & Processor Plugins + +As of release 1.1.0, Telegraf has the concept of Aggregator and Processor Plugins. + +These plugins sit in-between Input & Output plugins, aggregating and processing +metrics as they pass through Telegraf: + +``` +┌───────────┐ +│ │ +│ CPU │───┐ +│ │ │ +└───────────┘ │ + │ +┌───────────┐ │ ┌───────────┐ +│ │ │ │ │ +│ Memory │───┤ ┌──▶│ InfluxDB │ +│ │ │ │ │ │ +└───────────┘ │ ┌─────────────┐ ┌─────────────┐ │ └───────────┘ + │ │ │ │Aggregate │ │ +┌───────────┐ │ │Process │ │ - mean │ │ ┌───────────┐ +│ │ │ │ - transform │ │ - quantiles │ │ │ │ +│ MySQL │───┼───▶│ - decorate │────▶│ - min/max │───┼──▶│ File │ +│ │ │ │ - filter │ │ - count │ │ │ │ +└───────────┘ │ │ │ │ │ │ └───────────┘ + │ └─────────────┘ └─────────────┘ │ +┌───────────┐ │ │ ┌───────────┐ +│ │ │ │ │ │ +│ SNMP │───┤ └──▶│ Kafka │ +│ │ │ │ │ +└───────────┘ │ └───────────┘ + │ +┌───────────┐ │ +│ │ │ +│ Docker │───┘ +│ │ +└───────────┘ +``` + +Both Aggregators and Processors analyze metrics as they pass through Telegraf. + +**Processor** plugins process metrics as they pass through and immediately emit +results based on the values they process. For example, this could be printing +all metrics or adding a tag to all metrics that pass through. + +**Aggregator** plugins, on the other hand, are a bit more complicated. Aggregators +are typically for emitting new _aggregate_ metrics, such as a running mean, +minimum, maximum, quantiles, or standard deviation. For this reason, all _aggregator_ +plugins are configured with a `period`. The `period` is the size of the window +of metrics that each _aggregate_ represents. In other words, the emitted +_aggregate_ metric will be the aggregated value of the past `period` seconds. +Since many users will only care about their aggregates and not every single metric +gathered, there is also a `drop_original` argument, which tells Telegraf to only +emit the aggregates and not the original metrics. + +**NOTE** That since aggregators only aggregate metrics within their period, that +historical data is not supported. In other words, if your metric timestamp is more +than `now() - period` in the past, it will not be aggregated. If this is a feature +that you need, please comment on this [github issue](https://github.com/influxdata/telegraf/issues/1992) diff --git a/plugins/aggregators/minmax/README.md b/plugins/aggregators/minmax/README.md new file mode 100644 index 000000000..f7405b78c --- /dev/null +++ b/plugins/aggregators/minmax/README.md @@ -0,0 +1,42 @@ +# MinMax Aggregator Plugin + +The minmax aggregator plugin aggregates min & max values of each field it sees, +emitting the aggrate every `period` seconds. + +### Configuration: + +```toml +# Keep the aggregate min/max of each metric passing through. +[[aggregators.minmax]] + ## General Aggregator Arguments: + ## The period on which to flush & clear the aggregator. + period = "30s" + ## If true, the original metric will be dropped by the + ## aggregator and will not get sent to the output plugins. + drop_original = false +``` + +### Measurements & Fields: + +- measurement1 + - field1_max + - field1_min + +### Tags: + +No tags are applied by this aggregator. + +### Example Output: + +``` +$ telegraf --config telegraf.conf --quiet +system,host=tars load1=1.72 1475583980000000000 +system,host=tars load1=1.6 1475583990000000000 +system,host=tars load1=1.66 1475584000000000000 +system,host=tars load1=1.63 1475584010000000000 +system,host=tars load1_max=1.72,load1_min=1.6 1475584010000000000 +system,host=tars load1=1.46 1475584020000000000 +system,host=tars load1=1.39 1475584030000000000 +system,host=tars load1=1.41 1475584040000000000 +system,host=tars load1_max=1.46,load1_min=1.39 1475584040000000000 +``` diff --git a/plugins/processors/printer/README.md b/plugins/processors/printer/README.md new file mode 100644 index 000000000..9a79e16fd --- /dev/null +++ b/plugins/processors/printer/README.md @@ -0,0 +1,14 @@ +# Printer Processor Plugin + +The printer processor plugin simple prints every metric passing through it. + +### Configuration: + +```toml +# Print all metrics that pass through this filter. +[[processors.printer]] +``` + +### Tags: + +No tags are applied by this processor.