From cce35da366ac95cb41568047b908d14fac5fe9ce Mon Sep 17 00:00:00 2001 From: Cameron Sparr Date: Thu, 7 Apr 2016 09:26:35 -0600 Subject: [PATCH 01/25] Godeps_windows: update file --- Godeps_windows | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/Godeps_windows b/Godeps_windows index f499fa915..ab3004bb8 100644 --- a/Godeps_windows +++ b/Godeps_windows @@ -1,3 +1,4 @@ +github.com/Microsoft/go-winio 9f57cbbcbcb41dea496528872a4f0e37a4f7ae98 github.com/Shopify/sarama 8aadb476e66ca998f2f6bb3c993e9a2daa3666b9 github.com/Sirupsen/logrus 219c8cb75c258c552e999735be6df753ffc7afdc github.com/StackExchange/wmi f3e2bae1e0cb5aef83e319133eabfee30013a4a5 @@ -9,24 +10,24 @@ github.com/couchbase/go-couchbase cb664315a324d87d19c879d9cc67fda6be8c2ac1 github.com/couchbase/gomemcached a5ea6356f648fec6ab89add00edd09151455b4b2 github.com/couchbase/goutils 5823a0cbaaa9008406021dc5daf80125ea30bba6 github.com/dancannon/gorethink e7cac92ea2bc52638791a021f212145acfedb1fc -github.com/davecgh/go-spew fc32781af5e85e548d3f1abaf0fa3dbe8a72495c +github.com/davecgh/go-spew 5215b55f46b2b919f50a1df0eaa5886afe4e3b3d +github.com/docker/engine-api 8924d6900370b4c7e7984be5adc61f50a80d7537 +github.com/docker/go-connections f549a9393d05688dff0992ef3efd8bbe6c628aeb +github.com/docker/go-units 5d2041e26a699eaca682e2ea41c8f891e1060444 github.com/eapache/go-resiliency b86b1ec0dd4209a588dc1285cdd471e73525c0b3 github.com/eapache/queue ded5959c0d4e360646dc9e9908cff48666781367 github.com/eclipse/paho.mqtt.golang 4ab3e867810d1ec5f35157c59e965054dbf43a0d -github.com/fsouza/go-dockerclient a49c8269a6899cae30da1f8a4b82e0ce945f9967 -github.com/go-ini/ini 776aa739ce9373377cd16f526cdf06cb4c89b40f github.com/go-ole/go-ole 50055884d646dd9434f16bbb5c9801749b9bafe4 github.com/go-sql-driver/mysql 1fca743146605a172a266e1654e01e5cd5669bee github.com/golang/protobuf 552c7b9542c194800fd493123b3798ef0a832032 -github.com/golang/snappy 5979233c5d6225d4a8e438cdd0b411888449ddab +github.com/golang/snappy 427fb6fc07997f43afa32f35e850833760e489a7 github.com/gonuts/go-shellquote e842a11b24c6abfb3dd27af69a17f482e4b483c2 github.com/gorilla/context 1ea25387ff6f684839d82767c1733ff4d4d15d0a github.com/gorilla/mux c9e326e2bdec29039a3761c07bece13133863e1e github.com/hailocab/go-hostpool e80d13ce29ede4452c43dea11e79b9bc8a15b478 github.com/influxdata/config b79f6829346b8d6e78ba73544b1e1038f1f1c9da -github.com/influxdata/influxdb c190778997f4154294e6160c41b90140641ac915 +github.com/influxdata/influxdb e3fef5593c21644f2b43af55d6e17e70910b0e48 github.com/influxdata/toml af4df43894b16e3fd2b788d01bd27ad0776ef2d0 -github.com/jmespath/go-jmespath 0b12d6b521d83fc7f755e7cfc1b1fbdd35a01a74 github.com/klauspost/crc32 19b0b332c9e4516a6370a0456e6182c3b5036720 github.com/lib/pq e182dc4027e2ded4b19396d638610f2653295f36 github.com/lxn/win 9a7734ea4db26bc593d52f6a8a957afdad39c5c1 @@ -37,7 +38,6 @@ github.com/naoina/go-stringutil 6b638e95a32d0c1131db0e7fe83775cbea4a0d0b github.com/nats-io/nats b13fc9d12b0b123ebc374e6b808c6228ae4234a3 github.com/nats-io/nuid 4f84f5f3b2786224e336af2e13dba0a0a80b76fa github.com/nsqio/go-nsq 0b80d6f05e15ca1930e0c5e1d540ed627e299980 -github.com/pmezard/go-difflib 792786c7400a136282c1664665ae0a8db921c6c2 github.com/prometheus/client_golang 18acf9993a863f4c4b40612e19cdd243e7c86831 github.com/prometheus/client_model fa8ad6fec33561be4280a8f0514318c79d7f6cb6 github.com/prometheus/common e8eabff8812b05acf522b45fdcd725a785188e37 @@ -47,9 +47,8 @@ github.com/shirou/gopsutil 1f32ce1bb380845be7f5d174ac641a2c592c0c42 github.com/shirou/w32 ada3ba68f000aa1b58580e45c9d308fe0b7fc5c5 github.com/soniah/gosnmp b1b4f885b12c5dcbd021c5cee1c904110de6db7d github.com/streadway/amqp b4f3ceab0337f013208d31348b578d83c0064744 -github.com/stretchr/objx 1a9d0bb9f541897e62256577b352fdbc1fb4fd94 github.com/stretchr/testify 1f4a1643a57e798696635ea4c126e9127adb7d3c -github.com/wvanbergen/kafka 1a8639a45164fcc245d5c7b4bd3ccfbd1a0ffbf3 +github.com/wvanbergen/kafka 46f9a1cf3f670edec492029fadded9c2d9e18866 github.com/wvanbergen/kazoo-go 0f768712ae6f76454f987c3356177e138df258f8 github.com/zensqlmonitor/go-mssqldb ffe5510c6fa5e15e6d983210ab501c815b56b363 golang.org/x/net 6acef71eb69611914f7a30939ea9f6e194c78172 From dfbe231a51bea200b45cd84d0c014223c6009fdd Mon Sep 17 00:00:00 2001 From: Luke Swithenbank Date: Thu, 31 Mar 2016 18:33:28 +1100 Subject: [PATCH 02/25] add http_response plugin --- plugins/inputs/http_response/README.md | 36 +++++++ plugins/inputs/http_response/http_response.go | 95 +++++++++++++++++++ 2 files changed, 131 insertions(+) create mode 100644 plugins/inputs/http_response/README.md create mode 100644 plugins/inputs/http_response/http_response.go diff --git a/plugins/inputs/http_response/README.md b/plugins/inputs/http_response/README.md new file mode 100644 index 000000000..13da76097 --- /dev/null +++ b/plugins/inputs/http_response/README.md @@ -0,0 +1,36 @@ +# Example Input Plugin + +This input plugin will test HTTP/HTTPS connections. + +### Configuration: + +``` +# List of UDP/TCP connections you want to check +[[inputs.http_response]] + # Server address (default http://localhost) + address = "http://github.com:80" + # Set http response timeout (default 1.0) + response_timeout = 1.0 + # HTTP Method (default "GET") + method = "GET" +``` + +### Measurements & Fields: + +- http_response + - response_time (float, seconds) + - http_response_code (int) #The code received + +### Tags: + +- All measurements have the following tags: + - server + - port + - protocol + +### Example Output: + +``` +$ ./telegraf -config telegraf.conf -input-filter http_response -test +http_response,server=http://192.168.2.2:2000,method=GET response_time=0.18070360500000002,http_response_code=200 1454785464182527094 +``` diff --git a/plugins/inputs/http_response/http_response.go b/plugins/inputs/http_response/http_response.go new file mode 100644 index 000000000..e19c698a8 --- /dev/null +++ b/plugins/inputs/http_response/http_response.go @@ -0,0 +1,95 @@ +package http_response + +import ( + "errors" + "net/http" + "net/url" + "time" + + "github.com/influxdata/telegraf" + "github.com/influxdata/telegraf/plugins/inputs" +) + +// HttpResponses struct +type HttpResponse struct { + Address string + Method string + ResponseTimeout int +} + +func (_ *HttpResponse) Description() string { + return "HTTP/HTTPS request given an address a method and a timeout" +} + +var sampleConfig = ` + ## Server address (default http://localhost) + address = "http://github.com:80" + ## Set response_timeout (default 1 seconds) + response_timeout = 1 + ## HTTP Method + method = "GET" +` + +func (_ *HttpResponse) SampleConfig() string { + return sampleConfig +} + +func (h *HttpResponse) HttpGather() (map[string]interface{}, error) { + // Prepare fields + fields := make(map[string]interface{}) + + client := &http.Client{ + Timeout: time.Second * time.Duration(h.ResponseTimeout), + } + request, err := http.NewRequest(h.Method, h.Address, nil) + if err != nil { + return nil, err + } + // Start Timer + start := time.Now() + resp, err := client.Do(request) + if err != nil { + return nil, err + } + fields["response_time"] = time.Since(start).Seconds() + fields["http_response_code"] = resp.StatusCode + return fields, nil +} + +func (c *HttpResponse) Gather(acc telegraf.Accumulator) error { + // Set default values + if c.ResponseTimeout < 1 { + c.ResponseTimeout = 1 + } + // Check send and expected string + if c.Method == "" { + c.Method = "GET" + } + if c.Address == "" { + c.Address = "http://localhost" + } + addr, err := url.Parse(c.Address) + if err != nil { + return err + } + if addr.Scheme != "http" && addr.Scheme != "https" { + return errors.New("Only http and https are supported") + } + // Prepare data + tags := map[string]string{"server": c.Address, "method": c.Method} + var fields map[string]interface{} + // Gather data + fields, err = c.HttpGather() + if err != nil { + return err + } + // Add metrics + acc.AddFields("http_response", fields, tags) + return nil +} + +func init() { + inputs.Add("http_response", func() telegraf.Input { + return &HttpResponse{} + }) +} From 70aa0ef85da3bd67c0978c3cd23fca2ca9f06af4 Mon Sep 17 00:00:00 2001 From: Luke Swithenbank Date: Thu, 31 Mar 2016 19:47:10 +1100 Subject: [PATCH 03/25] add plugin to all --- plugins/inputs/all/all.go | 1 + 1 file changed, 1 insertion(+) diff --git a/plugins/inputs/all/all.go b/plugins/inputs/all/all.go index 52ee6c13d..b8534fd6d 100644 --- a/plugins/inputs/all/all.go +++ b/plugins/inputs/all/all.go @@ -16,6 +16,7 @@ import ( _ "github.com/influxdata/telegraf/plugins/inputs/github_webhooks" _ "github.com/influxdata/telegraf/plugins/inputs/haproxy" _ "github.com/influxdata/telegraf/plugins/inputs/httpjson" + _ "github.com/influxdata/telegraf/plugins/inputs/http_response" _ "github.com/influxdata/telegraf/plugins/inputs/influxdb" _ "github.com/influxdata/telegraf/plugins/inputs/ipmi_sensor" _ "github.com/influxdata/telegraf/plugins/inputs/jolokia" From 207ab5a0d1d9f1c33aa168c99b21e487a8d38462 Mon Sep 17 00:00:00 2001 From: Luke Swithenbank Date: Thu, 31 Mar 2016 19:54:08 +1100 Subject: [PATCH 04/25] update to make a working sample_config --- plugins/inputs/http_response/README.md | 6 +++--- plugins/inputs/http_response/http_response.go | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/plugins/inputs/http_response/README.md b/plugins/inputs/http_response/README.md index 13da76097..b70bbde72 100644 --- a/plugins/inputs/http_response/README.md +++ b/plugins/inputs/http_response/README.md @@ -8,9 +8,9 @@ This input plugin will test HTTP/HTTPS connections. # List of UDP/TCP connections you want to check [[inputs.http_response]] # Server address (default http://localhost) - address = "http://github.com:80" - # Set http response timeout (default 1.0) - response_timeout = 1.0 + address = "https://github.com" + # Set http response timeout (default 10) + response_timeout = 10 # HTTP Method (default "GET") method = "GET" ``` diff --git a/plugins/inputs/http_response/http_response.go b/plugins/inputs/http_response/http_response.go index e19c698a8..d2d35025a 100644 --- a/plugins/inputs/http_response/http_response.go +++ b/plugins/inputs/http_response/http_response.go @@ -23,9 +23,9 @@ func (_ *HttpResponse) Description() string { var sampleConfig = ` ## Server address (default http://localhost) - address = "http://github.com:80" + address = "https://github.com" ## Set response_timeout (default 1 seconds) - response_timeout = 1 + response_timeout = 10 ## HTTP Method method = "GET" ` From b7435b9cd18949b5291301b571fa32f473b434d2 Mon Sep 17 00:00:00 2001 From: Luke Swithenbank Date: Thu, 31 Mar 2016 19:55:17 +1100 Subject: [PATCH 05/25] fmt --- plugins/inputs/all/all.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/inputs/all/all.go b/plugins/inputs/all/all.go index b8534fd6d..b28291c24 100644 --- a/plugins/inputs/all/all.go +++ b/plugins/inputs/all/all.go @@ -15,8 +15,8 @@ import ( _ "github.com/influxdata/telegraf/plugins/inputs/exec" _ "github.com/influxdata/telegraf/plugins/inputs/github_webhooks" _ "github.com/influxdata/telegraf/plugins/inputs/haproxy" - _ "github.com/influxdata/telegraf/plugins/inputs/httpjson" _ "github.com/influxdata/telegraf/plugins/inputs/http_response" + _ "github.com/influxdata/telegraf/plugins/inputs/httpjson" _ "github.com/influxdata/telegraf/plugins/inputs/influxdb" _ "github.com/influxdata/telegraf/plugins/inputs/ipmi_sensor" _ "github.com/influxdata/telegraf/plugins/inputs/jolokia" From 7219efbdb76006fe52c475994b5608af19dcdaaf Mon Sep 17 00:00:00 2001 From: Luke Swithenbank Date: Thu, 31 Mar 2016 20:53:51 +1100 Subject: [PATCH 06/25] add the ability to parse http headers --- plugins/inputs/http_response/http_response.go | 52 +++++++++++++------ 1 file changed, 36 insertions(+), 16 deletions(-) diff --git a/plugins/inputs/http_response/http_response.go b/plugins/inputs/http_response/http_response.go index d2d35025a..df21311ae 100644 --- a/plugins/inputs/http_response/http_response.go +++ b/plugins/inputs/http_response/http_response.go @@ -1,23 +1,28 @@ package http_response import ( + "bufio" "errors" "net/http" + "net/textproto" "net/url" + "strings" "time" "github.com/influxdata/telegraf" "github.com/influxdata/telegraf/plugins/inputs" ) -// HttpResponses struct -type HttpResponse struct { +// HTTPResponse struct +type HTTPResponse struct { Address string Method string ResponseTimeout int + Headers string } -func (_ *HttpResponse) Description() string { +// Description returns the plugin Description +func (h *HTTPResponse) Description() string { return "HTTP/HTTPS request given an address a method and a timeout" } @@ -28,13 +33,19 @@ var sampleConfig = ` response_timeout = 10 ## HTTP Method method = "GET" + ## HTTP Request Headers + headers = ''' + Host: github.com + ''' ` -func (_ *HttpResponse) SampleConfig() string { +// SampleConfig returns the plugin SampleConfig +func (h *HTTPResponse) SampleConfig() string { return sampleConfig } -func (h *HttpResponse) HttpGather() (map[string]interface{}, error) { +// HTTPGather gathers all fields and returns any errors it encounters +func (h *HTTPResponse) HTTPGather() (map[string]interface{}, error) { // Prepare fields fields := make(map[string]interface{}) @@ -45,6 +56,14 @@ func (h *HttpResponse) HttpGather() (map[string]interface{}, error) { if err != nil { return nil, err } + h.Headers = strings.TrimSpace(h.Headers) + "\n\n" + reader := bufio.NewReader(strings.NewReader(h.Headers)) + tp := textproto.NewReader(reader) + mimeHeader, err := tp.ReadMIMEHeader() + if err != nil { + return nil, err + } + request.Header = http.Header(mimeHeader) // Start Timer start := time.Now() resp, err := client.Do(request) @@ -56,19 +75,20 @@ func (h *HttpResponse) HttpGather() (map[string]interface{}, error) { return fields, nil } -func (c *HttpResponse) Gather(acc telegraf.Accumulator) error { +// Gather gets all metric fields and tags and returns any errors it encounters +func (h *HTTPResponse) Gather(acc telegraf.Accumulator) error { // Set default values - if c.ResponseTimeout < 1 { - c.ResponseTimeout = 1 + if h.ResponseTimeout < 1 { + h.ResponseTimeout = 1 } // Check send and expected string - if c.Method == "" { - c.Method = "GET" + if h.Method == "" { + h.Method = "GET" } - if c.Address == "" { - c.Address = "http://localhost" + if h.Address == "" { + h.Address = "http://localhost" } - addr, err := url.Parse(c.Address) + addr, err := url.Parse(h.Address) if err != nil { return err } @@ -76,10 +96,10 @@ func (c *HttpResponse) Gather(acc telegraf.Accumulator) error { return errors.New("Only http and https are supported") } // Prepare data - tags := map[string]string{"server": c.Address, "method": c.Method} + tags := map[string]string{"server": h.Address, "method": h.Method} var fields map[string]interface{} // Gather data - fields, err = c.HttpGather() + fields, err = h.HTTPGather() if err != nil { return err } @@ -90,6 +110,6 @@ func (c *HttpResponse) Gather(acc telegraf.Accumulator) error { func init() { inputs.Add("http_response", func() telegraf.Input { - return &HttpResponse{} + return &HTTPResponse{} }) } From f947fa86e37a1679eacb7ae98b06a8826d03a5d3 Mon Sep 17 00:00:00 2001 From: Luke Swithenbank Date: Thu, 31 Mar 2016 21:18:19 +1100 Subject: [PATCH 07/25] update to allow for following redirects --- plugins/inputs/http_response/README.md | 19 +++++++---- plugins/inputs/http_response/http_response.go | 33 ++++++++++++++++--- 2 files changed, 41 insertions(+), 11 deletions(-) diff --git a/plugins/inputs/http_response/README.md b/plugins/inputs/http_response/README.md index b70bbde72..99770e526 100644 --- a/plugins/inputs/http_response/README.md +++ b/plugins/inputs/http_response/README.md @@ -7,12 +7,18 @@ This input plugin will test HTTP/HTTPS connections. ``` # List of UDP/TCP connections you want to check [[inputs.http_response]] - # Server address (default http://localhost) - address = "https://github.com" - # Set http response timeout (default 10) + ## Server address (default http://localhost) + address = "http://github.com" + ## Set response_timeout (default 10 seconds) response_timeout = 10 - # HTTP Method (default "GET") + ## HTTP Method method = "GET" + ## HTTP Request Headers + headers = ''' + Host: github.com + ''' + ## Whether to follow redirects from the server (defaults to false) + follow_redirects = true ``` ### Measurements & Fields: @@ -25,12 +31,11 @@ This input plugin will test HTTP/HTTPS connections. - All measurements have the following tags: - server - - port - - protocol + - method ### Example Output: ``` $ ./telegraf -config telegraf.conf -input-filter http_response -test -http_response,server=http://192.168.2.2:2000,method=GET response_time=0.18070360500000002,http_response_code=200 1454785464182527094 +http_response,method=GET,server=http://www.github.com http_response_code=200i,response_time=6.223266528 1459419354977857955 ``` diff --git a/plugins/inputs/http_response/http_response.go b/plugins/inputs/http_response/http_response.go index df21311ae..09569fe73 100644 --- a/plugins/inputs/http_response/http_response.go +++ b/plugins/inputs/http_response/http_response.go @@ -3,9 +3,11 @@ package http_response import ( "bufio" "errors" + "fmt" "net/http" "net/textproto" "net/url" + "os" "strings" "time" @@ -19,6 +21,7 @@ type HTTPResponse struct { Method string ResponseTimeout int Headers string + FollowRedirects bool } // Description returns the plugin Description @@ -28,8 +31,8 @@ func (h *HTTPResponse) Description() string { var sampleConfig = ` ## Server address (default http://localhost) - address = "https://github.com" - ## Set response_timeout (default 1 seconds) + address = "http://github.com" + ## Set response_timeout (default 10 seconds) response_timeout = 10 ## HTTP Method method = "GET" @@ -37,6 +40,8 @@ var sampleConfig = ` headers = ''' Host: github.com ''' + ## Whether to follow redirects from the server (defaults to false) + follow_redirects = true ` // SampleConfig returns the plugin SampleConfig @@ -44,6 +49,8 @@ func (h *HTTPResponse) SampleConfig() string { return sampleConfig } +var ErrRedirectAttempted = errors.New("redirect") + // HTTPGather gathers all fields and returns any errors it encounters func (h *HTTPResponse) HTTPGather() (map[string]interface{}, error) { // Prepare fields @@ -52,6 +59,14 @@ func (h *HTTPResponse) HTTPGather() (map[string]interface{}, error) { client := &http.Client{ Timeout: time.Second * time.Duration(h.ResponseTimeout), } + + if h.FollowRedirects == false { + fmt.Println(h.FollowRedirects) + client.CheckRedirect = func(req *http.Request, via []*http.Request) error { + return ErrRedirectAttempted + } + } + request, err := http.NewRequest(h.Method, h.Address, nil) if err != nil { return nil, err @@ -66,9 +81,19 @@ func (h *HTTPResponse) HTTPGather() (map[string]interface{}, error) { request.Header = http.Header(mimeHeader) // Start Timer start := time.Now() + request.Write(os.Stdout) resp, err := client.Do(request) if err != nil { - return nil, err + if h.FollowRedirects { + return nil, err + } + if urlError, ok := err.(*url.Error); ok && + urlError.Err == ErrRedirectAttempted { + fmt.Println(err) + err = nil + } else { + return nil, err + } } fields["response_time"] = time.Since(start).Seconds() fields["http_response_code"] = resp.StatusCode @@ -79,7 +104,7 @@ func (h *HTTPResponse) HTTPGather() (map[string]interface{}, error) { func (h *HTTPResponse) Gather(acc telegraf.Accumulator) error { // Set default values if h.ResponseTimeout < 1 { - h.ResponseTimeout = 1 + h.ResponseTimeout = 10 } // Check send and expected string if h.Method == "" { From 73a7916ce3f7b43005097ba591157f43fda45d2e Mon Sep 17 00:00:00 2001 From: Luke Swithenbank Date: Thu, 31 Mar 2016 22:06:47 +1100 Subject: [PATCH 08/25] take a request body as a param --- plugins/inputs/http_response/README.md | 6 +++++- plugins/inputs/http_response/http_response.go | 20 ++++++++++++------- 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/plugins/inputs/http_response/README.md b/plugins/inputs/http_response/README.md index 99770e526..f2f45b2af 100644 --- a/plugins/inputs/http_response/README.md +++ b/plugins/inputs/http_response/README.md @@ -11,7 +11,7 @@ This input plugin will test HTTP/HTTPS connections. address = "http://github.com" ## Set response_timeout (default 10 seconds) response_timeout = 10 - ## HTTP Method + ## HTTP Request Method method = "GET" ## HTTP Request Headers headers = ''' @@ -19,6 +19,10 @@ This input plugin will test HTTP/HTTPS connections. ''' ## Whether to follow redirects from the server (defaults to false) follow_redirects = true + ## Optional HTTP Request Body + body = ''' + {'fake':'data'} + ''' ``` ### Measurements & Fields: diff --git a/plugins/inputs/http_response/http_response.go b/plugins/inputs/http_response/http_response.go index 09569fe73..dc4b2df60 100644 --- a/plugins/inputs/http_response/http_response.go +++ b/plugins/inputs/http_response/http_response.go @@ -3,11 +3,10 @@ package http_response import ( "bufio" "errors" - "fmt" + "io" "net/http" "net/textproto" "net/url" - "os" "strings" "time" @@ -18,6 +17,7 @@ import ( // HTTPResponse struct type HTTPResponse struct { Address string + Body string Method string ResponseTimeout int Headers string @@ -34,7 +34,7 @@ var sampleConfig = ` address = "http://github.com" ## Set response_timeout (default 10 seconds) response_timeout = 10 - ## HTTP Method + ## HTTP Request Method method = "GET" ## HTTP Request Headers headers = ''' @@ -42,6 +42,10 @@ var sampleConfig = ` ''' ## Whether to follow redirects from the server (defaults to false) follow_redirects = true + ## Optional HTTP Request Body + body = ''' + {'fake':'data'} + ''' ` // SampleConfig returns the plugin SampleConfig @@ -49,6 +53,7 @@ func (h *HTTPResponse) SampleConfig() string { return sampleConfig } +// ErrRedirectAttempted indicates that a redirect occurred var ErrRedirectAttempted = errors.New("redirect") // HTTPGather gathers all fields and returns any errors it encounters @@ -61,13 +66,16 @@ func (h *HTTPResponse) HTTPGather() (map[string]interface{}, error) { } if h.FollowRedirects == false { - fmt.Println(h.FollowRedirects) client.CheckRedirect = func(req *http.Request, via []*http.Request) error { return ErrRedirectAttempted } } - request, err := http.NewRequest(h.Method, h.Address, nil) + var body io.Reader + if h.Body != "" { + body = strings.NewReader(h.Body) + } + request, err := http.NewRequest(h.Method, h.Address, body) if err != nil { return nil, err } @@ -81,7 +89,6 @@ func (h *HTTPResponse) HTTPGather() (map[string]interface{}, error) { request.Header = http.Header(mimeHeader) // Start Timer start := time.Now() - request.Write(os.Stdout) resp, err := client.Do(request) if err != nil { if h.FollowRedirects { @@ -89,7 +96,6 @@ func (h *HTTPResponse) HTTPGather() (map[string]interface{}, error) { } if urlError, ok := err.(*url.Error); ok && urlError.Err == ErrRedirectAttempted { - fmt.Println(err) err = nil } else { return nil, err From 437bd87d7c4994052def798b5e21e8d9ee1f7c28 Mon Sep 17 00:00:00 2001 From: Luke Swithenbank Date: Mon, 4 Apr 2016 12:20:07 +1000 Subject: [PATCH 09/25] added tests and did some refactoring --- plugins/inputs/http_response/http_response.go | 56 ++-- .../http_response/http_response_test.go | 245 ++++++++++++++++++ 2 files changed, 281 insertions(+), 20 deletions(-) create mode 100644 plugins/inputs/http_response/http_response_test.go diff --git a/plugins/inputs/http_response/http_response.go b/plugins/inputs/http_response/http_response.go index dc4b2df60..cee33795a 100644 --- a/plugins/inputs/http_response/http_response.go +++ b/plugins/inputs/http_response/http_response.go @@ -40,12 +40,12 @@ var sampleConfig = ` headers = ''' Host: github.com ''' - ## Whether to follow redirects from the server (defaults to false) - follow_redirects = true - ## Optional HTTP Request Body - body = ''' - {'fake':'data'} - ''' + ## Whether to follow redirects from the server (defaults to false) + follow_redirects = true + ## Optional HTTP Request Body + body = ''' + {'fake':'data'} + ''' ` // SampleConfig returns the plugin SampleConfig @@ -56,20 +56,40 @@ func (h *HTTPResponse) SampleConfig() string { // ErrRedirectAttempted indicates that a redirect occurred var ErrRedirectAttempted = errors.New("redirect") +// CreateHttpClient creates an http client which will timeout at the specified +// timeout period and can follow redirects if specified +func CreateHttpClient(followRedirects bool, ResponseTimeout time.Duration) *http.Client { + client := &http.Client{ + Timeout: time.Second * ResponseTimeout, + } + + if followRedirects == false { + client.CheckRedirect = func(req *http.Request, via []*http.Request) error { + return ErrRedirectAttempted + } + } + return client +} + +// ParseHeaders takes a string of newline seperated http headers and returns a +// http.Header object. An error is returned if the headers cannot be parsed. +func ParseHeaders(headers string) (http.Header, error) { + headers = strings.TrimSpace(headers) + "\n\n" + reader := bufio.NewReader(strings.NewReader(headers)) + tp := textproto.NewReader(reader) + mimeHeader, err := tp.ReadMIMEHeader() + if err != nil { + return nil, err + } + return http.Header(mimeHeader), nil +} + // HTTPGather gathers all fields and returns any errors it encounters func (h *HTTPResponse) HTTPGather() (map[string]interface{}, error) { // Prepare fields fields := make(map[string]interface{}) - client := &http.Client{ - Timeout: time.Second * time.Duration(h.ResponseTimeout), - } - - if h.FollowRedirects == false { - client.CheckRedirect = func(req *http.Request, via []*http.Request) error { - return ErrRedirectAttempted - } - } + client := CreateHttpClient(h.FollowRedirects, time.Duration(h.ResponseTimeout)) var body io.Reader if h.Body != "" { @@ -79,14 +99,10 @@ func (h *HTTPResponse) HTTPGather() (map[string]interface{}, error) { if err != nil { return nil, err } - h.Headers = strings.TrimSpace(h.Headers) + "\n\n" - reader := bufio.NewReader(strings.NewReader(h.Headers)) - tp := textproto.NewReader(reader) - mimeHeader, err := tp.ReadMIMEHeader() + request.Header, err = ParseHeaders(h.Headers) if err != nil { return nil, err } - request.Header = http.Header(mimeHeader) // Start Timer start := time.Now() resp, err := client.Do(request) diff --git a/plugins/inputs/http_response/http_response_test.go b/plugins/inputs/http_response/http_response_test.go new file mode 100644 index 000000000..0f568e3b4 --- /dev/null +++ b/plugins/inputs/http_response/http_response_test.go @@ -0,0 +1,245 @@ +package http_response + +import ( + "fmt" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "io/ioutil" + "net/http" + "net/http/httptest" + "testing" + "time" +) + +func TestParseHeaders(t *testing.T) { + fakeHeaders := ` +Accept: text/plain +Content-Type: application/json +Cache-Control: no-cache +` + headers, err := ParseHeaders(fakeHeaders) + require.NoError(t, err) + testHeaders := make(http.Header) + testHeaders.Add("Accept", "text/plain") + testHeaders.Add("Content-Type", "application/json") + testHeaders.Add("Cache-Control", "no-cache") + assert.Equal(t, testHeaders, headers) + + headers, err = ParseHeaders("Accept text/plain") + require.Error(t, err) +} + +func setUpTestMux() http.Handler { + mux := http.NewServeMux() + mux.HandleFunc("/redirect", func(w http.ResponseWriter, req *http.Request) { + http.Redirect(w, req, "/good", http.StatusMovedPermanently) + }) + mux.HandleFunc("/good", func(w http.ResponseWriter, req *http.Request) { + fmt.Fprintf(w, "hit the good page!") + }) + mux.HandleFunc("/badredirect", func(w http.ResponseWriter, req *http.Request) { + http.Redirect(w, req, "/badredirect", http.StatusMovedPermanently) + }) + mux.HandleFunc("/mustbepostmethod", func(w http.ResponseWriter, req *http.Request) { + if req.Method != "POST" { + http.Error(w, "method wasn't post", http.StatusMethodNotAllowed) + return + } + fmt.Fprintf(w, "used post correctly!") + }) + mux.HandleFunc("/musthaveabody", func(w http.ResponseWriter, req *http.Request) { + body, err := ioutil.ReadAll(req.Body) + req.Body.Close() + if err != nil { + http.Error(w, "couldn't read request body", http.StatusBadRequest) + return + } + if string(body) == "" { + http.Error(w, "body was empty", http.StatusBadRequest) + return + } + fmt.Fprintf(w, "sent a body!") + }) + mux.HandleFunc("/twosecondnap", func(w http.ResponseWriter, req *http.Request) { + time.Sleep(time.Second * 2) + return + }) + return mux +} + +func TestFields(t *testing.T) { + mux := setUpTestMux() + ts := httptest.NewServer(mux) + defer ts.Close() + + h := &HTTPResponse{ + Address: ts.URL + "/good", + Body: "{ 'test': 'data'}", + Method: "GET", + ResponseTimeout: 20, + Headers: ` +Content-Type: application/json +`, + FollowRedirects: true, + } + fields, err := h.HTTPGather() + require.NoError(t, err) + assert.NotEmpty(t, fields) + if assert.NotNil(t, fields["http_response_code"]) { + assert.Equal(t, http.StatusOK, fields["http_response_code"]) + } + assert.NotNil(t, fields["response_time"]) + +} + +func TestRedirects(t *testing.T) { + mux := setUpTestMux() + ts := httptest.NewServer(mux) + defer ts.Close() + + h := &HTTPResponse{ + Address: ts.URL + "/redirect", + Body: "{ 'test': 'data'}", + Method: "GET", + ResponseTimeout: 20, + Headers: ` +Content-Type: application/json +`, + FollowRedirects: true, + } + fields, err := h.HTTPGather() + require.NoError(t, err) + assert.NotEmpty(t, fields) + if assert.NotNil(t, fields["http_response_code"]) { + assert.Equal(t, http.StatusOK, fields["http_response_code"]) + } + + h = &HTTPResponse{ + Address: ts.URL + "/badredirect", + Body: "{ 'test': 'data'}", + Method: "GET", + ResponseTimeout: 20, + Headers: ` +Content-Type: application/json +`, + FollowRedirects: true, + } + fields, err = h.HTTPGather() + require.Error(t, err) +} + +func TestMethod(t *testing.T) { + mux := setUpTestMux() + ts := httptest.NewServer(mux) + defer ts.Close() + + h := &HTTPResponse{ + Address: ts.URL + "/mustbepostmethod", + Body: "{ 'test': 'data'}", + Method: "POST", + ResponseTimeout: 20, + Headers: ` +Content-Type: application/json +`, + FollowRedirects: true, + } + fields, err := h.HTTPGather() + require.NoError(t, err) + assert.NotEmpty(t, fields) + if assert.NotNil(t, fields["http_response_code"]) { + assert.Equal(t, http.StatusOK, fields["http_response_code"]) + } + + h = &HTTPResponse{ + Address: ts.URL + "/mustbepostmethod", + Body: "{ 'test': 'data'}", + Method: "GET", + ResponseTimeout: 20, + Headers: ` +Content-Type: application/json +`, + FollowRedirects: true, + } + fields, err = h.HTTPGather() + require.NoError(t, err) + assert.NotEmpty(t, fields) + if assert.NotNil(t, fields["http_response_code"]) { + assert.Equal(t, http.StatusMethodNotAllowed, fields["http_response_code"]) + } + + //check that lowercase methods work correctly + h = &HTTPResponse{ + Address: ts.URL + "/mustbepostmethod", + Body: "{ 'test': 'data'}", + Method: "head", + ResponseTimeout: 20, + Headers: ` +Content-Type: application/json +`, + FollowRedirects: true, + } + fields, err = h.HTTPGather() + require.NoError(t, err) + assert.NotEmpty(t, fields) + if assert.NotNil(t, fields["http_response_code"]) { + assert.Equal(t, http.StatusMethodNotAllowed, fields["http_response_code"]) + } +} + +func TestBody(t *testing.T) { + mux := setUpTestMux() + ts := httptest.NewServer(mux) + defer ts.Close() + + h := &HTTPResponse{ + Address: ts.URL + "/musthaveabody", + Body: "{ 'test': 'data'}", + Method: "GET", + ResponseTimeout: 20, + Headers: ` +Content-Type: application/json +`, + FollowRedirects: true, + } + fields, err := h.HTTPGather() + require.NoError(t, err) + assert.NotEmpty(t, fields) + if assert.NotNil(t, fields["http_response_code"]) { + assert.Equal(t, http.StatusOK, fields["http_response_code"]) + } + + h = &HTTPResponse{ + Address: ts.URL + "/musthaveabody", + Method: "GET", + ResponseTimeout: 20, + Headers: ` +Content-Type: application/json +`, + FollowRedirects: true, + } + fields, err = h.HTTPGather() + require.NoError(t, err) + assert.NotEmpty(t, fields) + if assert.NotNil(t, fields["http_response_code"]) { + assert.Equal(t, http.StatusBadRequest, fields["http_response_code"]) + } +} + +func TestTimeout(t *testing.T) { + mux := setUpTestMux() + ts := httptest.NewServer(mux) + defer ts.Close() + + h := &HTTPResponse{ + Address: ts.URL + "/twosecondnap", + Body: "{ 'test': 'data'}", + Method: "GET", + ResponseTimeout: 1, + Headers: ` +Content-Type: application/json +`, + FollowRedirects: true, + } + _, err := h.HTTPGather() + require.Error(t, err) +} From 377b030d88d78a8046f7d1c155a240e7212d60a2 Mon Sep 17 00:00:00 2001 From: Luke Swithenbank Date: Thu, 7 Apr 2016 11:57:49 +1000 Subject: [PATCH 10/25] update to 5 second default and string map for headers --- plugins/inputs/http_response/README.md | 9 ++- plugins/inputs/http_response/http_response.go | 40 +++++------ .../http_response/http_response_test.go | 72 +++++++++---------- 3 files changed, 54 insertions(+), 67 deletions(-) diff --git a/plugins/inputs/http_response/README.md b/plugins/inputs/http_response/README.md index f2f45b2af..e2bf75b5f 100644 --- a/plugins/inputs/http_response/README.md +++ b/plugins/inputs/http_response/README.md @@ -9,14 +9,13 @@ This input plugin will test HTTP/HTTPS connections. [[inputs.http_response]] ## Server address (default http://localhost) address = "http://github.com" - ## Set response_timeout (default 10 seconds) - response_timeout = 10 + ## Set response_timeout (default 5 seconds) + response_timeout = 5 ## HTTP Request Method method = "GET" ## HTTP Request Headers - headers = ''' - Host: github.com - ''' + [inputs.http_response.headers] + Host = github.com ## Whether to follow redirects from the server (defaults to false) follow_redirects = true ## Optional HTTP Request Body diff --git a/plugins/inputs/http_response/http_response.go b/plugins/inputs/http_response/http_response.go index cee33795a..73533fed4 100644 --- a/plugins/inputs/http_response/http_response.go +++ b/plugins/inputs/http_response/http_response.go @@ -1,11 +1,9 @@ package http_response import ( - "bufio" "errors" "io" "net/http" - "net/textproto" "net/url" "strings" "time" @@ -20,7 +18,7 @@ type HTTPResponse struct { Body string Method string ResponseTimeout int - Headers string + Headers map[string]string FollowRedirects bool } @@ -32,14 +30,13 @@ func (h *HTTPResponse) Description() string { var sampleConfig = ` ## Server address (default http://localhost) address = "http://github.com" - ## Set response_timeout (default 10 seconds) - response_timeout = 10 + ## Set response_timeout (default 5 seconds) + response_timeout = 5 ## HTTP Request Method method = "GET" - ## HTTP Request Headers - headers = ''' - Host: github.com - ''' + ## HTTP Request Headers (all values must be strings) + [inputs.http_response.headers] + # Host = "github.com" ## Whether to follow redirects from the server (defaults to false) follow_redirects = true ## Optional HTTP Request Body @@ -71,17 +68,14 @@ func CreateHttpClient(followRedirects bool, ResponseTimeout time.Duration) *http return client } -// ParseHeaders takes a string of newline seperated http headers and returns a -// http.Header object. An error is returned if the headers cannot be parsed. -func ParseHeaders(headers string) (http.Header, error) { - headers = strings.TrimSpace(headers) + "\n\n" - reader := bufio.NewReader(strings.NewReader(headers)) - tp := textproto.NewReader(reader) - mimeHeader, err := tp.ReadMIMEHeader() - if err != nil { - return nil, err +// CreateHeaders takes a map of header strings and puts them +// into a http.Header Object +func CreateHeaders(headers map[string]string) http.Header { + httpHeaders := make(http.Header) + for key := range headers { + httpHeaders.Add(key, headers[key]) } - return http.Header(mimeHeader), nil + return httpHeaders } // HTTPGather gathers all fields and returns any errors it encounters @@ -99,10 +93,8 @@ func (h *HTTPResponse) HTTPGather() (map[string]interface{}, error) { if err != nil { return nil, err } - request.Header, err = ParseHeaders(h.Headers) - if err != nil { - return nil, err - } + request.Header = CreateHeaders(h.Headers) + // Start Timer start := time.Now() resp, err := client.Do(request) @@ -126,7 +118,7 @@ func (h *HTTPResponse) HTTPGather() (map[string]interface{}, error) { func (h *HTTPResponse) Gather(acc telegraf.Accumulator) error { // Set default values if h.ResponseTimeout < 1 { - h.ResponseTimeout = 10 + h.ResponseTimeout = 5 } // Check send and expected string if h.Method == "" { diff --git a/plugins/inputs/http_response/http_response_test.go b/plugins/inputs/http_response/http_response_test.go index 0f568e3b4..acdfeac75 100644 --- a/plugins/inputs/http_response/http_response_test.go +++ b/plugins/inputs/http_response/http_response_test.go @@ -11,22 +11,18 @@ import ( "time" ) -func TestParseHeaders(t *testing.T) { - fakeHeaders := ` -Accept: text/plain -Content-Type: application/json -Cache-Control: no-cache -` - headers, err := ParseHeaders(fakeHeaders) - require.NoError(t, err) +func TestCreateHeaders(t *testing.T) { + fakeHeaders := map[string]string{ + "Accept": "text/plain", + "Content-Type": "application/json", + "Cache-Control": "no-cache", + } + headers := CreateHeaders(fakeHeaders) testHeaders := make(http.Header) testHeaders.Add("Accept", "text/plain") testHeaders.Add("Content-Type", "application/json") testHeaders.Add("Cache-Control", "no-cache") assert.Equal(t, testHeaders, headers) - - headers, err = ParseHeaders("Accept text/plain") - require.Error(t, err) } func setUpTestMux() http.Handler { @@ -77,9 +73,9 @@ func TestFields(t *testing.T) { Body: "{ 'test': 'data'}", Method: "GET", ResponseTimeout: 20, - Headers: ` -Content-Type: application/json -`, + Headers: map[string]string{ + "Content-Type": "application/json", + }, FollowRedirects: true, } fields, err := h.HTTPGather() @@ -102,9 +98,9 @@ func TestRedirects(t *testing.T) { Body: "{ 'test': 'data'}", Method: "GET", ResponseTimeout: 20, - Headers: ` -Content-Type: application/json -`, + Headers: map[string]string{ + "Content-Type": "application/json", + }, FollowRedirects: true, } fields, err := h.HTTPGather() @@ -119,9 +115,9 @@ Content-Type: application/json Body: "{ 'test': 'data'}", Method: "GET", ResponseTimeout: 20, - Headers: ` -Content-Type: application/json -`, + Headers: map[string]string{ + "Content-Type": "application/json", + }, FollowRedirects: true, } fields, err = h.HTTPGather() @@ -138,9 +134,9 @@ func TestMethod(t *testing.T) { Body: "{ 'test': 'data'}", Method: "POST", ResponseTimeout: 20, - Headers: ` -Content-Type: application/json -`, + Headers: map[string]string{ + "Content-Type": "application/json", + }, FollowRedirects: true, } fields, err := h.HTTPGather() @@ -155,9 +151,9 @@ Content-Type: application/json Body: "{ 'test': 'data'}", Method: "GET", ResponseTimeout: 20, - Headers: ` -Content-Type: application/json -`, + Headers: map[string]string{ + "Content-Type": "application/json", + }, FollowRedirects: true, } fields, err = h.HTTPGather() @@ -173,9 +169,9 @@ Content-Type: application/json Body: "{ 'test': 'data'}", Method: "head", ResponseTimeout: 20, - Headers: ` -Content-Type: application/json -`, + Headers: map[string]string{ + "Content-Type": "application/json", + }, FollowRedirects: true, } fields, err = h.HTTPGather() @@ -196,9 +192,9 @@ func TestBody(t *testing.T) { Body: "{ 'test': 'data'}", Method: "GET", ResponseTimeout: 20, - Headers: ` -Content-Type: application/json -`, + Headers: map[string]string{ + "Content-Type": "application/json", + }, FollowRedirects: true, } fields, err := h.HTTPGather() @@ -212,9 +208,9 @@ Content-Type: application/json Address: ts.URL + "/musthaveabody", Method: "GET", ResponseTimeout: 20, - Headers: ` -Content-Type: application/json -`, + Headers: map[string]string{ + "Content-Type": "application/json", + }, FollowRedirects: true, } fields, err = h.HTTPGather() @@ -235,9 +231,9 @@ func TestTimeout(t *testing.T) { Body: "{ 'test': 'data'}", Method: "GET", ResponseTimeout: 1, - Headers: ` -Content-Type: application/json -`, + Headers: map[string]string{ + "Content-Type": "application/json", + }, FollowRedirects: true, } _, err := h.HTTPGather() From 90185dc6b369981e82ceedc844ce435a000fc094 Mon Sep 17 00:00:00 2001 From: Cameron Sparr Date: Thu, 7 Apr 2016 10:31:06 -0600 Subject: [PATCH 11/25] cleanup & comment http_response def config closes #332 --- CHANGELOG.md | 1 + README.md | 3 ++- etc/telegraf.conf | 19 +++++++++++++++++++ plugins/inputs/http_response/http_response.go | 12 ++++++------ 4 files changed, 28 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 09a00f069..699d0f602 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ - [#976](https://github.com/influxdata/telegraf/pull/976): Reduce allocations in the UDP and statsd inputs. - [#979](https://github.com/influxdata/telegraf/pull/979): Reduce allocations in the TCP listener. - [#935](https://github.com/influxdata/telegraf/pull/935): AWS Cloudwatch input plugin. Thanks @joshhardy & @ljosa! +- [#943](https://github.com/influxdata/telegraf/pull/943): http_response input plugin. Thanks @Lswith! ### Bugfixes - [#968](https://github.com/influxdata/telegraf/issues/968): Processes plugin gets unknown state when spaces are in (command name) diff --git a/README.md b/README.md index caa562a6d..8621238dd 100644 --- a/README.md +++ b/README.md @@ -169,7 +169,8 @@ Currently implemented sources: * [elasticsearch](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/elasticsearch) * [exec](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/exec ) (generic executable plugin, support JSON, influx, graphite and nagios) * [haproxy](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/haproxy) -* [httpjson ](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/httpjson ) (generic JSON-emitting http service plugin) +* [http_response](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/http_response) +* [httpjson](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/httpjson) (generic JSON-emitting http service plugin) * [influxdb](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/influxdb) * [ipmi_sensor](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/ipmi_sensor) * [jolokia](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/jolokia) diff --git a/etc/telegraf.conf b/etc/telegraf.conf index 1b534d888..fa77a3a34 100644 --- a/etc/telegraf.conf +++ b/etc/telegraf.conf @@ -569,6 +569,25 @@ # ## servers = ["socket://run/haproxy/admin.sock"] +# # HTTP/HTTPS request given an address a method and a timeout +# [[inputs.http_response]] +# ## Server address (default http://localhost) +# address = "http://github.com" +# ## Set response_timeout (default 5 seconds) +# response_timeout = 5 +# ## HTTP Request Method +# method = "GET" +# ## Whether to follow redirects from the server (defaults to false) +# follow_redirects = true +# ## HTTP Request Headers (all values must be strings) +# # [inputs.http_response.headers] +# # Host = "github.com" +# ## Optional HTTP Request Body +# # body = ''' +# # {'fake':'data'} +# # ''' + + # # Read flattened metrics from one or more JSON HTTP endpoints # [[inputs.httpjson]] # ## NOTE This plugin only reads numerical measurements, strings and booleans diff --git a/plugins/inputs/http_response/http_response.go b/plugins/inputs/http_response/http_response.go index 73533fed4..69c8fcd06 100644 --- a/plugins/inputs/http_response/http_response.go +++ b/plugins/inputs/http_response/http_response.go @@ -34,15 +34,15 @@ var sampleConfig = ` response_timeout = 5 ## HTTP Request Method method = "GET" - ## HTTP Request Headers (all values must be strings) - [inputs.http_response.headers] - # Host = "github.com" ## Whether to follow redirects from the server (defaults to false) follow_redirects = true + ## HTTP Request Headers (all values must be strings) + # [inputs.http_response.headers] + # Host = "github.com" ## Optional HTTP Request Body - body = ''' - {'fake':'data'} - ''' + # body = ''' + # {'fake':'data'} + # ''' ` // SampleConfig returns the plugin SampleConfig From c4ea122d6679b72e7b0a4e66135438f2dffb4a0e Mon Sep 17 00:00:00 2001 From: Rene Zbinden Date: Tue, 29 Mar 2016 21:38:07 +0200 Subject: [PATCH 12/25] add sysstat plugin --- plugins/inputs/all/all.go | 1 + plugins/inputs/sysstat/README.md | 448 +++++++++++++++++++++++++ plugins/inputs/sysstat/sysstat.go | 315 +++++++++++++++++ plugins/inputs/sysstat/sysstat_test.go | 305 +++++++++++++++++ 4 files changed, 1069 insertions(+) create mode 100644 plugins/inputs/sysstat/README.md create mode 100644 plugins/inputs/sysstat/sysstat.go create mode 100644 plugins/inputs/sysstat/sysstat_test.go diff --git a/plugins/inputs/all/all.go b/plugins/inputs/all/all.go index b28291c24..2d784ca27 100644 --- a/plugins/inputs/all/all.go +++ b/plugins/inputs/all/all.go @@ -52,6 +52,7 @@ import ( _ "github.com/influxdata/telegraf/plugins/inputs/snmp" _ "github.com/influxdata/telegraf/plugins/inputs/sqlserver" _ "github.com/influxdata/telegraf/plugins/inputs/statsd" + _ "github.com/influxdata/telegraf/plugins/inputs/sysstat" _ "github.com/influxdata/telegraf/plugins/inputs/system" _ "github.com/influxdata/telegraf/plugins/inputs/tcp_listener" _ "github.com/influxdata/telegraf/plugins/inputs/trig" diff --git a/plugins/inputs/sysstat/README.md b/plugins/inputs/sysstat/README.md new file mode 100644 index 000000000..7191a83d3 --- /dev/null +++ b/plugins/inputs/sysstat/README.md @@ -0,0 +1,448 @@ +# sysstat Input Plugin + +Collect [sysstat](https://github.com/sysstat/sysstat) metrics - requires the sysstat +package installed. + +This plugin collects system metrics with the sysstat collector utility `sadc` and parses +the created binary data file with the `sadf` utility. + +### Configuration: + +```toml +# Sysstat metrics collector +[[inputs.sysstat]] + ## Collect interval in seconds. This value has to be equal + ## to the telegraf collect interval. + collect_interval = 30 # required + # + # + ## Path to the sadc command. + sadc_path = "/usr/lib/sa/sadc" # required + # + # + ## Path to the sadf command, if it is not in PATH + # sadf_path = "/usr/bin/sadf" + # + # + ## Activities is a list of activities, that are passed as argument to the + ## sadc collector utility (e.g: DISK, SNMP etc...) + ## The more activities that are added, the more data is collected. + # activities = ["DISK"] + # + # + ## Group metrics to measurements. + ## + ## If group is false each metric will be prefixed with a description + ## and represents itself a measurement. + ## + ## If Group is true, corresponding metrics are grouped to a single measurement. + # group = false + # + # + ## Options for the sadf command. The values on the left represent the sadf options and + ## the values on the right their description (wich are used for grouping and prefixing metrics). + [inputs.sysstat.options] + -C = "cpu" + -B = "paging" + -b = "io" + -d = "disk" # requires DISK activity + -H = "hugepages" + "-I ALL" = "interrupts" # requires INT activity + "-n ALL" = "network" + "-P ALL" = "per_cpu" + -q = "queue" + -R = "mem" + "-r ALL" = "mem_util" + -S = "swap_util" + -u = "cpu_util" + -v = "inode" + -W = "swap" + -w = "task" + # + ## Device tags can be used to add additional tags for devices. For example the configuration below + ## adds a tag vg=rootvg for all metrics with sda devices. + # [[inputs.sysstat.device_tags.sda]] + # vg = "rootvg" +``` + +### Measurements & Fields: +#### group=true +- cpu + - pct_idle (float) + - pct_iowait (float) + - pct_nice (float) + - pct_steal (float) + - pct_system (float) + - pct_user (float) + +- disk + - avgqu-sz (float) + - avgrq-sz (float) + - await (float) + - pct_util (float) + - rd_sec_pers (float) + - svctm (float) + - tps (float) + +And much more, depending on the options you configure. + +#### group=false +- cpu_pct_idle + - value (float) +- cpu_pct_iowait + - value (float) +- cpu_pct_nice + - value (float) +- cpu_pct_steal + - value (float) +- cpu_pct_system + - value (float) +- cpu_pct_user + - value (float) +- disk_avgqu-sz + - value (float) +- disk_avgrq-sz + - value (float) +- disk_await + - value (float) +- disk_pct_util + - value (float) +- disk_rd_sec_per_s + - value (float) +- disk_svctm + - value (float) +- disk_tps + - value (float) + +And much more, depending on the options you configure. + +### Tags: + +- All measurements have the following tags: + - device + +And more if you define some `device_tags`. +### Example Output: + +With the configuration below: +```toml +[[inputs.sysstat]] + collect_interval = 30 + sadc_path = "/usr/lib/sa/sadc" # required + activities = ["DISK", "SNMP", "INT"] + group = true + [inputs.sysstat.options] + -C = "cpu" + -B = "paging" + -b = "io" + -d = "disk" # requires DISK activity + -H = "hugepages" + "-I ALL" = "interrupts" # requires INT activity + "-n ALL" = "network" + "-P ALL" = "per_cpu" + -q = "queue" + -R = "mem" + "-r ALL" = "mem_util" + -S = "swap_util" + -u = "cpu_util" + -v = "inode" + -W = "swap" + -w = "task" + [[inputs.sysstat.device_tags.sda]] + vg = "rootvg" +``` + +you get the following output: +``` +$ telegraf -config telegraf.conf -input-filter sysstat -test +* Plugin: sysstat, Collection 1 +> cpu_util,device=all pct_idle=98.85,pct_iowait=0,pct_nice=0.38,pct_steal=0,pct_system=0.64,pct_user=0.13 1459255626657883725 +> swap pswpin_per_s=0,pswpout_per_s=0 1459255626658387650 +> per_cpu,device=cpu1 pct_idle=98.98,pct_iowait=0,pct_nice=0.26,pct_steal=0,pct_system=0.51,pct_user=0.26 1459255626659630437 +> per_cpu,device=all pct_idle=98.85,pct_iowait=0,pct_nice=0.38,pct_steal=0,pct_system=0.64,pct_user=0.13 1459255626659670744 +> per_cpu,device=cpu0 pct_idle=98.73,pct_iowait=0,pct_nice=0.76,pct_steal=0,pct_system=0.51,pct_user=0 1459255626659697515 +> hugepages kbhugfree=0,kbhugused=0,pct_hugused=0 1459255626660057517 +> network,device=lo coll_per_s=0,pct_ifutil=0,rxcmp_per_s=0,rxdrop_per_s=0,rxerr_per_s=0,rxfifo_per_s=0,rxfram_per_s=0,rxkB_per_s=0.81,rxmcst_per_s=0,rxpck_per_s=16,txcarr_per_s=0,txcmp_per_s=0,txdrop_per_s=0,txerr_per_s=0,txfifo_per_s=0,txkB_per_s=0.81,txpck_per_s=16 1459255626661197666 +> network access_per_s=0,active_per_s=0,asmf_per_s=0,asmok_per_s=0,asmrq_per_s=0,atmptf_per_s=0,badcall_per_s=0,call_per_s=0,estres_per_s=0,fragcrt_per_s=0,fragf_per_s=0,fragok_per_s=0,fwddgm_per_s=0,getatt_per_s=0,hit_per_s=0,iadrerr_per_s=0,iadrmk_per_s=0,iadrmkr_per_s=0,idel_per_s=16,idgm_per_s=0,idgmerr_per_s=0,idisc_per_s=0,idstunr_per_s=0,iech_per_s=0,iechr_per_s=0,ierr_per_s=0,ihdrerr_per_s=0,imsg_per_s=0,ip-frag=0,iparmpb_per_s=0,irec_per_s=16,iredir_per_s=0,iseg_per_s=16,isegerr_per_s=0,isrcq_per_s=0,itm_per_s=0,itmex_per_s=0,itmr_per_s=0,iukwnpr_per_s=0,miss_per_s=0,noport_per_s=0,oadrmk_per_s=0,oadrmkr_per_s=0,odgm_per_s=0,odisc_per_s=0,odstunr_per_s=0,oech_per_s=0,oechr_per_s=0,oerr_per_s=0,omsg_per_s=0,onort_per_s=0,oparmpb_per_s=0,oredir_per_s=0,orq_per_s=16,orsts_per_s=0,oseg_per_s=16,osrcq_per_s=0,otm_per_s=0,otmex_per_s=0,otmr_per_s=0,packet_per_s=0,passive_per_s=0,rawsck=0,read_per_s=0,retrans_per_s=0,saccess_per_s=0,scall_per_s=0,sgetatt_per_s=0,sread_per_s=0,swrite_per_s=0,tcp-tw=7,tcp_per_s=0,tcpsck=1543,totsck=4052,udp_per_s=0,udpsck=2,write_per_s=0 1459255626661381788 +> network,device=ens33 coll_per_s=0,pct_ifutil=0,rxcmp_per_s=0,rxdrop_per_s=0,rxerr_per_s=0,rxfifo_per_s=0,rxfram_per_s=0,rxkB_per_s=0,rxmcst_per_s=0,rxpck_per_s=0,txcarr_per_s=0,txcmp_per_s=0,txdrop_per_s=0,txerr_per_s=0,txfifo_per_s=0,txkB_per_s=0,txpck_per_s=0 1459255626661533072 +> disk,device=sda,vg=rootvg avgqu-sz=0.01,avgrq-sz=8.5,await=3.31,pct_util=0.1,rd_sec_per_s=0,svctm=0.25,tps=4,wr_sec_per_s=34 1459255626663974389 +> queue blocked=0,ldavg-1=1.61,ldavg-15=1.34,ldavg-5=1.67,plist-sz=1415,runq-sz=0 1459255626664159054 +> paging fault_per_s=0.25,majflt_per_s=0,pct_vmeff=0,pgfree_per_s=19,pgpgin_per_s=0,pgpgout_per_s=17,pgscand_per_s=0,pgscank_per_s=0,pgsteal_per_s=0 1459255626664304249 +> mem_util kbactive=2206568,kbanonpg=1472208,kbbuffers=118020,kbcached=1035252,kbcommit=8717200,kbdirty=156,kbinact=418912,kbkstack=24672,kbmemfree=1744868,kbmemused=3610272,kbpgtbl=87116,kbslab=233804,kbvmused=0,pct_commit=136.13,pct_memused=67.42 1459255626664554981 +> io bread_per_s=0,bwrtn_per_s=34,rtps=0,tps=4,wtps=4 1459255626664596198 +> inode dentunusd=235039,file-nr=17120,inode-nr=94505,pty-nr=14 1459255626664663693 +> interrupts,device=i000 intr_per_s=0 1459255626664800109 +> interrupts,device=i003 intr_per_s=0 1459255626665255145 +> interrupts,device=i004 intr_per_s=0 1459255626665281776 +> interrupts,device=i006 intr_per_s=0 1459255626665297416 +> interrupts,device=i007 intr_per_s=0 1459255626665321008 +> interrupts,device=i010 intr_per_s=0 1459255626665339413 +> interrupts,device=i012 intr_per_s=0 1459255626665361510 +> interrupts,device=i013 intr_per_s=0 1459255626665381327 +> interrupts,device=i015 intr_per_s=1 1459255626665397313 +> interrupts,device=i001 intr_per_s=0.25 1459255626665412985 +> interrupts,device=i002 intr_per_s=0 1459255626665430475 +> interrupts,device=i005 intr_per_s=0 1459255626665453944 +> interrupts,device=i008 intr_per_s=0 1459255626665470650 +> interrupts,device=i011 intr_per_s=0 1459255626665486069 +> interrupts,device=i009 intr_per_s=0 1459255626665502913 +> interrupts,device=i014 intr_per_s=0 1459255626665518152 +> task cswch_per_s=722.25,proc_per_s=0 1459255626665849646 +> cpu,device=all pct_idle=98.85,pct_iowait=0,pct_nice=0.38,pct_steal=0,pct_system=0.64,pct_user=0.13 1459255626666639715 +> mem bufpg_per_s=0,campg_per_s=1.75,frmpg_per_s=-8.25 1459255626666770205 +> swap_util kbswpcad=0,kbswpfree=1048572,kbswpused=0,pct_swpcad=0,pct_swpused=0 1459255626667313276 +``` + +If you change the group value to false like below: +```toml +[[inputs.sysstat]] + collect_interval = 30 + sadc_path = "/usr/lib/sa/sadc" # required + activities = ["DISK", "SNMP", "INT"] + group = false + [inputs.sysstat.options] + -C = "cpu" + -B = "paging" + -b = "io" + -d = "disk" # requires DISK activity + -H = "hugepages" + "-I ALL" = "interrupts" # requires INT activity + "-n ALL" = "network" + "-P ALL" = "per_cpu" + -q = "queue" + -R = "mem" + "-r ALL" = "mem_util" + -S = "swap_util" + -u = "cpu_util" + -v = "inode" + -W = "swap" + -w = "task" + [[inputs.sysstat.device_tags.sda]] + vg = "rootvg" +``` + +you get the following output: +``` +$ telegraf -config telegraf.conf -input-filter sysstat -test +* Plugin: sysstat, Collection 1 +> io_tps value=0.5 1459255780126025822 +> io_rtps value=0 1459255780126025822 +> io_wtps value=0.5 1459255780126025822 +> io_bread_per_s value=0 1459255780126025822 +> io_bwrtn_per_s value=38 1459255780126025822 +> cpu_util_pct_user,device=all value=39.07 1459255780126025822 +> cpu_util_pct_nice,device=all value=0 1459255780126025822 +> cpu_util_pct_system,device=all value=47.94 1459255780126025822 +> cpu_util_pct_iowait,device=all value=0 1459255780126025822 +> cpu_util_pct_steal,device=all value=0 1459255780126025822 +> cpu_util_pct_idle,device=all value=12.98 1459255780126025822 +> swap_pswpin_per_s value=0 1459255780126025822 +> cpu_pct_user,device=all value=39.07 1459255780126025822 +> cpu_pct_nice,device=all value=0 1459255780126025822 +> cpu_pct_system,device=all value=47.94 1459255780126025822 +> cpu_pct_iowait,device=all value=0 1459255780126025822 +> cpu_pct_steal,device=all value=0 1459255780126025822 +> cpu_pct_idle,device=all value=12.98 1459255780126025822 +> per_cpu_pct_user,device=all value=39.07 1459255780126025822 +> per_cpu_pct_nice,device=all value=0 1459255780126025822 +> per_cpu_pct_system,device=all value=47.94 1459255780126025822 +> per_cpu_pct_iowait,device=all value=0 1459255780126025822 +> per_cpu_pct_steal,device=all value=0 1459255780126025822 +> per_cpu_pct_idle,device=all value=12.98 1459255780126025822 +> per_cpu_pct_user,device=cpu0 value=33.5 1459255780126025822 +> per_cpu_pct_nice,device=cpu0 value=0 1459255780126025822 +> per_cpu_pct_system,device=cpu0 value=65.25 1459255780126025822 +> per_cpu_pct_iowait,device=cpu0 value=0 1459255780126025822 +> per_cpu_pct_steal,device=cpu0 value=0 1459255780126025822 +> per_cpu_pct_idle,device=cpu0 value=1.25 1459255780126025822 +> per_cpu_pct_user,device=cpu1 value=44.85 1459255780126025822 +> per_cpu_pct_nice,device=cpu1 value=0 1459255780126025822 +> per_cpu_pct_system,device=cpu1 value=29.55 1459255780126025822 +> per_cpu_pct_iowait,device=cpu1 value=0 1459255780126025822 +> per_cpu_pct_steal,device=cpu1 value=0 1459255780126025822 +> per_cpu_pct_idle,device=cpu1 value=25.59 1459255780126025822 +> hugepages_kbhugfree value=0 1459255780126025822 +> hugepages_kbhugused value=0 1459255780126025822 +> hugepages_pct_hugused value=0 1459255780126025822 +> interrupts_intr_per_s,device=i000 value=0 1459255780126025822 +> inode_dentunusd value=252876 1459255780126025822 +> mem_util_kbmemfree value=1613612 1459255780126025822 +> disk_tps,device=sda,vg=rootvg value=0.5 1459255780126025822 +> swap_pswpout_per_s value=0 1459255780126025822 +> network_rxpck_per_s,device=ens33 value=0 1459255780126025822 +> queue_runq-sz value=4 1459255780126025822 +> task_proc_per_s value=0 1459255780126025822 +> task_cswch_per_s value=2019 1459255780126025822 +> mem_frmpg_per_s value=0 1459255780126025822 +> mem_bufpg_per_s value=0.5 1459255780126025822 +> mem_campg_per_s value=1.25 1459255780126025822 +> interrupts_intr_per_s,device=i001 value=0 1459255780126025822 +> inode_file-nr value=19104 1459255780126025822 +> mem_util_kbmemused value=3741528 1459255780126025822 +> disk_rd_sec_per_s,device=sda,vg=rootvg value=0 1459255780126025822 +> network_txpck_per_s,device=ens33 value=0 1459255780126025822 +> queue_plist-sz value=1512 1459255780126025822 +> paging_pgpgin_per_s value=0 1459255780126025822 +> paging_pgpgout_per_s value=19 1459255780126025822 +> paging_fault_per_s value=0.25 1459255780126025822 +> paging_majflt_per_s value=0 1459255780126025822 +> paging_pgfree_per_s value=34.25 1459255780126025822 +> paging_pgscank_per_s value=0 1459255780126025822 +> paging_pgscand_per_s value=0 1459255780126025822 +> paging_pgsteal_per_s value=0 1459255780126025822 +> paging_pct_vmeff value=0 1459255780126025822 +> interrupts_intr_per_s,device=i002 value=0 1459255780126025822 +> interrupts_intr_per_s,device=i003 value=0 1459255780126025822 +> interrupts_intr_per_s,device=i004 value=0 1459255780126025822 +> interrupts_intr_per_s,device=i005 value=0 1459255780126025822 +> interrupts_intr_per_s,device=i006 value=0 1459255780126025822 +> interrupts_intr_per_s,device=i007 value=0 1459255780126025822 +> interrupts_intr_per_s,device=i008 value=0 1459255780126025822 +> interrupts_intr_per_s,device=i009 value=0 1459255780126025822 +> interrupts_intr_per_s,device=i010 value=0 1459255780126025822 +> interrupts_intr_per_s,device=i011 value=0 1459255780126025822 +> interrupts_intr_per_s,device=i012 value=0 1459255780126025822 +> interrupts_intr_per_s,device=i013 value=0 1459255780126025822 +> interrupts_intr_per_s,device=i014 value=0 1459255780126025822 +> interrupts_intr_per_s,device=i015 value=1 1459255780126025822 +> inode_inode-nr value=94709 1459255780126025822 +> inode_pty-nr value=14 1459255780126025822 +> mem_util_pct_memused value=69.87 1459255780126025822 +> mem_util_kbbuffers value=118252 1459255780126025822 +> mem_util_kbcached value=1045240 1459255780126025822 +> mem_util_kbcommit value=9628152 1459255780126025822 +> mem_util_pct_commit value=150.35 1459255780126025822 +> mem_util_kbactive value=2303752 1459255780126025822 +> mem_util_kbinact value=428340 1459255780126025822 +> mem_util_kbdirty value=104 1459255780126025822 +> mem_util_kbanonpg value=1568676 1459255780126025822 +> mem_util_kbslab value=240032 1459255780126025822 +> mem_util_kbkstack value=26224 1459255780126025822 +> mem_util_kbpgtbl value=98056 1459255780126025822 +> mem_util_kbvmused value=0 1459255780126025822 +> disk_wr_sec_per_s,device=sda,vg=rootvg value=38 1459255780126025822 +> disk_avgrq-sz,device=sda,vg=rootvg value=76 1459255780126025822 +> disk_avgqu-sz,device=sda,vg=rootvg value=0 1459255780126025822 +> disk_await,device=sda,vg=rootvg value=2 1459255780126025822 +> disk_svctm,device=sda,vg=rootvg value=2 1459255780126025822 +> disk_pct_util,device=sda,vg=rootvg value=0.1 1459255780126025822 +> network_rxkB_per_s,device=ens33 value=0 1459255780126025822 +> network_txkB_per_s,device=ens33 value=0 1459255780126025822 +> network_rxcmp_per_s,device=ens33 value=0 1459255780126025822 +> network_txcmp_per_s,device=ens33 value=0 1459255780126025822 +> network_rxmcst_per_s,device=ens33 value=0 1459255780126025822 +> network_pct_ifutil,device=ens33 value=0 1459255780126025822 +> network_rxpck_per_s,device=lo value=10.75 1459255780126025822 +> network_txpck_per_s,device=lo value=10.75 1459255780126025822 +> network_rxkB_per_s,device=lo value=0.77 1459255780126025822 +> network_txkB_per_s,device=lo value=0.77 1459255780126025822 +> network_rxcmp_per_s,device=lo value=0 1459255780126025822 +> network_txcmp_per_s,device=lo value=0 1459255780126025822 +> network_rxmcst_per_s,device=lo value=0 1459255780126025822 +> network_pct_ifutil,device=lo value=0 1459255780126025822 +> network_rxerr_per_s,device=ens33 value=0 1459255780126025822 +> network_txerr_per_s,device=ens33 value=0 1459255780126025822 +> network_coll_per_s,device=ens33 value=0 1459255780126025822 +> network_rxdrop_per_s,device=ens33 value=0 1459255780126025822 +> network_txdrop_per_s,device=ens33 value=0 1459255780126025822 +> network_txcarr_per_s,device=ens33 value=0 1459255780126025822 +> network_rxfram_per_s,device=ens33 value=0 1459255780126025822 +> network_rxfifo_per_s,device=ens33 value=0 1459255780126025822 +> network_txfifo_per_s,device=ens33 value=0 1459255780126025822 +> network_rxerr_per_s,device=lo value=0 1459255780126025822 +> network_txerr_per_s,device=lo value=0 1459255780126025822 +> network_coll_per_s,device=lo value=0 1459255780126025822 +> network_rxdrop_per_s,device=lo value=0 1459255780126025822 +> network_txdrop_per_s,device=lo value=0 1459255780126025822 +> network_txcarr_per_s,device=lo value=0 1459255780126025822 +> network_rxfram_per_s,device=lo value=0 1459255780126025822 +> network_rxfifo_per_s,device=lo value=0 1459255780126025822 +> network_txfifo_per_s,device=lo value=0 1459255780126025822 +> network_call_per_s value=0 1459255780126025822 +> network_retrans_per_s value=0 1459255780126025822 +> network_read_per_s value=0 1459255780126025822 +> network_write_per_s value=0 1459255780126025822 +> network_access_per_s value=0 1459255780126025822 +> network_getatt_per_s value=0 1459255780126025822 +> network_scall_per_s value=0 1459255780126025822 +> network_badcall_per_s value=0 1459255780126025822 +> network_packet_per_s value=0 1459255780126025822 +> network_udp_per_s value=0 1459255780126025822 +> network_tcp_per_s value=0 1459255780126025822 +> network_hit_per_s value=0 1459255780126025822 +> network_miss_per_s value=0 1459255780126025822 +> network_sread_per_s value=0 1459255780126025822 +> network_swrite_per_s value=0 1459255780126025822 +> network_saccess_per_s value=0 1459255780126025822 +> network_sgetatt_per_s value=0 1459255780126025822 +> network_totsck value=4234 1459255780126025822 +> network_tcpsck value=1637 1459255780126025822 +> network_udpsck value=2 1459255780126025822 +> network_rawsck value=0 1459255780126025822 +> network_ip-frag value=0 1459255780126025822 +> network_tcp-tw value=4 1459255780126025822 +> network_irec_per_s value=10.75 1459255780126025822 +> network_fwddgm_per_s value=0 1459255780126025822 +> network_idel_per_s value=10.75 1459255780126025822 +> network_orq_per_s value=10.75 1459255780126025822 +> network_asmrq_per_s value=0 1459255780126025822 +> network_asmok_per_s value=0 1459255780126025822 +> network_fragok_per_s value=0 1459255780126025822 +> network_fragcrt_per_s value=0 1459255780126025822 +> network_ihdrerr_per_s value=0 1459255780126025822 +> network_iadrerr_per_s value=0 1459255780126025822 +> network_iukwnpr_per_s value=0 1459255780126025822 +> network_idisc_per_s value=0 1459255780126025822 +> network_odisc_per_s value=0 1459255780126025822 +> network_onort_per_s value=0 1459255780126025822 +> network_asmf_per_s value=0 1459255780126025822 +> network_fragf_per_s value=0 1459255780126025822 +> network_imsg_per_s value=0 1459255780126025822 +> network_omsg_per_s value=0 1459255780126025822 +> network_iech_per_s value=0 1459255780126025822 +> network_iechr_per_s value=0 1459255780126025822 +> network_oech_per_s value=0 1459255780126025822 +> network_oechr_per_s value=0 1459255780126025822 +> network_itm_per_s value=0 1459255780126025822 +> network_itmr_per_s value=0 1459255780126025822 +> network_otm_per_s value=0 1459255780126025822 +> network_otmr_per_s value=0 1459255780126025822 +> network_iadrmk_per_s value=0 1459255780126025822 +> network_iadrmkr_per_s value=0 1459255780126025822 +> network_oadrmk_per_s value=0 1459255780126025822 +> network_oadrmkr_per_s value=0 1459255780126025822 +> network_ierr_per_s value=0 1459255780126025822 +> network_oerr_per_s value=0 1459255780126025822 +> network_idstunr_per_s value=0 1459255780126025822 +> network_odstunr_per_s value=0 1459255780126025822 +> network_itmex_per_s value=0 1459255780126025822 +> network_otmex_per_s value=0 1459255780126025822 +> network_iparmpb_per_s value=0 1459255780126025822 +> network_oparmpb_per_s value=0 1459255780126025822 +> network_isrcq_per_s value=0 1459255780126025822 +> network_osrcq_per_s value=0 1459255780126025822 +> network_iredir_per_s value=0 1459255780126025822 +> network_oredir_per_s value=0 1459255780126025822 +> network_active_per_s value=0 1459255780126025822 +> network_passive_per_s value=0 1459255780126025822 +> network_iseg_per_s value=10.75 1459255780126025822 +> network_oseg_per_s value=9.5 1459255780126025822 +> network_atmptf_per_s value=0 1459255780126025822 +> network_estres_per_s value=0 1459255780126025822 +> network_retrans_per_s value=1.5 1459255780126025822 +> network_isegerr_per_s value=0.25 1459255780126025822 +> network_orsts_per_s value=0 1459255780126025822 +> network_idgm_per_s value=0 1459255780126025822 +> network_odgm_per_s value=0 1459255780126025822 +> network_noport_per_s value=0 1459255780126025822 +> network_idgmerr_per_s value=0 1459255780126025822 +> queue_ldavg-1 value=2.1 1459255780126025822 +> queue_ldavg-5 value=1.82 1459255780126025822 +> queue_ldavg-15 value=1.44 1459255780126025822 +> queue_blocked value=0 1459255780126025822 +> swap_util_kbswpfree value=1048572 1459255780126025822 +> swap_util_kbswpused value=0 1459255780126025822 +> swap_util_pct_swpused value=0 1459255780126025822 +> swap_util_kbswpcad value=0 1459255780126025822 +> swap_util_pct_swpcad value=0 1459255780126025822 +``` diff --git a/plugins/inputs/sysstat/sysstat.go b/plugins/inputs/sysstat/sysstat.go new file mode 100644 index 000000000..2b605b3ad --- /dev/null +++ b/plugins/inputs/sysstat/sysstat.go @@ -0,0 +1,315 @@ +// build +linux + +package sysstat + +import ( + "bufio" + "encoding/csv" + "errors" + "fmt" + "io" + "os" + "os/exec" + "path" + "strconv" + "strings" + "sync" + "time" + + "github.com/influxdata/telegraf" + "github.com/influxdata/telegraf/plugins/inputs" +) + +var ( + execCommand = exec.Command // execCommand is used to mock commands in tests. + dfltActivities = []string{"DISK"} +) + +const parseInterval = 1 // parseInterval is the interval (in seconds) where the parsing takes place. + +type Sysstat struct { + // Interval that defines how long data is collected by Sadc cmd. + // + // This value has to be the same as the thelegraf collection interval. + Interval int `toml:"collect_interval"` + + // Sadc represents the path to the sadc collector utility. + Sadc string `toml:"sadc_path"` + + // Sadf represents the path to the sadf cmd. + Sadf string `toml:"sadf_path"` + + // Activities is a list of activities that are passed as argument to the + // collector utility (e.g: DISK, SNMP etc...) + // The more activities that are added, the more data is collected. + Activities []string + + // Options is a map of options. + // + // The key represents the actual option that the Sadf command is called with and + // the value represents the description for that option. + // + // For example, if you have the following options map: + // map[string]string{"-C": "cpu", "-d": "disk"} + // The Sadf command is run with the options -C and -d to extract cpu and + // disk metrics from the collected binary file. + // + // If Group is false (see below), each metric will be prefixed with the corresponding description + // and represents itself a measurement. + // + // If Group is true, metrics are grouped to a single measurement with the corresponding description as name. + Options map[string]string + + // Group determines if metrics are grouped or not. + Group bool + + // DeviceTags adds the possibility to add additional tags for devices. + DeviceTags map[string][]map[string]string `toml:"device_tags"` + tmpFile string +} + +func (*Sysstat) Description() string { + return "Sysstat metrics collector" +} + +var sampleConfig = ` + ## Collect interval in seconds. This value has to be equal + ## to the telegraf collect interval. + collect_interval = 5 # required + # + # + ## Path to the sadc command. + sadc_path = "/usr/lib/sa/sadc" # required + # + # + ## Path to the sadf command, if it is not in PATH + # sadf_path = "/usr/bin/sadf" + # + # + ## Activities is a list of activities, that are passed as argument to the + ## sadc collector utility (e.g: DISK, SNMP etc...) + ## The more activities that are added, the more data is collected. + # activities = ["DISK"] + # + # + ## Group metrics to measurements. + ## + ## If group is false each metric will be prefixed with a description + ## and represents itself a measurement. + ## + ## If Group is true, corresponding metrics are grouped to a single measurement. + # group = false + # + # + ## Options for the sasf command. The values on the left represent the sadf options and + ## the values on the right their description (wich are used for grouping and prefixing metrics). + [inputs.sysstat.options] + -C = "cpu" + -B = "paging" + -b = "io" + -d = "disk" # requires DISK activity + -H = "hugepages" + "-I ALL" = "interrupts" # requires INT activity + "-n ALL" = "network" + "-P ALL" = "per_cpu" + -q = "queue" + -R = "mem" + "-r ALL" = "mem_util" + -S = "swap_util" + -u = "cpu_util" + -v = "inode" + -W = "swap" + -w = "task" + # + # + ## Device tags can be used to add additional tags for devices. For example the configuration below + ## adds a tag vg with value rootvg for all metrics with sda devices. + # [[inputs.sysstat.device_tags.sda]] + # vg = "rootvg" +` + +func (*Sysstat) SampleConfig() string { + return sampleConfig +} + +func (s *Sysstat) Gather(acc telegraf.Accumulator) error { + ts := time.Now().Add(time.Duration(s.Interval) * time.Second) + if err := s.collect(); err != nil { + return err + } + var wg sync.WaitGroup + errorChannel := make(chan error, len(s.Options)*2) + for option := range s.Options { + wg.Add(1) + go func(acc telegraf.Accumulator, option string) { + defer wg.Done() + if err := s.parse(acc, option, ts); err != nil { + errorChannel <- err + } + }(acc, option) + } + wg.Wait() + close(errorChannel) + + errorStrings := []string{} + for err := range errorChannel { + errorStrings = append(errorStrings, err.Error()) + } + + if _, err := os.Stat(s.tmpFile); err == nil { + if err := os.Remove(s.tmpFile); err != nil { + errorStrings = append(errorStrings, err.Error()) + } + } + + if len(errorStrings) == 0 { + return nil + } + return errors.New(strings.Join(errorStrings, "\n")) +} + +// collect collects sysstat data with the collector utility sadc. It runs the following command: +// Sadc -S -S ... 2 tmpFile +// The above command collects system metrics during and saves it in binary form to tmpFile. +func (s *Sysstat) collect() error { + if len(s.Activities) == 0 { + s.Activities = dfltActivities + } + options := []string{} + for _, act := range s.Activities { + options = append(options, "-S", act) + } + s.tmpFile = path.Join("/tmp", fmt.Sprintf("sysstat-%d", time.Now().Unix())) + collectInterval := s.Interval - parseInterval // collectInterval has to be smaller than the telegraf data collection interval + options = append(options, strconv.Itoa(collectInterval), "2", s.tmpFile) + cmd := execCommand(s.Sadc, options...) + out, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("failed to run command %s: %s", strings.Join(cmd.Args, " "), string(out)) + } + return nil +} + +// parse runs Sadf on the previously saved tmpFile: +// Sadf -p -- -p