From 9a59512f75af8c4830ce9d14888ed1a2662f3953 Mon Sep 17 00:00:00 2001 From: codehate Date: Fri, 5 Feb 2016 17:24:08 +0530 Subject: [PATCH 01/17] Add: Telegraf CouchDB Plugin CouchDB Plugin - Formatted Code closes #652 Minor fix for CouchDB Plugin Formatted code fix for CouchDB Plugin CouchDB Plugin - Changed hosts to full urls CouchDB Plugin - Formatted Code CouchDB Plugin - Fatal commit from local fix CouchDB Plugin - Updated test case --- CHANGELOG.md | 1 + README.md | 1 + plugins/inputs/all/all.go | 1 + plugins/inputs/couchdb/README.md | 255 ++++++++++++++++++++ plugins/inputs/couchdb/couchdb.go | 205 ++++++++++++++++ plugins/inputs/couchdb/couchdb_test.go | 320 +++++++++++++++++++++++++ 6 files changed, 783 insertions(+) create mode 100644 plugins/inputs/couchdb/README.md create mode 100644 plugins/inputs/couchdb/couchdb.go create mode 100644 plugins/inputs/couchdb/couchdb_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index baabc1b53..234af5903 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ### Release Notes ### Features +- [#652](https://github.com/influxdata/telegraf/pull/652): CouchDB Input Plugin ### Bugfixes diff --git a/README.md b/README.md index 160a8fa62..b079df888 100644 --- a/README.md +++ b/README.md @@ -155,6 +155,7 @@ Currently implemented sources: * aerospike * apache * bcache +* couchdb * disque * docker * elasticsearch diff --git a/plugins/inputs/all/all.go b/plugins/inputs/all/all.go index 52ab428f8..f263c3f68 100644 --- a/plugins/inputs/all/all.go +++ b/plugins/inputs/all/all.go @@ -4,6 +4,7 @@ import ( _ "github.com/influxdata/telegraf/plugins/inputs/aerospike" _ "github.com/influxdata/telegraf/plugins/inputs/apache" _ "github.com/influxdata/telegraf/plugins/inputs/bcache" + _ "github.com/influxdata/telegraf/plugins/inputs/couchdb" _ "github.com/influxdata/telegraf/plugins/inputs/disque" _ "github.com/influxdata/telegraf/plugins/inputs/docker" _ "github.com/influxdata/telegraf/plugins/inputs/elasticsearch" diff --git a/plugins/inputs/couchdb/README.md b/plugins/inputs/couchdb/README.md new file mode 100644 index 000000000..313b57011 --- /dev/null +++ b/plugins/inputs/couchdb/README.md @@ -0,0 +1,255 @@ +# CouchDB Input Plugin +--- + +The CouchDB plugin gathers metrics of CouchDB using [_stats](http://docs.couchdb.org/en/1.6.1/api/server/common.html?highlight=stats#get--_stats) endpoint. + +### Configuration: + +``` +# Sample Config: +[[inputs.couchdb]] + hosts = ["http://localhost:5984/_stats"] +``` + +### Measurements & Fields: + +Statistics specific to the internals of CouchDB: + +- couchdb_auth_cache_misses +- couchdb_database_writes +- couchdb_open_databases +- couchdb_auth_cache_hits +- couchdb_request_time +- couchdb_database_reads +- couchdb_open_os_files + +Statistics of HTTP requests by method: + +- httpd_request_methods_put +- httpd_request_methods_get +- httpd_request_methods_copy +- httpd_request_methods_delete +- httpd_request_methods_post +- httpd_request_methods_head + +Statistics of HTTP requests by response code: + +- httpd_status_codes_200 +- httpd_status_codes_201 +- httpd_status_codes_202 +- httpd_status_codes_301 +- httpd_status_codes_304 +- httpd_status_codes_400 +- httpd_status_codes_401 +- httpd_status_codes_403 +- httpd_status_codes_404 +- httpd_status_codes_405 +- httpd_status_codes_409 +- httpd_status_codes_412 +- httpd_status_codes_500 + +httpd statistics: + +- httpd_clients_requesting_changes +- httpd_temporary_view_reads +- httpd_requests +- httpd_bulk_requests +- httpd_view_reads + +### Tags: + +- server (url of the couchdb _stats endpoint) + +### Example output: + +``` +➜ telegraf git:(master) ✗ ./telegraf -config ./config.conf -input-filter couchdb -test +* Plugin: couchdb, + Collection 1 +> couchdb,server=http://localhost:5984/_stats couchdb_auth_cache_hits_current=0, +couchdb_auth_cache_hits_max=0, +couchdb_auth_cache_hits_mean=0, +couchdb_auth_cache_hits_min=0, +couchdb_auth_cache_hits_stddev=0, +couchdb_auth_cache_hits_sum=0, +couchdb_auth_cache_misses_current=0, +couchdb_auth_cache_misses_max=0, +couchdb_auth_cache_misses_mean=0, +couchdb_auth_cache_misses_min=0, +couchdb_auth_cache_misses_stddev=0, +couchdb_auth_cache_misses_sum=0, +couchdb_database_reads_current=0, +couchdb_database_reads_max=0, +couchdb_database_reads_mean=0, +couchdb_database_reads_min=0, +couchdb_database_reads_stddev=0, +couchdb_database_reads_sum=0, +couchdb_database_writes_current=1102, +couchdb_database_writes_max=131, +couchdb_database_writes_mean=0.116, +couchdb_database_writes_min=0, +couchdb_database_writes_stddev=3.536, +couchdb_database_writes_sum=1102, +couchdb_open_databases_current=1, +couchdb_open_databases_max=1, +couchdb_open_databases_mean=0, +couchdb_open_databases_min=0, +couchdb_open_databases_stddev=0.01, +couchdb_open_databases_sum=1, +couchdb_open_os_files_current=2, +couchdb_open_os_files_max=2, +couchdb_open_os_files_mean=0, +couchdb_open_os_files_min=0, +couchdb_open_os_files_stddev=0.02, +couchdb_open_os_files_sum=2, +couchdb_request_time_current=242.21, +couchdb_request_time_max=102, +couchdb_request_time_mean=5.767, +couchdb_request_time_min=1, +couchdb_request_time_stddev=17.369, +couchdb_request_time_sum=242.21, +httpd_bulk_requests_current=0, +httpd_bulk_requests_max=0, +httpd_bulk_requests_mean=0, +httpd_bulk_requests_min=0, +httpd_bulk_requests_stddev=0, +httpd_bulk_requests_sum=0, +httpd_clients_requesting_changes_current=0, +httpd_clients_requesting_changes_max=0, +httpd_clients_requesting_changes_mean=0, +httpd_clients_requesting_changes_min=0, +httpd_clients_requesting_changes_stddev=0, +httpd_clients_requesting_changes_sum=0, +httpd_request_methods_copy_current=0, +httpd_request_methods_copy_max=0, +httpd_request_methods_copy_mean=0, +httpd_request_methods_copy_min=0, +httpd_request_methods_copy_stddev=0, +httpd_request_methods_copy_sum=0, +httpd_request_methods_delete_current=0, +httpd_request_methods_delete_max=0, +httpd_request_methods_delete_mean=0, +httpd_request_methods_delete_min=0, +httpd_request_methods_delete_stddev=0, +httpd_request_methods_delete_sum=0, +httpd_request_methods_get_current=31, +httpd_request_methods_get_max=1, +httpd_request_methods_get_mean=0.003, +httpd_request_methods_get_min=0, +httpd_request_methods_get_stddev=0.057, +httpd_request_methods_get_sum=31, +httpd_request_methods_head_current=0, +httpd_request_methods_head_max=0, +httpd_request_methods_head_mean=0, +httpd_request_methods_head_min=0, +httpd_request_methods_head_stddev=0, +httpd_request_methods_head_sum=0, +httpd_request_methods_post_current=1102, +httpd_request_methods_post_max=131, +httpd_request_methods_post_mean=0.116, +httpd_request_methods_post_min=0, +httpd_request_methods_post_stddev=3.536, +httpd_request_methods_post_sum=1102, +httpd_request_methods_put_current=1, +httpd_request_methods_put_max=1, +httpd_request_methods_put_mean=0, +httpd_request_methods_put_min=0, +httpd_request_methods_put_stddev=0.01, +httpd_request_methods_put_sum=1, +httpd_requests_current=1133, +httpd_requests_max=130, +httpd_requests_mean=0.118, +httpd_requests_min=0, +httpd_requests_stddev=3.512, +httpd_requests_sum=1133, +httpd_status_codes_200_current=31, +httpd_status_codes_200_max=1, +httpd_status_codes_200_mean=0.003, +httpd_status_codes_200_min=0, +httpd_status_codes_200_stddev=0.057, +httpd_status_codes_200_sum=31, +httpd_status_codes_201_current=1103, +httpd_status_codes_201_max=130, +httpd_status_codes_201_mean=0.116, +httpd_status_codes_201_min=0, +httpd_status_codes_201_stddev=3.532, +httpd_status_codes_201_sum=1103, +httpd_status_codes_202_current=0, +httpd_status_codes_202_max=0, +httpd_status_codes_202_mean=0, +httpd_status_codes_202_min=0, +httpd_status_codes_202_stddev=0, +httpd_status_codes_202_sum=0, +httpd_status_codes_301_current=0, +httpd_status_codes_301_max=0, +httpd_status_codes_301_mean=0, +httpd_status_codes_301_min=0, +httpd_status_codes_301_stddev=0, +httpd_status_codes_301_sum=0, +httpd_status_codes_304_current=0, +httpd_status_codes_304_max=0, +httpd_status_codes_304_mean=0, +httpd_status_codes_304_min=0, +httpd_status_codes_304_stddev=0, +httpd_status_codes_304_sum=0, +httpd_status_codes_400_current=0, +httpd_status_codes_400_max=0, +httpd_status_codes_400_mean=0, +httpd_status_codes_400_min=0, +httpd_status_codes_400_stddev=0, +httpd_status_codes_400_sum=0, +httpd_status_codes_401_current=0, +httpd_status_codes_401_max=0, +httpd_status_codes_401_mean=0, +httpd_status_codes_401_min=0, +httpd_status_codes_401_stddev=0, +httpd_status_codes_401_sum=0, +httpd_status_codes_403_current=0, +httpd_status_codes_403_max=0, +httpd_status_codes_403_mean=0, +httpd_status_codes_403_min=0, +httpd_status_codes_403_stddev=0, +httpd_status_codes_403_sum=0, +httpd_status_codes_404_current=0, +httpd_status_codes_404_max=0, +httpd_status_codes_404_mean=0, +httpd_status_codes_404_min=0, +httpd_status_codes_404_stddev=0, +httpd_status_codes_404_sum=0, +httpd_status_codes_405_current=0, +httpd_status_codes_405_max=0, +httpd_status_codes_405_mean=0, +httpd_status_codes_405_min=0, +httpd_status_codes_405_stddev=0, +httpd_status_codes_405_sum=0, +httpd_status_codes_409_current=0, +httpd_status_codes_409_max=0, +httpd_status_codes_409_mean=0, +httpd_status_codes_409_min=0, +httpd_status_codes_409_stddev=0, +httpd_status_codes_409_sum=0, +httpd_status_codes_412_current=0, +httpd_status_codes_412_max=0, +httpd_status_codes_412_mean=0, +httpd_status_codes_412_min=0, +httpd_status_codes_412_stddev=0, +httpd_status_codes_412_sum=0, +httpd_status_codes_500_current=0, +httpd_status_codes_500_max=0, +httpd_status_codes_500_mean=0, +httpd_status_codes_500_min=0, +httpd_status_codes_500_stddev=0, +httpd_status_codes_500_sum=0, +httpd_temporary_view_reads_current=0, +httpd_temporary_view_reads_max=0, +httpd_temporary_view_reads_mean=0, +httpd_temporary_view_reads_min=0, +httpd_temporary_view_reads_stddev=0, +httpd_temporary_view_reads_sum=0, +httpd_view_reads_current=0, +httpd_view_reads_max=0, +httpd_view_reads_mean=0, +httpd_view_reads_min=0, +httpd_view_reads_stddev=0, +httpd_view_reads_sum=0 1454692257621938169 +``` diff --git a/plugins/inputs/couchdb/couchdb.go b/plugins/inputs/couchdb/couchdb.go new file mode 100644 index 000000000..65b4bff4c --- /dev/null +++ b/plugins/inputs/couchdb/couchdb.go @@ -0,0 +1,205 @@ +package couchdb + +import ( + "encoding/json" + "errors" + "fmt" + "github.com/influxdata/telegraf" + "github.com/influxdata/telegraf/plugins/inputs" + "net/http" + "reflect" + "strings" + "sync" +) + +// Schema: +type metaData struct { + Description string `json:"description"` + Current float64 `json:"current"` + Sum float64 `json:"sum"` + Mean float64 `json:"mean"` + Stddev float64 `json:"stddev"` + Min float64 `json:"min"` + Max float64 `json:"max"` +} + +type Stats struct { + Couchdb struct { + AuthCacheMisses metaData `json:"auth_cache_misses"` + DatabaseWrites metaData `json:"database_writes"` + OpenDatabases metaData `json:"open_databases"` + AuthCacheHits metaData `json:"auth_cache_hits"` + RequestTime metaData `json:"request_time"` + DatabaseReads metaData `json:"database_reads"` + OpenOsFiles metaData `json:"open_os_files"` + } `json:"couchdb"` + HttpdRequestMethods struct { + Put metaData `json:"PUT"` + Get metaData `json:"GET"` + Copy metaData `json:"COPY"` + Delete metaData `json:"DELETE"` + Post metaData `json:"POST"` + Head metaData `json:"HEAD"` + } `json:"httpd_request_methods"` + HttpdStatusCodes struct { + Status200 metaData `json:"200"` + Status201 metaData `json:"201"` + Status202 metaData `json:"202"` + Status301 metaData `json:"301"` + Status304 metaData `json:"304"` + Status400 metaData `json:"400"` + Status401 metaData `json:"401"` + Status403 metaData `json:"403"` + Status404 metaData `json:"404"` + Status405 metaData `json:"405"` + Status409 metaData `json:"409"` + Status412 metaData `json:"412"` + Status500 metaData `json:"500"` + } `json:"httpd_status_codes"` + Httpd struct { + ClientsRequestingChanges metaData `json:"clients_requesting_changes"` + TemporaryViewReads metaData `json:"temporary_view_reads"` + Requests metaData `json:"requests"` + BulkRequests metaData `json:"bulk_requests"` + ViewReads metaData `json:"view_reads"` + } `json:"httpd"` +} + +type CouchDB struct { + HOSTs []string `toml:"hosts"` +} + +func (*CouchDB) Description() string { + return "Read CouchDB Stats from one or more servers" +} + +func (*CouchDB) SampleConfig() string { + return ` + # Works with CouchDB stats endpoints out of the box + # Multiple HOSTs from which to read CouchDB stats: + hosts = ["http://localhost:8086/_stats"] + ` +} + +func (c *CouchDB) Gather(accumulator telegraf.Accumulator) error { + errorChannel := make(chan error, len(c.HOSTs)) + var wg sync.WaitGroup + for _, u := range c.HOSTs { + wg.Add(1) + go func(host string) { + defer wg.Done() + if err := c.fetchAndInsertData(accumulator, host); err != nil { + errorChannel <- fmt.Errorf("[host=%s]: %s", host, err) + } + }(u) + } + + wg.Wait() + close(errorChannel) + + // If there weren't any errors, we can return nil now. + if len(errorChannel) == 0 { + return nil + } + + // There were errors, so join them all together as one big error. + errorStrings := make([]string, 0, len(errorChannel)) + for err := range errorChannel { + errorStrings = append(errorStrings, err.Error()) + } + + return errors.New(strings.Join(errorStrings, "\n")) + +} + +func (c *CouchDB) fetchAndInsertData(accumulator telegraf.Accumulator, host string) error { + + response, error := http.Get(host) + if error != nil { + return error + } + defer response.Body.Close() + + var stats Stats + decoder := json.NewDecoder(response.Body) + decoder.Decode(&stats) + + fields := map[string]interface{}{} + + // CouchDB meta stats: + c.MapCopy(fields, c.generateFields("couchdb_auth_cache_misses", stats.Couchdb.AuthCacheMisses)) + c.MapCopy(fields, c.generateFields("couchdb_database_writes", stats.Couchdb.DatabaseWrites)) + c.MapCopy(fields, c.generateFields("couchdb_open_databases", stats.Couchdb.OpenDatabases)) + c.MapCopy(fields, c.generateFields("couchdb_auth_cache_hits", stats.Couchdb.AuthCacheHits)) + c.MapCopy(fields, c.generateFields("couchdb_request_time", stats.Couchdb.RequestTime)) + c.MapCopy(fields, c.generateFields("couchdb_database_reads", stats.Couchdb.DatabaseReads)) + c.MapCopy(fields, c.generateFields("couchdb_open_os_files", stats.Couchdb.OpenOsFiles)) + + // http request methods stats: + c.MapCopy(fields, c.generateFields("httpd_request_methods_put", stats.HttpdRequestMethods.Put)) + c.MapCopy(fields, c.generateFields("httpd_request_methods_get", stats.HttpdRequestMethods.Get)) + c.MapCopy(fields, c.generateFields("httpd_request_methods_copy", stats.HttpdRequestMethods.Copy)) + c.MapCopy(fields, c.generateFields("httpd_request_methods_delete", stats.HttpdRequestMethods.Delete)) + c.MapCopy(fields, c.generateFields("httpd_request_methods_post", stats.HttpdRequestMethods.Post)) + c.MapCopy(fields, c.generateFields("httpd_request_methods_head", stats.HttpdRequestMethods.Head)) + + // status code stats: + c.MapCopy(fields, c.generateFields("httpd_status_codes_200", stats.HttpdStatusCodes.Status200)) + c.MapCopy(fields, c.generateFields("httpd_status_codes_201", stats.HttpdStatusCodes.Status201)) + c.MapCopy(fields, c.generateFields("httpd_status_codes_202", stats.HttpdStatusCodes.Status202)) + c.MapCopy(fields, c.generateFields("httpd_status_codes_301", stats.HttpdStatusCodes.Status301)) + c.MapCopy(fields, c.generateFields("httpd_status_codes_304", stats.HttpdStatusCodes.Status304)) + c.MapCopy(fields, c.generateFields("httpd_status_codes_400", stats.HttpdStatusCodes.Status400)) + c.MapCopy(fields, c.generateFields("httpd_status_codes_401", stats.HttpdStatusCodes.Status401)) + c.MapCopy(fields, c.generateFields("httpd_status_codes_403", stats.HttpdStatusCodes.Status403)) + c.MapCopy(fields, c.generateFields("httpd_status_codes_404", stats.HttpdStatusCodes.Status404)) + c.MapCopy(fields, c.generateFields("httpd_status_codes_405", stats.HttpdStatusCodes.Status405)) + c.MapCopy(fields, c.generateFields("httpd_status_codes_409", stats.HttpdStatusCodes.Status409)) + c.MapCopy(fields, c.generateFields("httpd_status_codes_412", stats.HttpdStatusCodes.Status412)) + c.MapCopy(fields, c.generateFields("httpd_status_codes_500", stats.HttpdStatusCodes.Status500)) + + // httpd stats: + c.MapCopy(fields, c.generateFields("httpd_clients_requesting_changes", stats.Httpd.ClientsRequestingChanges)) + c.MapCopy(fields, c.generateFields("httpd_temporary_view_reads", stats.Httpd.TemporaryViewReads)) + c.MapCopy(fields, c.generateFields("httpd_requests", stats.Httpd.Requests)) + c.MapCopy(fields, c.generateFields("httpd_bulk_requests", stats.Httpd.BulkRequests)) + c.MapCopy(fields, c.generateFields("httpd_view_reads", stats.Httpd.ViewReads)) + + tags := map[string]string{ + "server": host, + } + accumulator.AddFields("couchdb", fields, tags) + return nil +} + +func (*CouchDB) MapCopy(dst, src interface{}) { + dv, sv := reflect.ValueOf(dst), reflect.ValueOf(src) + for _, k := range sv.MapKeys() { + dv.SetMapIndex(k, sv.MapIndex(k)) + } +} + +func (*CouchDB) safeCheck(value interface{}) interface{} { + if value == nil { + return 0.0 + } + return value +} + +func (c *CouchDB) generateFields(prefix string, obj metaData) map[string]interface{} { + fields := map[string]interface{}{ + prefix + "_current": c.safeCheck(obj.Current), + prefix + "_sum": c.safeCheck(obj.Sum), + prefix + "_mean": c.safeCheck(obj.Mean), + prefix + "_stddev": c.safeCheck(obj.Stddev), + prefix + "_min": c.safeCheck(obj.Min), + prefix + "_max": c.safeCheck(obj.Max), + } + return fields +} + +func init() { + inputs.Add("couchdb", func() telegraf.Input { + return &CouchDB{} + }) +} diff --git a/plugins/inputs/couchdb/couchdb_test.go b/plugins/inputs/couchdb/couchdb_test.go new file mode 100644 index 000000000..7b824e748 --- /dev/null +++ b/plugins/inputs/couchdb/couchdb_test.go @@ -0,0 +1,320 @@ +package couchdb_test + +import ( + "github.com/influxdata/telegraf/plugins/inputs/couchdb" + "github.com/influxdata/telegraf/testutil" + "github.com/stretchr/testify/require" + "net/http" + "net/http/httptest" + "testing" +) + +func TestBasic(t *testing.T) { + js := ` +{ + "couchdb": { + "auth_cache_misses": { + "description": "number of authentication cache misses", + "current": null, + "sum": null, + "mean": null, + "stddev": null, + "min": null, + "max": null + }, + "database_writes": { + "description": "number of times a database was changed", + "current": null, + "sum": null, + "mean": null, + "stddev": null, + "min": null, + "max": null + }, + "open_databases": { + "description": "number of open databases", + "current": null, + "sum": null, + "mean": null, + "stddev": null, + "min": null, + "max": null + }, + "auth_cache_hits": { + "description": "number of authentication cache hits", + "current": null, + "sum": null, + "mean": null, + "stddev": null, + "min": null, + "max": null + }, + "request_time": { + "description": "length of a request inside CouchDB without MochiWeb", + "current": 18.0, + "sum": 18.0, + "mean": 18.0, + "stddev": null, + "min": 18.0, + "max": 18.0 + }, + "database_reads": { + "description": "number of times a document was read from a database", + "current": null, + "sum": null, + "mean": null, + "stddev": null, + "min": null, + "max": null + }, + "open_os_files": { + "description": "number of file descriptors CouchDB has open", + "current": null, + "sum": null, + "mean": null, + "stddev": null, + "min": null, + "max": null + } + }, + "httpd_request_methods": { + "PUT": { + "description": "number of HTTP PUT requests", + "current": null, + "sum": null, + "mean": null, + "stddev": null, + "min": null, + "max": null + }, + "GET": { + "description": "number of HTTP GET requests", + "current": 2.0, + "sum": 2.0, + "mean": 0.25, + "stddev": 0.70699999999999996181, + "min": 0, + "max": 2 + }, + "COPY": { + "description": "number of HTTP COPY requests", + "current": null, + "sum": null, + "mean": null, + "stddev": null, + "min": null, + "max": null + }, + "DELETE": { + "description": "number of HTTP DELETE requests", + "current": null, + "sum": null, + "mean": null, + "stddev": null, + "min": null, + "max": null + }, + "POST": { + "description": "number of HTTP POST requests", + "current": null, + "sum": null, + "mean": null, + "stddev": null, + "min": null, + "max": null + }, + "HEAD": { + "description": "number of HTTP HEAD requests", + "current": null, + "sum": null, + "mean": null, + "stddev": null, + "min": null, + "max": null + } + }, + "httpd_status_codes": { + "403": { + "description": "number of HTTP 403 Forbidden responses", + "current": null, + "sum": null, + "mean": null, + "stddev": null, + "min": null, + "max": null + }, + "202": { + "description": "number of HTTP 202 Accepted responses", + "current": null, + "sum": null, + "mean": null, + "stddev": null, + "min": null, + "max": null + }, + "401": { + "description": "number of HTTP 401 Unauthorized responses", + "current": null, + "sum": null, + "mean": null, + "stddev": null, + "min": null, + "max": null + }, + "409": { + "description": "number of HTTP 409 Conflict responses", + "current": null, + "sum": null, + "mean": null, + "stddev": null, + "min": null, + "max": null + }, + "200": { + "description": "number of HTTP 200 OK responses", + "current": 1.0, + "sum": 1.0, + "mean": 0.125, + "stddev": 0.35399999999999998135, + "min": 0, + "max": 1 + }, + "405": { + "description": "number of HTTP 405 Method Not Allowed responses", + "current": null, + "sum": null, + "mean": null, + "stddev": null, + "min": null, + "max": null + }, + "400": { + "description": "number of HTTP 400 Bad Request responses", + "current": null, + "sum": null, + "mean": null, + "stddev": null, + "min": null, + "max": null + }, + "201": { + "description": "number of HTTP 201 Created responses", + "current": null, + "sum": null, + "mean": null, + "stddev": null, + "min": null, + "max": null + }, + "404": { + "description": "number of HTTP 404 Not Found responses", + "current": null, + "sum": null, + "mean": null, + "stddev": null, + "min": null, + "max": null + }, + "500": { + "description": "number of HTTP 500 Internal Server Error responses", + "current": null, + "sum": null, + "mean": null, + "stddev": null, + "min": null, + "max": null + }, + "412": { + "description": "number of HTTP 412 Precondition Failed responses", + "current": null, + "sum": null, + "mean": null, + "stddev": null, + "min": null, + "max": null + }, + "301": { + "description": "number of HTTP 301 Moved Permanently responses", + "current": null, + "sum": null, + "mean": null, + "stddev": null, + "min": null, + "max": null + }, + "304": { + "description": "number of HTTP 304 Not Modified responses", + "current": null, + "sum": null, + "mean": null, + "stddev": null, + "min": null, + "max": null + } + }, + "httpd": { + "clients_requesting_changes": { + "description": "number of clients for continuous _changes", + "current": null, + "sum": null, + "mean": null, + "stddev": null, + "min": null, + "max": null + }, + "temporary_view_reads": { + "description": "number of temporary view reads", + "current": null, + "sum": null, + "mean": null, + "stddev": null, + "min": null, + "max": null + }, + "requests": { + "description": "number of HTTP requests", + "current": 2.0, + "sum": 2.0, + "mean": 0.25, + "stddev": 0.70699999999999996181, + "min": 0, + "max": 2 + }, + "bulk_requests": { + "description": "number of bulk requests", + "current": null, + "sum": null, + "mean": null, + "stddev": null, + "min": null, + "max": null + }, + "view_reads": { + "description": "number of view reads", + "current": null, + "sum": null, + "mean": null, + "stddev": null, + "min": null, + "max": null + } + } +} + +` + fakeServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/_stats" { + _, _ = w.Write([]byte(js)) + } else { + w.WriteHeader(http.StatusNotFound) + } + })) + defer fakeServer.Close() + + plugin := &couchdb.CouchDB{ + HOSTs: []string{fakeServer.URL + "/_stats"}, + } + + var acc testutil.Accumulator + require.NoError(t, plugin.Gather(&acc)) +} From 1e03a9440b51ad865ac48f6c89e4c9bc8c740dc0 Mon Sep 17 00:00:00 2001 From: Cameron Sparr Date: Mon, 1 Feb 2016 18:32:01 -0700 Subject: [PATCH 02/17] Try ping plugin with -n and -s options added --- plugins/inputs/ping/ping.go | 2 +- plugins/inputs/ping/ping_test.go | 8 ++++---- scripts/circle-test.sh | 2 ++ 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/plugins/inputs/ping/ping.go b/plugins/inputs/ping/ping.go index b415c6d4d..3368a25e0 100644 --- a/plugins/inputs/ping/ping.go +++ b/plugins/inputs/ping/ping.go @@ -128,7 +128,7 @@ func hostPinger(args ...string) (string, error) { // args returns the arguments for the 'ping' executable func (p *Ping) args(url string) []string { // Build the ping command args based on toml config - args := []string{"-c", strconv.Itoa(p.Count)} + args := []string{"-c", strconv.Itoa(p.Count), "-n", "-s", "16"} if p.PingInterval > 0 { args = append(args, "-i", strconv.FormatFloat(p.PingInterval, 'f', 1, 64)) } diff --git a/plugins/inputs/ping/ping_test.go b/plugins/inputs/ping/ping_test.go index 64da15b51..6663207a0 100644 --- a/plugins/inputs/ping/ping_test.go +++ b/plugins/inputs/ping/ping_test.go @@ -76,7 +76,7 @@ func TestArgs(t *testing.T) { // Actual and Expected arg lists must be sorted for reflect.DeepEqual actual := p.args("www.google.com") - expected := []string{"-c", "2", "www.google.com"} + expected := []string{"-c", "2", "-n", "-s", "16", "www.google.com"} sort.Strings(actual) sort.Strings(expected) assert.True(t, reflect.DeepEqual(expected, actual), @@ -84,7 +84,7 @@ func TestArgs(t *testing.T) { p.Interface = "eth0" actual = p.args("www.google.com") - expected = []string{"-c", "2", "-I", "eth0", "www.google.com"} + expected = []string{"-c", "2", "-n", "-s", "16", "-I", "eth0", "www.google.com"} sort.Strings(actual) sort.Strings(expected) assert.True(t, reflect.DeepEqual(expected, actual), @@ -92,7 +92,7 @@ func TestArgs(t *testing.T) { p.Timeout = 12.0 actual = p.args("www.google.com") - expected = []string{"-c", "2", "-I", "eth0", "-t", "12.0", "www.google.com"} + expected = []string{"-c", "2", "-n", "-s", "16", "-I", "eth0", "-t", "12.0", "www.google.com"} sort.Strings(actual) sort.Strings(expected) assert.True(t, reflect.DeepEqual(expected, actual), @@ -100,7 +100,7 @@ func TestArgs(t *testing.T) { p.PingInterval = 1.2 actual = p.args("www.google.com") - expected = []string{"-c", "2", "-I", "eth0", "-t", "12.0", "-i", "1.2", + expected = []string{"-c", "2", "-n", "-s", "16", "-I", "eth0", "-t", "12.0", "-i", "1.2", "www.google.com"} sort.Strings(actual) sort.Strings(expected) diff --git a/scripts/circle-test.sh b/scripts/circle-test.sh index bbad51506..d4f150c83 100755 --- a/scripts/circle-test.sh +++ b/scripts/circle-test.sh @@ -68,4 +68,6 @@ tmpdir=$(mktemp -d) exit_if_fail ./telegraf -config $tmpdir/config.toml \ -test -input-filter cpu:mem +mv ./telegraf $CIRCLE_ARTIFACTS + exit $rc From f5f43e6d1bf50ddb3f105f615ef224a85a5d8f50 Mon Sep 17 00:00:00 2001 From: Cameron Sparr Date: Tue, 2 Feb 2016 18:43:03 -0700 Subject: [PATCH 03/17] ping plugin: use -W for linux, -t for bsd/darwin closes #443 --- plugins/inputs/ping/ping.go | 11 ++++++++++- plugins/inputs/ping/ping_test.go | 24 ++++++++++++++++++++---- 2 files changed, 30 insertions(+), 5 deletions(-) diff --git a/plugins/inputs/ping/ping.go b/plugins/inputs/ping/ping.go index 3368a25e0..9ca6b3498 100644 --- a/plugins/inputs/ping/ping.go +++ b/plugins/inputs/ping/ping.go @@ -5,6 +5,7 @@ package ping import ( "errors" "os/exec" + "runtime" "strconv" "strings" "sync" @@ -133,7 +134,15 @@ func (p *Ping) args(url string) []string { args = append(args, "-i", strconv.FormatFloat(p.PingInterval, 'f', 1, 64)) } if p.Timeout > 0 { - args = append(args, "-t", strconv.FormatFloat(p.Timeout, 'f', 1, 64)) + switch runtime.GOOS { + case "darwin", "freebsd": + args = append(args, "-t", strconv.FormatFloat(p.Timeout, 'f', 1, 64)) + case "linux": + args = append(args, "-W", strconv.FormatFloat(p.Timeout, 'f', 1, 64)) + default: + // Not sure the best option here, just assume GNU ping? + args = append(args, "-W", strconv.FormatFloat(p.Timeout, 'f', 1, 64)) + } } if p.Interface != "" { args = append(args, "-I", p.Interface) diff --git a/plugins/inputs/ping/ping_test.go b/plugins/inputs/ping/ping_test.go index 6663207a0..cd61a4fb2 100644 --- a/plugins/inputs/ping/ping_test.go +++ b/plugins/inputs/ping/ping_test.go @@ -5,6 +5,7 @@ package ping import ( "errors" "reflect" + "runtime" "sort" "testing" @@ -84,7 +85,8 @@ func TestArgs(t *testing.T) { p.Interface = "eth0" actual = p.args("www.google.com") - expected = []string{"-c", "2", "-n", "-s", "16", "-I", "eth0", "www.google.com"} + expected = []string{"-c", "2", "-n", "-s", "16", "-I", "eth0", + "www.google.com"} sort.Strings(actual) sort.Strings(expected) assert.True(t, reflect.DeepEqual(expected, actual), @@ -92,7 +94,15 @@ func TestArgs(t *testing.T) { p.Timeout = 12.0 actual = p.args("www.google.com") - expected = []string{"-c", "2", "-n", "-s", "16", "-I", "eth0", "-t", "12.0", "www.google.com"} + switch runtime.GOOS { + case "darwin", "freebsd": + expected = []string{"-c", "2", "-n", "-s", "16", "-I", "eth0", "-t", + "12.0", "www.google.com"} + default: + expected = []string{"-c", "2", "-n", "-s", "16", "-I", "eth0", "-W", + "12.0", "www.google.com"} + } + sort.Strings(actual) sort.Strings(expected) assert.True(t, reflect.DeepEqual(expected, actual), @@ -100,8 +110,14 @@ func TestArgs(t *testing.T) { p.PingInterval = 1.2 actual = p.args("www.google.com") - expected = []string{"-c", "2", "-n", "-s", "16", "-I", "eth0", "-t", "12.0", "-i", "1.2", - "www.google.com"} + switch runtime.GOOS { + case "darwin", "freebsd": + expected = []string{"-c", "2", "-n", "-s", "16", "-I", "eth0", "-t", + "12.0", "-i", "1.2", "www.google.com"} + default: + expected = []string{"-c", "2", "-n", "-s", "16", "-I", "eth0", "-W", + "12.0", "-i", "1.2", "www.google.com"} + } sort.Strings(actual) sort.Strings(expected) assert.True(t, reflect.DeepEqual(expected, actual), From b55a93a3e13e010b97e2ab8aa7c366ef705e5114 Mon Sep 17 00:00:00 2001 From: Cameron Sparr Date: Sun, 7 Feb 2016 09:06:51 -0700 Subject: [PATCH 04/17] update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 234af5903..e3a034fab 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ - [#652](https://github.com/influxdata/telegraf/pull/652): CouchDB Input Plugin ### Bugfixes +- [#443](https://github.com/influxdata/telegraf/issues/443): Fix Ping command timeout parameter on Linux. ## v0.10.2 [2016-02-04] From 6b06a231027aa488c5df803eee5faee68da57d20 Mon Sep 17 00:00:00 2001 From: Cameron Sparr Date: Mon, 8 Feb 2016 15:56:43 -0700 Subject: [PATCH 05/17] Change [tags] to [global_tags] to deal with toml bug closes #662 --- CHANGELOG.md | 1 + CONFIGURATION.md | 6 +++--- etc/telegraf.conf | 2 +- internal/config/config.go | 6 +++--- internal/config/testdata/telegraf-agent.toml | 2 +- 5 files changed, 9 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e3a034fab..c92be7b36 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ ### Bugfixes - [#443](https://github.com/influxdata/telegraf/issues/443): Fix Ping command timeout parameter on Linux. +- [#662](https://github.com/influxdata/telegraf/pull/667): Change `[tags]` to `[global_tags]` to fix multiple-plugin tags bug. ## v0.10.2 [2016-02-04] diff --git a/CONFIGURATION.md b/CONFIGURATION.md index 421fe4f72..f4214b5d4 100644 --- a/CONFIGURATION.md +++ b/CONFIGURATION.md @@ -9,9 +9,9 @@ To generate a file with specific inputs and outputs, you can use the -input-filter and -output-filter flags: `telegraf -sample-config -input-filter cpu:mem:net:swap -output-filter influxdb:kafka` -## `[tags]` Configuration +## `[global_tags]` Configuration -Global tags can be specific in the `[tags]` section of the config file in +Global tags can be specific in the `[global_tags]` section of the config file in key="value" format. All metrics being gathered on this host will be tagged with the tags specified here. @@ -76,7 +76,7 @@ measurements at a 10s interval and will collect per-cpu data, dropping any fields which begin with `time_`. ```toml -[tags] +[global_tags] dc = "denver-1" [agent] diff --git a/etc/telegraf.conf b/etc/telegraf.conf index b5b028559..b62e50263 100644 --- a/etc/telegraf.conf +++ b/etc/telegraf.conf @@ -10,7 +10,7 @@ # file would generate. # Global tags can be specified here in key="value" format. -[tags] +[global_tags] # dc = "us-east-1" # will tag all metrics with dc=us-east-1 # rack = "1a" diff --git a/internal/config/config.go b/internal/config/config.go index 9b35cd407..68d36388e 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -138,7 +138,7 @@ var header = `# Telegraf configuration # file would generate. # Global tags can be specified here in key="value" format. -[tags] +[global_tags] # dc = "us-east-1" # will tag all metrics with dc=us-east-1 # rack = "1a" @@ -333,9 +333,9 @@ func (c *Config) LoadConfig(path string) error { log.Printf("Could not parse [agent] config\n") return err } - case "tags": + case "global_tags", "tags": if err = config.UnmarshalTable(subTable, c.Tags); err != nil { - log.Printf("Could not parse [tags] config\n") + log.Printf("Could not parse [global_tags] config\n") return err } case "outputs": diff --git a/internal/config/testdata/telegraf-agent.toml b/internal/config/testdata/telegraf-agent.toml index 5ede47016..b2ffa0cf0 100644 --- a/internal/config/testdata/telegraf-agent.toml +++ b/internal/config/testdata/telegraf-agent.toml @@ -20,7 +20,7 @@ # with 'required'. Be sure to edit those to make this configuration work. # Tags can also be specified via a normal map, but only one form at a time: -[tags] +[global_tags] dc = "us-east-1" # Configuration for telegraf agent From 1449c8b88791125c1a6df1381f7fe97762dc1a4c Mon Sep 17 00:00:00 2001 From: Henry Hu Date: Mon, 1 Feb 2016 11:43:38 +0800 Subject: [PATCH 06/17] Add Graphite line protocol parsing to exec plugin closes #637 --- Godeps | 2 +- Godeps_windows | 2 +- README.md | 2 +- internal/encoding/encoder.go | 31 ++ internal/encoding/graphite/config.go | 135 +++++++++ internal/encoding/graphite/errors.go | 14 + internal/encoding/graphite/parser.go | 414 +++++++++++++++++++++++++++ internal/encoding/influx/parser.go | 48 ++++ internal/encoding/json/parser.go | 68 +++++ plugins/inputs/exec/README.md | 117 +++++++- plugins/inputs/exec/exec.go | 158 +++++++--- plugins/inputs/exec/exec_test.go | 20 +- 12 files changed, 951 insertions(+), 60 deletions(-) create mode 100644 internal/encoding/encoder.go create mode 100644 internal/encoding/graphite/config.go create mode 100644 internal/encoding/graphite/errors.go create mode 100644 internal/encoding/graphite/parser.go create mode 100644 internal/encoding/influx/parser.go create mode 100644 internal/encoding/json/parser.go diff --git a/Godeps b/Godeps index 3393a1cee..7e43ed610 100644 --- a/Godeps +++ b/Godeps @@ -56,4 +56,4 @@ golang.org/x/text 6d3c22c4525a4da167968fa2479be5524d2e8bd0 gopkg.in/dancannon/gorethink.v1 6f088135ff288deb9d5546f4c71919207f891a70 gopkg.in/fatih/pool.v2 cba550ebf9bce999a02e963296d4bc7a486cb715 gopkg.in/mgo.v2 03c9f3ee4c14c8e51ee521a6a7d0425658dd6f64 -gopkg.in/yaml.v2 f7716cbe52baa25d2e9b0d0da546fcf909fc16b4 +gopkg.in/yaml.v2 f7716cbe52baa25d2e9b0d0da546fcf909fc16b4 \ No newline at end of file diff --git a/Godeps_windows b/Godeps_windows index 8f147ed87..829e2cb35 100644 --- a/Godeps_windows +++ b/Godeps_windows @@ -60,4 +60,4 @@ golang.org/x/text 6fc2e00a0d64b1f7fc1212dae5b0c939cf6d9ac4 gopkg.in/dancannon/gorethink.v1 6f088135ff288deb9d5546f4c71919207f891a70 gopkg.in/fatih/pool.v2 cba550ebf9bce999a02e963296d4bc7a486cb715 gopkg.in/mgo.v2 03c9f3ee4c14c8e51ee521a6a7d0425658dd6f64 -gopkg.in/yaml.v2 f7716cbe52baa25d2e9b0d0da546fcf909fc16b4 +gopkg.in/yaml.v2 f7716cbe52baa25d2e9b0d0da546fcf909fc16b4 \ No newline at end of file diff --git a/README.md b/README.md index b079df888..c5db3c6e2 100644 --- a/README.md +++ b/README.md @@ -159,7 +159,7 @@ Currently implemented sources: * disque * docker * elasticsearch -* exec (generic JSON-emitting executable plugin) +* exec (generic executable plugin, support JSON, influx and graphite) * haproxy * httpjson (generic JSON-emitting http service plugin) * influxdb diff --git a/internal/encoding/encoder.go b/internal/encoding/encoder.go new file mode 100644 index 000000000..129906ce5 --- /dev/null +++ b/internal/encoding/encoder.go @@ -0,0 +1,31 @@ +package encoding + +import ( + "fmt" + + "github.com/influxdata/telegraf" +) + +type Parser interface { + InitConfig(configs map[string]interface{}) error + Parse(buf []byte) ([]telegraf.Metric, error) + ParseLine(line string) (telegraf.Metric, error) +} + +type Creator func() Parser + +var Parsers = map[string]Creator{} + +func Add(name string, creator Creator) { + Parsers[name] = creator +} + +func NewParser(dataFormat string, configs map[string]interface{}) (parser Parser, err error) { + creator := Parsers[dataFormat] + if creator == nil { + return nil, fmt.Errorf("Unsupported data format: %s. ", dataFormat) + } + parser = creator() + err = parser.InitConfig(configs) + return parser, err +} diff --git a/internal/encoding/graphite/config.go b/internal/encoding/graphite/config.go new file mode 100644 index 000000000..7a5c759e7 --- /dev/null +++ b/internal/encoding/graphite/config.go @@ -0,0 +1,135 @@ +package graphite + +import ( + "fmt" + "strings" +) + +const ( + // DefaultSeparator is the default join character to use when joining multiple + // measurment parts in a template. + DefaultSeparator = "." +) + +// Config represents the configuration for Graphite endpoints. +type Config struct { + Separator string + Templates []string +} + +// Validate validates the config's templates and tags. +func (c *Config) Validate() error { + if err := c.validateTemplates(); err != nil { + return err + } + + return nil +} + +func (c *Config) validateTemplates() error { + // map to keep track of filters we see + filters := map[string]struct{}{} + + for i, t := range c.Templates { + parts := strings.Fields(t) + // Ensure template string is non-empty + if len(parts) == 0 { + return fmt.Errorf("missing template at position: %d", i) + } + if len(parts) == 1 && parts[0] == "" { + return fmt.Errorf("missing template at position: %d", i) + } + + if len(parts) > 3 { + return fmt.Errorf("invalid template format: '%s'", t) + } + + template := t + filter := "" + tags := "" + if len(parts) >= 2 { + // We could have