diff --git a/.circleci/config.yml b/.circleci/config.yml index 122f28af0..578e0bcbd 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -16,11 +16,11 @@ jobs: steps: - checkout - restore_cache: - key: vendor-{{ .Branch }}-{{ checksum "Gopkg.lock" }} + key: vendor-{{ checksum "Gopkg.lock" }} - run: 'make deps' - save_cache: name: 'vendored deps' - key: vendor-{{ .Branch }}-{{ checksum "Gopkg.lock" }} + key: vendor-{{ checksum "Gopkg.lock" }} paths: - './vendor' - persist_to_workspace: diff --git a/CHANGELOG.md b/CHANGELOG.md index 6afdf9f8b..4cde04f74 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,7 @@ - [#4322](https://github.com/influxdata/telegraf/pull/4322): Add log message when tail is added or removed from a file. - [#4267](https://github.com/influxdata/telegraf/pull/4267): Add option to use of counter time in win perf counters. - [#4343](https://github.com/influxdata/telegraf/pull/4343): Add energy and power field and device id tag to fibaro input. +- [#4347](https://github.com/influxdata/telegraf/pull/4347): Add http path configuration for OpenTSDB output. ## v1.7.1 [unreleased] @@ -36,6 +37,7 @@ - [#4334](https://github.com/influxdata/telegraf/pull/4334): Fix syslog timestamp parsing with single digit day of month. - [#2910](https://github.com/influxdata/telegraf/issues/2910): Handle mysql input variations in the user_statistics collecting. - [#4293](https://github.com/influxdata/telegraf/issues/4293): Fix minmax and basicstats aggregators to use uint64. +- [#4290](https://github.com/influxdata/telegraf/issues/4290): Document swap input plugin. ## v1.7 [2018-06-12] diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index cef8fa511..9a89e3cbf 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -100,6 +100,13 @@ func init() { } ``` +### Input Plugin Development + +* Run `make static` followed by `make plugin-[pluginName]` to spin up a docker dev environment +using docker-compose. +* ***[Optional]*** When developing a plugin, add a `dev` directory with a `docker-compose.yml` and `telegraf.conf` +as well as any other supporting files, where sensible. + ## Adding Typed Metrics In addition the the `AddFields` function, the accumulator also supports an diff --git a/Makefile b/Makefile index 8650df267..2f3fcecea 100644 --- a/Makefile +++ b/Makefile @@ -92,4 +92,15 @@ docker-image: plugins/parsers/influx/machine.go: plugins/parsers/influx/machine.go.rl ragel -Z -G2 $^ -o $@ -.PHONY: deps telegraf install test test-windows lint vet test-all package clean docker-image fmtcheck uint64 +static: + @echo "Building static linux binary..." + @CGO_ENABLED=0 \ + GOOS=linux \ + GOARCH=amd64 \ + go build -ldflags "$(LDFLAGS)" ./cmd/telegraf + +plugin-%: + @echo "Starting dev environment for $${$(@)} input plugin..." + @docker-compose -f plugins/inputs/$${$(@)}/dev/docker-compose.yml up + +.PHONY: deps telegraf install test test-windows lint vet test-all package clean docker-image fmtcheck uint64 static diff --git a/etc/telegraf.conf b/etc/telegraf.conf index f51e973df..38942adee 100644 --- a/etc/telegraf.conf +++ b/etc/telegraf.conf @@ -723,6 +723,10 @@ # ## Not used with telnet API. # httpBatchSize = 50 # +# ## URI Path for Http requests to OpenTSDB. +# ## Used in cases where OpenTSDB is located behind a reverse proxy. +# httpPath = "/api/put" +# # ## Debug true - Prints OpenTSDB communication # debug = false # diff --git a/plugins/inputs/logparser/dev/docker-compose.yml b/plugins/inputs/logparser/dev/docker-compose.yml new file mode 100644 index 000000000..dfb8e7843 --- /dev/null +++ b/plugins/inputs/logparser/dev/docker-compose.yml @@ -0,0 +1,13 @@ +version: '3' + +services: + telegraf: + image: glinton/scratch + volumes: + - ./telegraf.conf:/telegraf.conf + - ../../../../telegraf:/telegraf + - ./test.log:/var/log/test.log + entrypoint: + - /telegraf + - --config + - /telegraf.conf diff --git a/plugins/inputs/logparser/dev/telegraf.conf b/plugins/inputs/logparser/dev/telegraf.conf new file mode 100644 index 000000000..77e26528c --- /dev/null +++ b/plugins/inputs/logparser/dev/telegraf.conf @@ -0,0 +1,12 @@ +[agent] + interval="1s" + flush_interval="1s" + +[[inputs.logparser]] + files = ["/var/log/test.log"] + from_beginning = true + [inputs.logparser.grok] + patterns = [ "%{COMBINED_LOG_FORMAT}", "%{CLIENT:client_ip} %{NOTSPACE:ident} %{NOTSPACE:auth} \\[%{TIMESTAMP_ISO8601:timestamp}\\] \"(?:%{WORD:verb:tag} %{NOTSPACE:request}(?: HTTP/%{NUMBER:http_version:float})?|%{DATA})\" %{NUMBER:resp_code:tag} (?:%{NUMBER:resp_bytes:int}|-) %{QS:referrer} %{QS:agent}"] + +[[outputs.file]] + files = ["stdout"] diff --git a/plugins/inputs/logparser/dev/test.log b/plugins/inputs/logparser/dev/test.log new file mode 100644 index 000000000..2ff2e4ef6 --- /dev/null +++ b/plugins/inputs/logparser/dev/test.log @@ -0,0 +1,2 @@ +127.0.0.1 ident auth [10/Oct/2000:13:55:36 -0700] "GET /anything HTTP/1.0" 200 2326 "http://localhost:8083/" "Chrome/51.0.2704.84" +127.0.0.1 ident auth [2018-02-21 13:10:34,555] "GET /peter HTTP/1.0" 200 2326 "http://localhost:8083/" "Chrome/51.0.2704.84" diff --git a/plugins/inputs/mongodb/dev/docker-compose.yml b/plugins/inputs/mongodb/dev/docker-compose.yml new file mode 100644 index 000000000..9bf80f244 --- /dev/null +++ b/plugins/inputs/mongodb/dev/docker-compose.yml @@ -0,0 +1,16 @@ +version: '3' +services: + mongodb: + image: mongo + + telegraf: + image: glinton/scratch + volumes: + - ./telegraf.conf:/telegraf.conf + - ../../../../telegraf:/telegraf + depends_on: + - mongodb + entrypoint: + - /telegraf + - --config + - /telegraf.conf diff --git a/plugins/inputs/mongodb/dev/telegraf.conf b/plugins/inputs/mongodb/dev/telegraf.conf new file mode 100644 index 000000000..35077b161 --- /dev/null +++ b/plugins/inputs/mongodb/dev/telegraf.conf @@ -0,0 +1,9 @@ +[agent] + interval="1s" + flush_interval="3s" + +[[inputs.mongodb]] + servers = ["mongodb://mongodb:27017"] + +[[outputs.file]] + files = ["stdout"] diff --git a/plugins/inputs/mysql/dev/docker-compose.yml b/plugins/inputs/mysql/dev/docker-compose.yml new file mode 100644 index 000000000..5f1773fbc --- /dev/null +++ b/plugins/inputs/mysql/dev/docker-compose.yml @@ -0,0 +1,42 @@ +version: '3' + +services: + mysql: + image: mysql:5.7 + restart: always + environment: + MYSQL_ROOT_PASSWORD: telegraf + MYSQL_DATABASE: telegraf + MYSQL_USER: telegraf + MYSQL_PASSWORD: telegraf + maria: + image: mariadb + restart: always + environment: + MYSQL_ROOT_PASSWORD: telegraf + MYSQL_DATABASE: telegraf + MYSQL_USER: telegraf + MYSQL_PASSWORD: telegraf + command: mysqld --userstat=1 + percona: + image: percona + restart: always + environment: + MYSQL_ROOT_PASSWORD: telegraf + MYSQL_DATABASE: telegraf + MYSQL_USER: telegraf + MYSQL_PASSWORD: telegraf + + telegraf: + image: glinton/scratch + depends_on: + - mysql + - maria + - percona + volumes: + - ./telegraf.conf:/telegraf.conf + - ../../../../telegraf:/telegraf + entrypoint: + - /telegraf + - --config + - /telegraf.conf diff --git a/plugins/inputs/mysql/dev/telegraf.conf b/plugins/inputs/mysql/dev/telegraf.conf new file mode 100644 index 000000000..9aa4801ea --- /dev/null +++ b/plugins/inputs/mysql/dev/telegraf.conf @@ -0,0 +1,61 @@ +# Uncomment each input as needed to test plugin + +## mysql +#[[inputs.mysql]] +# servers = ["root:telegraf@tcp(mysql:3306)/"] +# gather_table_schema = true +# gather_process_list = true +# gather_user_statistics = true +# gather_info_schema_auto_inc = true +# gather_innodb_metrics = true +# gather_slave_status = true +# gather_binary_logs = false +# gather_table_io_waits = true +# gather_table_lock_waits = true +# gather_index_io_waits = true +# gather_event_waits = true +# gather_file_events_stats = true +# gather_perf_events_statements = true +# interval_slow = "30m" +# table_schema_databases = [] +# +## mariadb +#[[inputs.mysql]] +# servers = ["root:telegraf@tcp(maria:3306)/"] +# gather_table_schema = true +# gather_process_list = true +# gather_user_statistics = true +# gather_info_schema_auto_inc = true +# gather_innodb_metrics = true +# gather_slave_status = true +# gather_binary_logs = false +# gather_table_io_waits = true +# gather_table_lock_waits = true +# gather_index_io_waits = true +# gather_event_waits = true +# gather_file_events_stats = true +# gather_perf_events_statements = true +# interval_slow = "30m" +# table_schema_databases = [] + +# percona +[[inputs.mysql]] + servers = ["root:telegraf@tcp(percona:3306)/"] + gather_table_schema = true + gather_process_list = true + gather_user_statistics = true + gather_info_schema_auto_inc = true + gather_innodb_metrics = true + gather_slave_status = true + gather_binary_logs = false + gather_table_io_waits = true + gather_table_lock_waits = true + gather_index_io_waits = true + gather_event_waits = true + gather_file_events_stats = true + gather_perf_events_statements = true + interval_slow = "30m" + table_schema_databases = [] + +[[outputs.file]] + files = ["stdout"] diff --git a/plugins/inputs/system/SWAP_README.md b/plugins/inputs/system/SWAP_README.md new file mode 100644 index 000000000..2647b33d0 --- /dev/null +++ b/plugins/inputs/system/SWAP_README.md @@ -0,0 +1,30 @@ +# Swap Input Plugin + +The swap plugin collects system swap metrics. + +For a more information on what swap memory is, read [All about Linux swap space](https://www.linux.com/news/all-about-linux-swap-space). + +### Configuration: + +```toml +# Read metrics about swap memory usage +[[inputs.swap]] + # no configuration +``` + +### Metrics: + +- swap + - fields: + - free (int) + - total (int) + - used (int) + - used_percent (float) + - in (int) + - out (int) + +### Example Output: + +``` +swap total=20855394304i,used_percent=45.43883523785713,used=9476448256i,free=1715331072i 1511894782000000000 +``` diff --git a/plugins/inputs/system/memory.go b/plugins/inputs/system/memory.go index 3cb95610e..b44fabc49 100644 --- a/plugins/inputs/system/memory.go +++ b/plugins/inputs/system/memory.go @@ -42,45 +42,9 @@ func (s *MemStats) Gather(acc telegraf.Accumulator) error { return nil } -type SwapStats struct { - ps PS -} - -func (_ *SwapStats) Description() string { - return "Read metrics about swap memory usage" -} - -func (_ *SwapStats) SampleConfig() string { return "" } - -func (s *SwapStats) Gather(acc telegraf.Accumulator) error { - swap, err := s.ps.SwapStat() - if err != nil { - return fmt.Errorf("error getting swap memory info: %s", err) - } - - fieldsG := map[string]interface{}{ - "total": swap.Total, - "used": swap.Used, - "free": swap.Free, - "used_percent": swap.UsedPercent, - } - fieldsC := map[string]interface{}{ - "in": swap.Sin, - "out": swap.Sout, - } - acc.AddGauge("swap", fieldsG, nil) - acc.AddCounter("swap", fieldsC, nil) - - return nil -} - func init() { ps := newSystemPS() inputs.Add("mem", func() telegraf.Input { return &MemStats{ps: ps} }) - - inputs.Add("swap", func() telegraf.Input { - return &SwapStats{ps: ps} - }) } diff --git a/plugins/inputs/system/memory_test.go b/plugins/inputs/system/memory_test.go index 5d5860a8e..34914db9c 100644 --- a/plugins/inputs/system/memory_test.go +++ b/plugins/inputs/system/memory_test.go @@ -30,17 +30,6 @@ func TestMemStats(t *testing.T) { mps.On("VMStat").Return(vms, nil) - sms := &mem.SwapMemoryStat{ - Total: 8123, - Used: 1232, - Free: 6412, - UsedPercent: 12.2, - Sin: 7, - Sout: 830, - } - - mps.On("SwapStat").Return(sms, nil) - err = (&MemStats{&mps}).Gather(&acc) require.NoError(t, err) @@ -61,15 +50,4 @@ func TestMemStats(t *testing.T) { acc.AssertContainsTaggedFields(t, "mem", memfields, make(map[string]string)) acc.Metrics = nil - - err = (&SwapStats{&mps}).Gather(&acc) - require.NoError(t, err) - - swapfields := map[string]interface{}{ - "total": uint64(8123), - "used": uint64(1232), - "used_percent": float64(12.2), - "free": uint64(6412), - } - acc.AssertContainsTaggedFields(t, "swap", swapfields, make(map[string]string)) } diff --git a/plugins/inputs/system/swap.go b/plugins/inputs/system/swap.go new file mode 100644 index 000000000..f1f7c8e23 --- /dev/null +++ b/plugins/inputs/system/swap.go @@ -0,0 +1,47 @@ +package system + +import ( + "fmt" + + "github.com/influxdata/telegraf" + "github.com/influxdata/telegraf/plugins/inputs" +) + +type SwapStats struct { + ps PS +} + +func (_ *SwapStats) Description() string { + return "Read metrics about swap memory usage" +} + +func (_ *SwapStats) SampleConfig() string { return "" } + +func (s *SwapStats) Gather(acc telegraf.Accumulator) error { + swap, err := s.ps.SwapStat() + if err != nil { + return fmt.Errorf("error getting swap memory info: %s", err) + } + + fieldsG := map[string]interface{}{ + "total": swap.Total, + "used": swap.Used, + "free": swap.Free, + "used_percent": swap.UsedPercent, + } + fieldsC := map[string]interface{}{ + "in": swap.Sin, + "out": swap.Sout, + } + acc.AddGauge("swap", fieldsG, nil) + acc.AddCounter("swap", fieldsC, nil) + + return nil +} + +func init() { + ps := newSystemPS() + inputs.Add("swap", func() telegraf.Input { + return &SwapStats{ps: ps} + }) +} diff --git a/plugins/inputs/system/swap_test.go b/plugins/inputs/system/swap_test.go new file mode 100644 index 000000000..ec9a0fe54 --- /dev/null +++ b/plugins/inputs/system/swap_test.go @@ -0,0 +1,38 @@ +package system + +import ( + "testing" + + "github.com/influxdata/telegraf/testutil" + "github.com/shirou/gopsutil/mem" + "github.com/stretchr/testify/require" +) + +func TestSwapStats(t *testing.T) { + var mps MockPS + var err error + defer mps.AssertExpectations(t) + var acc testutil.Accumulator + + sms := &mem.SwapMemoryStat{ + Total: 8123, + Used: 1232, + Free: 6412, + UsedPercent: 12.2, + Sin: 7, + Sout: 830, + } + + mps.On("SwapStat").Return(sms, nil) + + err = (&SwapStats{&mps}).Gather(&acc) + require.NoError(t, err) + + swapfields := map[string]interface{}{ + "total": uint64(8123), + "used": uint64(1232), + "used_percent": float64(12.2), + "free": uint64(6412), + } + acc.AssertContainsTaggedFields(t, "swap", swapfields, make(map[string]string)) +} diff --git a/plugins/outputs/opentsdb/opentsdb.go b/plugins/outputs/opentsdb/opentsdb.go index c38ad353b..964b1768f 100644 --- a/plugins/outputs/opentsdb/opentsdb.go +++ b/plugins/outputs/opentsdb/opentsdb.go @@ -22,6 +22,7 @@ var ( `%`, "-", "#", "-", "$", "-") + defaultHttpPath = "/api/put" defaultSeperator = "_" ) @@ -31,7 +32,8 @@ type OpenTSDB struct { Host string Port int - HttpBatchSize int + HttpBatchSize int // deprecated httpBatchSize form in 1.8 + HttpPath string Debug bool @@ -52,7 +54,11 @@ var sampleConfig = ` ## Number of data points to send to OpenTSDB in Http requests. ## Not used with telnet API. - httpBatchSize = 50 + http_batch_size = 50 + + ## URI Path for Http requests to OpenTSDB. + ## Used in cases where OpenTSDB is located behind a reverse proxy. + http_path = "/api/put" ## Debug true - Prints OpenTSDB communication debug = false @@ -121,6 +127,7 @@ func (o *OpenTSDB) WriteHttp(metrics []telegraf.Metric, u *url.URL) error { Scheme: u.Scheme, User: u.User, BatchSize: o.HttpBatchSize, + Path: o.HttpPath, Debug: o.Debug, } @@ -260,6 +267,7 @@ func sanitize(value string) string { func init() { outputs.Add("opentsdb", func() telegraf.Output { return &OpenTSDB{ + HttpPath: defaultHttpPath, Separator: defaultSeperator, } }) diff --git a/plugins/outputs/opentsdb/opentsdb_http.go b/plugins/outputs/opentsdb/opentsdb_http.go index e74e74f03..4f971abb6 100644 --- a/plugins/outputs/opentsdb/opentsdb_http.go +++ b/plugins/outputs/opentsdb/opentsdb_http.go @@ -26,6 +26,7 @@ type openTSDBHttp struct { Scheme string User *url.Userinfo BatchSize int + Path string Debug bool metricCounter int @@ -123,7 +124,7 @@ func (o *openTSDBHttp) flush() error { Scheme: o.Scheme, User: o.User, Host: fmt.Sprintf("%s:%d", o.Host, o.Port), - Path: "/api/put", + Path: o.Path, } if o.Debug { diff --git a/plugins/outputs/opentsdb/opentsdb_test.go b/plugins/outputs/opentsdb/opentsdb_test.go index d5d7aa7e9..096337c5c 100644 --- a/plugins/outputs/opentsdb/opentsdb_test.go +++ b/plugins/outputs/opentsdb/opentsdb_test.go @@ -156,6 +156,7 @@ func BenchmarkHttpSend(b *testing.B) { Port: port, Prefix: "", HttpBatchSize: BatchSize, + HttpPath: "/api/put", } b.ResetTimer()