Merge branch 'master' into bugfix/4335

.circleci/config.yml

@@ -16,11 +16,11 @@ jobs:
     steps:
       - checkout
       - restore_cache:
-          key: vendor-{{ .Branch }}-{{ checksum "Gopkg.lock" }}
+          key: vendor-{{ checksum "Gopkg.lock" }}
       - run: 'make deps'
       - save_cache:
           name: 'vendored deps'
-          key: vendor-{{ .Branch }}-{{ checksum "Gopkg.lock" }}
+          key: vendor-{{ checksum "Gopkg.lock" }}
           paths:
             - './vendor'
       - persist_to_workspace:

CHANGELOG.md

@@ -25,6 +25,7 @@
 - [#4322](https://github.com/influxdata/telegraf/pull/4322): Add log message when tail is added or removed from a file.
 - [#4267](https://github.com/influxdata/telegraf/pull/4267): Add option to use of counter time in win perf counters.
 - [#4343](https://github.com/influxdata/telegraf/pull/4343): Add energy and power field and device id tag to fibaro input.
+- [#4347](https://github.com/influxdata/telegraf/pull/4347): Add http path configuration for OpenTSDB output.
 
 ## v1.7.1 [unreleased]
 
@@ -36,6 +37,7 @@
 - [#4334](https://github.com/influxdata/telegraf/pull/4334): Fix syslog timestamp parsing with single digit day of month.
 - [#2910](https://github.com/influxdata/telegraf/issues/2910): Handle mysql input variations in the user_statistics collecting.
 - [#4293](https://github.com/influxdata/telegraf/issues/4293): Fix minmax and basicstats aggregators to use uint64.
+- [#4290](https://github.com/influxdata/telegraf/issues/4290): Document swap input plugin.
 
 ## v1.7 [2018-06-12]
 

CONTRIBUTING.md

@@ -100,6 +100,13 @@ func init() {
 }
 ```
 
+### Input Plugin Development
+
+* Run `make static` followed by `make plugin-[pluginName]` to spin up a docker dev environment
+using docker-compose.
+* ***[Optional]*** When developing a plugin, add a `dev` directory with a `docker-compose.yml` and `telegraf.conf`
+as well as any other supporting files, where sensible.
+
 ## Adding Typed Metrics
 
 In addition the the `AddFields` function, the accumulator also supports an

Makefile

@@ -92,4 +92,15 @@ docker-image:
 plugins/parsers/influx/machine.go: plugins/parsers/influx/machine.go.rl
 	ragel -Z -G2 $^ -o $@
 
-.PHONY: deps telegraf install test test-windows lint vet test-all package clean docker-image fmtcheck uint64
+static:
+	@echo "Building static linux binary..."
+	@CGO_ENABLED=0 \
+	GOOS=linux \
+	GOARCH=amd64 \
+	go build -ldflags "$(LDFLAGS)" ./cmd/telegraf
+
+plugin-%:
+	@echo "Starting dev environment for $${$(@)} input plugin..."
+	@docker-compose -f plugins/inputs/$${$(@)}/dev/docker-compose.yml up
+
+.PHONY: deps telegraf install test test-windows lint vet test-all package clean docker-image fmtcheck uint64 static

etc/telegraf.conf

@@ -723,6 +723,10 @@
 #   ## Not used with telnet API.
 #   httpBatchSize = 50
 #
+#   ## URI Path for Http requests to OpenTSDB.
+#   ## Used in cases where OpenTSDB is located behind a reverse proxy.
+#   httpPath = "/api/put"
+#
 #   ## Debug true - Prints OpenTSDB communication
 #   debug = false
 #

plugins/inputs/logparser/dev/docker-compose.yml

@@ -0,0 +1,13 @@
+version: '3'
+
+services:
+    telegraf:
+        image: glinton/scratch
+        volumes:
+          - ./telegraf.conf:/telegraf.conf
+          - ../../../../telegraf:/telegraf
+          - ./test.log:/var/log/test.log
+        entrypoint:
+          - /telegraf
+          - --config
+          - /telegraf.conf

plugins/inputs/logparser/dev/telegraf.conf

@@ -0,0 +1,12 @@
+[agent]
+  interval="1s"
+  flush_interval="1s"
+
+[[inputs.logparser]]
+   files = ["/var/log/test.log"]
+   from_beginning = true
+   [inputs.logparser.grok]
+      patterns = [ "%{COMBINED_LOG_FORMAT}", "%{CLIENT:client_ip} %{NOTSPACE:ident} %{NOTSPACE:auth} \\[%{TIMESTAMP_ISO8601:timestamp}\\] \"(?:%{WORD:verb:tag} %{NOTSPACE:request}(?: HTTP/%{NUMBER:http_version:float})?|%{DATA})\" %{NUMBER:resp_code:tag} (?:%{NUMBER:resp_bytes:int}|-) %{QS:referrer} %{QS:agent}"]
+
+[[outputs.file]]
+  files = ["stdout"]

plugins/inputs/logparser/dev/test.log

@@ -0,0 +1,2 @@
+127.0.0.1 ident auth [10/Oct/2000:13:55:36 -0700] "GET /anything HTTP/1.0" 200 2326 "http://localhost:8083/" "Chrome/51.0.2704.84"
+127.0.0.1 ident auth [2018-02-21 13:10:34,555] "GET /peter HTTP/1.0" 200 2326 "http://localhost:8083/" "Chrome/51.0.2704.84"

plugins/inputs/mongodb/dev/docker-compose.yml

@@ -0,0 +1,16 @@
+version: '3'
+services:
+    mongodb:
+        image: mongo
+
+    telegraf:
+        image: glinton/scratch
+        volumes:
+          - ./telegraf.conf:/telegraf.conf
+          - ../../../../telegraf:/telegraf
+        depends_on:
+          - mongodb
+        entrypoint:
+          - /telegraf
+          - --config
+          - /telegraf.conf

plugins/inputs/mongodb/dev/telegraf.conf

@@ -0,0 +1,9 @@
+[agent]
+  interval="1s"
+  flush_interval="3s"
+
+[[inputs.mongodb]]
+  servers = ["mongodb://mongodb:27017"]
+
+[[outputs.file]]
+  files = ["stdout"]

plugins/inputs/mysql/dev/docker-compose.yml

@@ -0,0 +1,42 @@
+version: '3'
+
+services:
+  mysql:
+    image: mysql:5.7
+    restart: always
+    environment:
+      MYSQL_ROOT_PASSWORD: telegraf
+      MYSQL_DATABASE: telegraf
+      MYSQL_USER: telegraf
+      MYSQL_PASSWORD: telegraf
+  maria:
+    image: mariadb
+    restart: always
+    environment:
+      MYSQL_ROOT_PASSWORD: telegraf
+      MYSQL_DATABASE: telegraf
+      MYSQL_USER: telegraf
+      MYSQL_PASSWORD: telegraf
+    command: mysqld --userstat=1
+  percona:
+    image: percona
+    restart: always
+    environment:
+      MYSQL_ROOT_PASSWORD: telegraf
+      MYSQL_DATABASE: telegraf
+      MYSQL_USER: telegraf
+      MYSQL_PASSWORD: telegraf
+
+  telegraf:
+    image: glinton/scratch
+    depends_on:
+      - mysql
+      - maria
+      - percona
+    volumes:
+      - ./telegraf.conf:/telegraf.conf
+      - ../../../../telegraf:/telegraf
+    entrypoint:
+      - /telegraf
+      - --config
+      - /telegraf.conf

plugins/inputs/mysql/dev/telegraf.conf

@@ -0,0 +1,61 @@
+# Uncomment each input as needed to test plugin
+
+## mysql
+#[[inputs.mysql]]
+#  servers = ["root:telegraf@tcp(mysql:3306)/"]
+#  gather_table_schema = true
+#  gather_process_list = true
+#  gather_user_statistics = true
+#  gather_info_schema_auto_inc = true
+#  gather_innodb_metrics = true
+#  gather_slave_status = true
+#  gather_binary_logs = false
+#  gather_table_io_waits = true
+#  gather_table_lock_waits = true
+#  gather_index_io_waits = true
+#  gather_event_waits = true
+#  gather_file_events_stats = true
+#  gather_perf_events_statements = true
+#  interval_slow = "30m"
+#  table_schema_databases = []
+#
+## mariadb
+#[[inputs.mysql]]
+#  servers = ["root:telegraf@tcp(maria:3306)/"]
+#  gather_table_schema = true
+#  gather_process_list = true
+#  gather_user_statistics = true
+#  gather_info_schema_auto_inc = true
+#  gather_innodb_metrics = true
+#  gather_slave_status = true
+#  gather_binary_logs = false
+#  gather_table_io_waits = true
+#  gather_table_lock_waits = true
+#  gather_index_io_waits = true
+#  gather_event_waits = true
+#  gather_file_events_stats = true
+#  gather_perf_events_statements = true
+#  interval_slow = "30m"
+#  table_schema_databases = []
+
+# percona
+[[inputs.mysql]]
+  servers = ["root:telegraf@tcp(percona:3306)/"]
+  gather_table_schema = true
+  gather_process_list = true
+  gather_user_statistics = true
+  gather_info_schema_auto_inc = true
+  gather_innodb_metrics = true
+  gather_slave_status = true
+  gather_binary_logs = false
+  gather_table_io_waits = true
+  gather_table_lock_waits = true
+  gather_index_io_waits = true
+  gather_event_waits = true
+  gather_file_events_stats = true
+  gather_perf_events_statements = true
+  interval_slow = "30m"
+  table_schema_databases = []
+
+[[outputs.file]]
+  files = ["stdout"]

plugins/inputs/system/SWAP_README.md

@@ -0,0 +1,30 @@
+# Swap Input Plugin
+
+The swap plugin collects system swap metrics.
+
+For a more information on what swap memory is, read [All about Linux swap space](https://www.linux.com/news/all-about-linux-swap-space).
+
+### Configuration:
+
+```toml
+# Read metrics about swap memory usage
+[[inputs.swap]]
+  # no configuration
+```
+
+### Metrics:
+
+- swap
+  - fields:
+    - free (int)
+    - total (int)
+    - used (int)
+    - used_percent (float)
+    - in (int)
+    - out (int)
+
+### Example Output:
+
+```
+swap total=20855394304i,used_percent=45.43883523785713,used=9476448256i,free=1715331072i 1511894782000000000
+```

plugins/inputs/system/memory.go

@@ -42,45 +42,9 @@ func (s *MemStats) Gather(acc telegraf.Accumulator) error {
 	return nil
 }
 
-type SwapStats struct {
-	ps PS
-}
-
-func (_ *SwapStats) Description() string {
-	return "Read metrics about swap memory usage"
-}
-
-func (_ *SwapStats) SampleConfig() string { return "" }
-
-func (s *SwapStats) Gather(acc telegraf.Accumulator) error {
-	swap, err := s.ps.SwapStat()
-	if err != nil {
-		return fmt.Errorf("error getting swap memory info: %s", err)
-	}
-
-	fieldsG := map[string]interface{}{
-		"total":        swap.Total,
-		"used":         swap.Used,
-		"free":         swap.Free,
-		"used_percent": swap.UsedPercent,
-	}
-	fieldsC := map[string]interface{}{
-		"in":  swap.Sin,
-		"out": swap.Sout,
-	}
-	acc.AddGauge("swap", fieldsG, nil)
-	acc.AddCounter("swap", fieldsC, nil)
-
-	return nil
-}
-
 func init() {
 	ps := newSystemPS()
 	inputs.Add("mem", func() telegraf.Input {
 		return &MemStats{ps: ps}
 	})
-
-	inputs.Add("swap", func() telegraf.Input {
-		return &SwapStats{ps: ps}
-	})
 }

plugins/inputs/system/memory_test.go

@@ -30,17 +30,6 @@ func TestMemStats(t *testing.T) {
 
 	mps.On("VMStat").Return(vms, nil)
 
-	sms := &mem.SwapMemoryStat{
-		Total:       8123,
-		Used:        1232,
-		Free:        6412,
-		UsedPercent: 12.2,
-		Sin:         7,
-		Sout:        830,
-	}
-
-	mps.On("SwapStat").Return(sms, nil)
-
 	err = (&MemStats{&mps}).Gather(&acc)
 	require.NoError(t, err)
 
@@ -61,15 +50,4 @@ func TestMemStats(t *testing.T) {
 	acc.AssertContainsTaggedFields(t, "mem", memfields, make(map[string]string))
 
 	acc.Metrics = nil
-
-	err = (&SwapStats{&mps}).Gather(&acc)
-	require.NoError(t, err)
-
-	swapfields := map[string]interface{}{
-		"total":        uint64(8123),
-		"used":         uint64(1232),
-		"used_percent": float64(12.2),
-		"free":         uint64(6412),
-	}
-	acc.AssertContainsTaggedFields(t, "swap", swapfields, make(map[string]string))
 }

plugins/inputs/system/swap.go

@@ -0,0 +1,47 @@
+package system
+
+import (
+	"fmt"
+
+	"github.com/influxdata/telegraf"
+	"github.com/influxdata/telegraf/plugins/inputs"
+)
+
+type SwapStats struct {
+	ps PS
+}
+
+func (_ *SwapStats) Description() string {
+	return "Read metrics about swap memory usage"
+}
+
+func (_ *SwapStats) SampleConfig() string { return "" }
+
+func (s *SwapStats) Gather(acc telegraf.Accumulator) error {
+	swap, err := s.ps.SwapStat()
+	if err != nil {
+		return fmt.Errorf("error getting swap memory info: %s", err)
+	}
+
+	fieldsG := map[string]interface{}{
+		"total":        swap.Total,
+		"used":         swap.Used,
+		"free":         swap.Free,
+		"used_percent": swap.UsedPercent,
+	}
+	fieldsC := map[string]interface{}{
+		"in":  swap.Sin,
+		"out": swap.Sout,
+	}
+	acc.AddGauge("swap", fieldsG, nil)
+	acc.AddCounter("swap", fieldsC, nil)
+
+	return nil
+}
+
+func init() {
+	ps := newSystemPS()
+	inputs.Add("swap", func() telegraf.Input {
+		return &SwapStats{ps: ps}
+	})
+}

plugins/inputs/system/swap_test.go

@@ -0,0 +1,38 @@
+package system
+
+import (
+	"testing"
+
+	"github.com/influxdata/telegraf/testutil"
+	"github.com/shirou/gopsutil/mem"
+	"github.com/stretchr/testify/require"
+)
+
+func TestSwapStats(t *testing.T) {
+	var mps MockPS
+	var err error
+	defer mps.AssertExpectations(t)
+	var acc testutil.Accumulator
+
+	sms := &mem.SwapMemoryStat{
+		Total:       8123,
+		Used:        1232,
+		Free:        6412,
+		UsedPercent: 12.2,
+		Sin:         7,
+		Sout:        830,
+	}
+
+	mps.On("SwapStat").Return(sms, nil)
+
+	err = (&SwapStats{&mps}).Gather(&acc)
+	require.NoError(t, err)
+
+	swapfields := map[string]interface{}{
+		"total":        uint64(8123),
+		"used":         uint64(1232),
+		"used_percent": float64(12.2),
+		"free":         uint64(6412),
+	}
+	acc.AssertContainsTaggedFields(t, "swap", swapfields, make(map[string]string))
+}

plugins/outputs/opentsdb/opentsdb.go

@@ -22,6 +22,7 @@ var (
 		`%`, "-",
 		"#", "-",
 		"$", "-")
+	defaultHttpPath  = "/api/put"
 	defaultSeperator = "_"
 )
 
@@ -31,7 +32,8 @@ type OpenTSDB struct {
 	Host string
 	Port int
 
-	HttpBatchSize int
+	HttpBatchSize int // deprecated httpBatchSize form in 1.8
+	HttpPath      string
 
 	Debug bool
 
@@ -52,7 +54,11 @@ var sampleConfig = `
 
   ## Number of data points to send to OpenTSDB in Http requests.
   ## Not used with telnet API.
-  httpBatchSize = 50
+  http_batch_size = 50
+
+  ## URI Path for Http requests to OpenTSDB.
+  ## Used in cases where OpenTSDB is located behind a reverse proxy.
+  http_path = "/api/put"
 
   ## Debug true - Prints OpenTSDB communication
   debug = false
@@ -121,6 +127,7 @@ func (o *OpenTSDB) WriteHttp(metrics []telegraf.Metric, u *url.URL) error {
 		Scheme:    u.Scheme,
 		User:      u.User,
 		BatchSize: o.HttpBatchSize,
+		Path:      o.HttpPath,
 		Debug:     o.Debug,
 	}
 
@@ -260,6 +267,7 @@ func sanitize(value string) string {
 func init() {
 	outputs.Add("opentsdb", func() telegraf.Output {
 		return &OpenTSDB{
+			HttpPath:  defaultHttpPath,
 			Separator: defaultSeperator,
 		}
 	})

plugins/outputs/opentsdb/opentsdb_http.go

@@ -26,6 +26,7 @@ type openTSDBHttp struct {
 	Scheme    string
 	User      *url.Userinfo
 	BatchSize int
+	Path      string
 	Debug     bool
 
 	metricCounter int
@@ -123,7 +124,7 @@ func (o *openTSDBHttp) flush() error {
 		Scheme: o.Scheme,
 		User:   o.User,
 		Host:   fmt.Sprintf("%s:%d", o.Host, o.Port),
-		Path:   "/api/put",
+		Path:   o.Path,
 	}
 
 	if o.Debug {

plugins/outputs/opentsdb/opentsdb_test.go

@@ -156,6 +156,7 @@ func BenchmarkHttpSend(b *testing.B) {
 		Port:          port,
 		Prefix:        "",
 		HttpBatchSize: BatchSize,
+		HttpPath:      "/api/put",
 	}
 
 	b.ResetTimer()