From 9fe90d71f445cab609c266a8e99de0da8f6e5f9c Mon Sep 17 00:00:00 2001 From: Ayrdrie Date: Mon, 2 Jul 2018 17:34:40 -0600 Subject: [PATCH] Add plugin development framework (#4324) --- CONTRIBUTING.md | 7 +++ Makefile | 13 +++- .../inputs/logparser/dev/docker-compose.yml | 13 ++++ plugins/inputs/logparser/dev/telegraf.conf | 12 ++++ plugins/inputs/logparser/dev/test.log | 2 + plugins/inputs/mongodb/dev/docker-compose.yml | 16 +++++ plugins/inputs/mongodb/dev/telegraf.conf | 9 +++ plugins/inputs/mysql/dev/docker-compose.yml | 42 +++++++++++++ plugins/inputs/mysql/dev/telegraf.conf | 61 +++++++++++++++++++ 9 files changed, 174 insertions(+), 1 deletion(-) create mode 100644 plugins/inputs/logparser/dev/docker-compose.yml create mode 100644 plugins/inputs/logparser/dev/telegraf.conf create mode 100644 plugins/inputs/logparser/dev/test.log create mode 100644 plugins/inputs/mongodb/dev/docker-compose.yml create mode 100644 plugins/inputs/mongodb/dev/telegraf.conf create mode 100644 plugins/inputs/mysql/dev/docker-compose.yml create mode 100644 plugins/inputs/mysql/dev/telegraf.conf diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index cef8fa511..9a89e3cbf 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -100,6 +100,13 @@ func init() { } ``` +### Input Plugin Development + +* Run `make static` followed by `make plugin-[pluginName]` to spin up a docker dev environment +using docker-compose. +* ***[Optional]*** When developing a plugin, add a `dev` directory with a `docker-compose.yml` and `telegraf.conf` +as well as any other supporting files, where sensible. + ## Adding Typed Metrics In addition the the `AddFields` function, the accumulator also supports an diff --git a/Makefile b/Makefile index 8650df267..2f3fcecea 100644 --- a/Makefile +++ b/Makefile @@ -92,4 +92,15 @@ docker-image: plugins/parsers/influx/machine.go: plugins/parsers/influx/machine.go.rl ragel -Z -G2 $^ -o $@ -.PHONY: deps telegraf install test test-windows lint vet test-all package clean docker-image fmtcheck uint64 +static: + @echo "Building static linux binary..." + @CGO_ENABLED=0 \ + GOOS=linux \ + GOARCH=amd64 \ + go build -ldflags "$(LDFLAGS)" ./cmd/telegraf + +plugin-%: + @echo "Starting dev environment for $${$(@)} input plugin..." + @docker-compose -f plugins/inputs/$${$(@)}/dev/docker-compose.yml up + +.PHONY: deps telegraf install test test-windows lint vet test-all package clean docker-image fmtcheck uint64 static diff --git a/plugins/inputs/logparser/dev/docker-compose.yml b/plugins/inputs/logparser/dev/docker-compose.yml new file mode 100644 index 000000000..dfb8e7843 --- /dev/null +++ b/plugins/inputs/logparser/dev/docker-compose.yml @@ -0,0 +1,13 @@ +version: '3' + +services: + telegraf: + image: glinton/scratch + volumes: + - ./telegraf.conf:/telegraf.conf + - ../../../../telegraf:/telegraf + - ./test.log:/var/log/test.log + entrypoint: + - /telegraf + - --config + - /telegraf.conf diff --git a/plugins/inputs/logparser/dev/telegraf.conf b/plugins/inputs/logparser/dev/telegraf.conf new file mode 100644 index 000000000..77e26528c --- /dev/null +++ b/plugins/inputs/logparser/dev/telegraf.conf @@ -0,0 +1,12 @@ +[agent] + interval="1s" + flush_interval="1s" + +[[inputs.logparser]] + files = ["/var/log/test.log"] + from_beginning = true + [inputs.logparser.grok] + patterns = [ "%{COMBINED_LOG_FORMAT}", "%{CLIENT:client_ip} %{NOTSPACE:ident} %{NOTSPACE:auth} \\[%{TIMESTAMP_ISO8601:timestamp}\\] \"(?:%{WORD:verb:tag} %{NOTSPACE:request}(?: HTTP/%{NUMBER:http_version:float})?|%{DATA})\" %{NUMBER:resp_code:tag} (?:%{NUMBER:resp_bytes:int}|-) %{QS:referrer} %{QS:agent}"] + +[[outputs.file]] + files = ["stdout"] diff --git a/plugins/inputs/logparser/dev/test.log b/plugins/inputs/logparser/dev/test.log new file mode 100644 index 000000000..2ff2e4ef6 --- /dev/null +++ b/plugins/inputs/logparser/dev/test.log @@ -0,0 +1,2 @@ +127.0.0.1 ident auth [10/Oct/2000:13:55:36 -0700] "GET /anything HTTP/1.0" 200 2326 "http://localhost:8083/" "Chrome/51.0.2704.84" +127.0.0.1 ident auth [2018-02-21 13:10:34,555] "GET /peter HTTP/1.0" 200 2326 "http://localhost:8083/" "Chrome/51.0.2704.84" diff --git a/plugins/inputs/mongodb/dev/docker-compose.yml b/plugins/inputs/mongodb/dev/docker-compose.yml new file mode 100644 index 000000000..9bf80f244 --- /dev/null +++ b/plugins/inputs/mongodb/dev/docker-compose.yml @@ -0,0 +1,16 @@ +version: '3' +services: + mongodb: + image: mongo + + telegraf: + image: glinton/scratch + volumes: + - ./telegraf.conf:/telegraf.conf + - ../../../../telegraf:/telegraf + depends_on: + - mongodb + entrypoint: + - /telegraf + - --config + - /telegraf.conf diff --git a/plugins/inputs/mongodb/dev/telegraf.conf b/plugins/inputs/mongodb/dev/telegraf.conf new file mode 100644 index 000000000..35077b161 --- /dev/null +++ b/plugins/inputs/mongodb/dev/telegraf.conf @@ -0,0 +1,9 @@ +[agent] + interval="1s" + flush_interval="3s" + +[[inputs.mongodb]] + servers = ["mongodb://mongodb:27017"] + +[[outputs.file]] + files = ["stdout"] diff --git a/plugins/inputs/mysql/dev/docker-compose.yml b/plugins/inputs/mysql/dev/docker-compose.yml new file mode 100644 index 000000000..5f1773fbc --- /dev/null +++ b/plugins/inputs/mysql/dev/docker-compose.yml @@ -0,0 +1,42 @@ +version: '3' + +services: + mysql: + image: mysql:5.7 + restart: always + environment: + MYSQL_ROOT_PASSWORD: telegraf + MYSQL_DATABASE: telegraf + MYSQL_USER: telegraf + MYSQL_PASSWORD: telegraf + maria: + image: mariadb + restart: always + environment: + MYSQL_ROOT_PASSWORD: telegraf + MYSQL_DATABASE: telegraf + MYSQL_USER: telegraf + MYSQL_PASSWORD: telegraf + command: mysqld --userstat=1 + percona: + image: percona + restart: always + environment: + MYSQL_ROOT_PASSWORD: telegraf + MYSQL_DATABASE: telegraf + MYSQL_USER: telegraf + MYSQL_PASSWORD: telegraf + + telegraf: + image: glinton/scratch + depends_on: + - mysql + - maria + - percona + volumes: + - ./telegraf.conf:/telegraf.conf + - ../../../../telegraf:/telegraf + entrypoint: + - /telegraf + - --config + - /telegraf.conf diff --git a/plugins/inputs/mysql/dev/telegraf.conf b/plugins/inputs/mysql/dev/telegraf.conf new file mode 100644 index 000000000..9aa4801ea --- /dev/null +++ b/plugins/inputs/mysql/dev/telegraf.conf @@ -0,0 +1,61 @@ +# Uncomment each input as needed to test plugin + +## mysql +#[[inputs.mysql]] +# servers = ["root:telegraf@tcp(mysql:3306)/"] +# gather_table_schema = true +# gather_process_list = true +# gather_user_statistics = true +# gather_info_schema_auto_inc = true +# gather_innodb_metrics = true +# gather_slave_status = true +# gather_binary_logs = false +# gather_table_io_waits = true +# gather_table_lock_waits = true +# gather_index_io_waits = true +# gather_event_waits = true +# gather_file_events_stats = true +# gather_perf_events_statements = true +# interval_slow = "30m" +# table_schema_databases = [] +# +## mariadb +#[[inputs.mysql]] +# servers = ["root:telegraf@tcp(maria:3306)/"] +# gather_table_schema = true +# gather_process_list = true +# gather_user_statistics = true +# gather_info_schema_auto_inc = true +# gather_innodb_metrics = true +# gather_slave_status = true +# gather_binary_logs = false +# gather_table_io_waits = true +# gather_table_lock_waits = true +# gather_index_io_waits = true +# gather_event_waits = true +# gather_file_events_stats = true +# gather_perf_events_statements = true +# interval_slow = "30m" +# table_schema_databases = [] + +# percona +[[inputs.mysql]] + servers = ["root:telegraf@tcp(percona:3306)/"] + gather_table_schema = true + gather_process_list = true + gather_user_statistics = true + gather_info_schema_auto_inc = true + gather_innodb_metrics = true + gather_slave_status = true + gather_binary_logs = false + gather_table_io_waits = true + gather_table_lock_waits = true + gather_index_io_waits = true + gather_event_waits = true + gather_file_events_stats = true + gather_perf_events_statements = true + interval_slow = "30m" + table_schema_databases = [] + +[[outputs.file]] + files = ["stdout"]