From 79d9ea476187878b91dcc742c6ecfd75f93806a6 Mon Sep 17 00:00:00 2001 From: Max U Date: Tue, 26 Jun 2018 10:26:48 -0700 Subject: [PATCH] add docker-image spin up for reader --- Makefile | 14 ++- internal/config/config.go | 9 ++ plugins/inputs/reader/dev/docker-compse.yml | 13 +++ plugins/inputs/reader/dev/telegraf.conf | 106 ++++++++++++++++++++ plugins/inputs/reader/reader.go | 8 +- 5 files changed, 148 insertions(+), 2 deletions(-) create mode 100644 plugins/inputs/reader/dev/docker-compse.yml create mode 100644 plugins/inputs/reader/dev/telegraf.conf diff --git a/Makefile b/Makefile index 8650df267..99d7a3a39 100644 --- a/Makefile +++ b/Makefile @@ -92,4 +92,16 @@ docker-image: plugins/parsers/influx/machine.go: plugins/parsers/influx/machine.go.rl ragel -Z -G2 $^ -o $@ -.PHONY: deps telegraf install test test-windows lint vet test-all package clean docker-image fmtcheck uint64 +static: + @echo "Building static linux binary..." + @CGO_ENABLED=0 \ + GOOS=linux \ + GOARCH=amd64 \ + go build -ldflags "$(LDFLAGS)" ./cmd/telegraf + +plugin-%: + @echo "Starting dev environment for $${$(@)} input plugin..." + @docker-compose -f plugins/inputs/$${$(@)}/dev/docker-compose.yml up + +.PHONY: deps telegraf install test test-windows lint vet test-all package clean docker-image fmtcheck uint64 static + diff --git a/internal/config/config.go b/internal/config/config.go index 1a98c61c5..845b0b92c 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -1393,6 +1393,14 @@ func buildParser(name string, tbl *ast.Table) (parsers.Parser, error) { c.MetricName = name + if node, ok := tbl.Fields["name_override"]; ok { + if kv, ok := node.(*ast.KeyValue); ok { + if str, ok := kv.Value.(*ast.String); ok { + c.MetricName = str.Value + } + } + } + delete(tbl.Fields, "data_format") delete(tbl.Fields, "separator") delete(tbl.Fields, "templates") @@ -1411,6 +1419,7 @@ func buildParser(name string, tbl *ast.Table) (parsers.Parser, error) { delete(tbl.Fields, "custom_patterns") delete(tbl.Fields, "custom_pattern_files") delete(tbl.Fields, "timezone") + delete(tbl.Fields, "name_override") return parsers.NewParser(c) } diff --git a/plugins/inputs/reader/dev/docker-compse.yml b/plugins/inputs/reader/dev/docker-compse.yml new file mode 100644 index 000000000..a5abde8ac --- /dev/null +++ b/plugins/inputs/reader/dev/docker-compse.yml @@ -0,0 +1,13 @@ +version: '3' + +# services: + telegraf: + image: glinton/scratch + volumes: + - ./telegraf.conf:/telegraf.conf + - ../../../../telegraf:/telegraf + - ./test.log:/var/log/test.log + entrypoint: + - /telegraf + - --config + - /telegraf.conf \ No newline at end of file diff --git a/plugins/inputs/reader/dev/telegraf.conf b/plugins/inputs/reader/dev/telegraf.conf new file mode 100644 index 000000000..971845682 --- /dev/null +++ b/plugins/inputs/reader/dev/telegraf.conf @@ -0,0 +1,106 @@ + +# Global tags can be specified here in key="value" format. +[global_tags] + # dc = "us-east-1" # will tag all metrics with dc=us-east-1 + # rack = "1a" + ## Environment variables can be used as tags, and throughout the config file + # user = "$USER" + + +# Configuration for telegraf agent +[agent] + ## Default data collection interval for all inputs + interval = "10s" + ## Rounds collection interval to 'interval' + ## ie, if interval="10s" then always collect on :00, :10, :20, etc. + round_interval = true + + ## Telegraf will send metrics to outputs in batches of at most + ## metric_batch_size metrics. + ## This controls the size of writes that Telegraf sends to output plugins. + metric_batch_size = 1000 + + ## For failed writes, telegraf will cache metric_buffer_limit metrics for each + ## output, and will flush this buffer on a successful write. Oldest metrics + ## are dropped first when this buffer fills. + ## This buffer only fills when writes fail to output plugin(s). + metric_buffer_limit = 10000 + + ## Collection jitter is used to jitter the collection by a random amount. + ## Each plugin will sleep for a random time within jitter before collecting. + ## This can be used to avoid many plugins querying things like sysfs at the + ## same time, which can have a measurable effect on the system. + collection_jitter = "0s" + + ## Default flushing interval for all outputs. You shouldn't set this below + ## interval. Maximum flush_interval will be flush_interval + flush_jitter + flush_interval = "10s" + ## Jitter the flush interval by a random amount. This is primarily to avoid + ## large write spikes for users running a large number of telegraf instances. + ## ie, a jitter of 5s and interval 10s means flushes will happen every 10-15s + flush_jitter = "0s" + + ## By default or when set to "0s", precision will be set to the same + ## timestamp order as the collection interval, with the maximum being 1s. + ## ie, when interval = "10s", precision will be "1s" + ## when interval = "250ms", precision will be "1ms" + ## Precision will NOT be used for service inputs. It is up to each individual + ## service input to set the timestamp at the appropriate precision. + ## Valid time units are "ns", "us" (or "µs"), "ms", "s". + precision = "" + + ## Logging configuration: + ## Run telegraf with debug log messages. + debug = false + ## Run telegraf in quiet mode (error log messages only). + quiet = false + ## Specify the log file name. The empty string means to log to stderr. + logfile = "" + + ## Override default hostname, if empty use os.Hostname() + hostname = "" + ## If set to true, do no set the "host" tag in the telegraf agent. + omit_hostname = false + +# # reload and gather from file[s] on telegraf's interval +[[inputs.reader]] +# ## These accept standard unix glob matching rules, but with the addition of +# ## ** as a "super asterisk". ie: +# ## /var/log/**.log -> recursively find all .log files in /var/log +# ## /var/log/*/*.log -> find all .log files with a parent dir in /var/log +# ## /var/log/apache.log -> only tail the apache log file +files = ["/Users/maxu/go/src/github.com/influxdata/telegraf/plugins/inputs/reader/testfiles/grok_a.log"] +# +# ## The dataformat to be read from files +# ## Each data format has its own unique set of configuration options, read +# ## more about them here: +# ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md +data_format = "grok" +# + +patterns = ["%{COMMON_LOG_FORMAT}"] +# +# ## Name of the outputted measurement name. +measurement = "grok_reader" +# +# ## Full path(s) to custom pattern files. +# custom_pattern_files = [] +# +# ## Custom patterns can also be defined here. Put one pattern per line. +# custom_patterns = ''' +# ''' +# +# ## Timezone allows you to provide an override for timestamps that +# ## don't already include an offset +# ## e.g. 04/06/2016 12:41:45 data one two 5.43µs +# ## +# ## Default: "" which renders UTC +# ## Options are as follows: +# ## 1. Local -- interpret based on machine localtime +# ## 2. "Canada/Eastern" -- Unix TZ values like those found in https://en.wikipedia.org/wiki/List_of_tz_database_time_zones +# ## 3. UTC -- or blank/unspecified, will return timestamp in UTC +# timezone = "Canada/Eastern" + + +[[outputs.file]] + files = ["stdout"] \ No newline at end of file diff --git a/plugins/inputs/reader/reader.go b/plugins/inputs/reader/reader.go index bfccb87d3..eb2ce4804 100644 --- a/plugins/inputs/reader/reader.go +++ b/plugins/inputs/reader/reader.go @@ -50,7 +50,13 @@ func (r *Reader) Gather(acc telegraf.Accumulator) error { return err } - for _, m := range metrics { + for i, m := range metrics { + + //error if m is nil + if m == nil { + log.Printf("E! Metric could not be parsed from: %v, on line %v", k, i) + continue + } acc.AddFields(m.Name(), m.Fields(), m.Tags()) } }