From 32f313a6a6b219f09cc90e9c11d0a72bdd709762 Mon Sep 17 00:00:00 2001 From: Daniel Nelson Date: Mon, 11 Sep 2017 11:56:04 -0700 Subject: [PATCH] Add polling method to logparser and tail inputs (#3213) --- plugins/inputs/logparser/README.md | 13 ++++++++----- plugins/inputs/logparser/logparser.go | 18 +++++++++++++++++- plugins/inputs/tail/README.md | 3 +++ plugins/inputs/tail/tail.go | 14 ++++++++++++++ 4 files changed, 42 insertions(+), 6 deletions(-) diff --git a/plugins/inputs/logparser/README.md b/plugins/inputs/logparser/README.md index a2ed0a689..f823cfe68 100644 --- a/plugins/inputs/logparser/README.md +++ b/plugins/inputs/logparser/README.md @@ -15,12 +15,15 @@ regex patterns. ## /var/log/*/*.log -> find all .log files with a parent dir in /var/log ## /var/log/apache.log -> only tail the apache log file files = ["/var/log/apache/access.log"] - + ## Read files that currently exist from the beginning. Files that are created ## while telegraf is running (and that match the "files" globs) will always ## be read from the beginning. from_beginning = false + ## Method used to watch for file updates. Can be either "inotify" or "poll". + # watch_method = "inotify" + ## Parse logstash-style "grok" patterns: ## Telegraf built-in parsing patterns: https://goo.gl/dkay10 [inputs.logparser.grok] @@ -34,15 +37,15 @@ regex patterns. ## Name of the outputted measurement name. measurement = "apache_access_log" - + ## Full path(s) to custom pattern files. custom_pattern_files = [] - + ## Custom patterns can also be defined here. Put one pattern per line. custom_patterns = ''' ''' - ## Timezone allows you to provide an override for timestamps that + ## Timezone allows you to provide an override for timestamps that ## don't already include an offset ## e.g. 04/06/2016 12:41:45 data one two 5.43µs ## @@ -145,7 +148,7 @@ Wed Apr 12 13:10:34 PST 2017 value=42 For cases where the timestamp itself is without offset, the `timezone` config var is available to denote an offset. By default (with `timezone` either omit, blank or set to `"UTC"`), the times are processed as if in the UTC timezone. If specified as `timezone = "Local"`, the timestamp -will be processed based on the current machine timezone configuration. Lastly, if using a +will be processed based on the current machine timezone configuration. Lastly, if using a timezone from the list of Unix [timezones](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones), the logparser grok will attempt to offset the timestamp accordingly. See test cases for more detailed examples. diff --git a/plugins/inputs/logparser/logparser.go b/plugins/inputs/logparser/logparser.go index 33b12fcc7..fd5a41f7f 100644 --- a/plugins/inputs/logparser/logparser.go +++ b/plugins/inputs/logparser/logparser.go @@ -19,6 +19,10 @@ import ( "github.com/influxdata/telegraf/plugins/inputs/logparser/grok" ) +const ( + defaultWatchMethod = "inotify" +) + // LogParser in the primary interface for the plugin type LogParser interface { ParseLine(line string) (telegraf.Metric, error) @@ -34,6 +38,7 @@ type logEntry struct { type LogParserPlugin struct { Files []string FromBeginning bool + WatchMethod string tailers map[string]*tail.Tail lines chan logEntry @@ -61,6 +66,9 @@ const sampleConfig = ` ## be read from the beginning. from_beginning = false + ## Method used to watch for file updates. Can be either "inotify" or "poll". + # watch_method = "inotify" + ## Parse logstash-style "grok" patterns: ## Telegraf built-in parsing patterns: https://goo.gl/dkay10 [inputs.logparser.grok] @@ -167,6 +175,11 @@ func (l *LogParserPlugin) tailNewfiles(fromBeginning bool) error { seek.Offset = 0 } + var poll bool + if l.WatchMethod == "poll" { + poll = true + } + // Create a "tailer" for each file for _, filepath := range l.Files { g, err := globpath.Compile(filepath) @@ -188,6 +201,7 @@ func (l *LogParserPlugin) tailNewfiles(fromBeginning bool) error { Follow: true, Location: &seek, MustExist: true, + Poll: poll, Logger: tail.DiscardingLogger, }) if err != nil { @@ -285,6 +299,8 @@ func (l *LogParserPlugin) Stop() { func init() { inputs.Add("logparser", func() telegraf.Input { - return &LogParserPlugin{} + return &LogParserPlugin{ + WatchMethod: defaultWatchMethod, + } }) } diff --git a/plugins/inputs/tail/README.md b/plugins/inputs/tail/README.md index 38f124174..5d7253649 100644 --- a/plugins/inputs/tail/README.md +++ b/plugins/inputs/tail/README.md @@ -39,6 +39,9 @@ The plugin expects messages in one of the ## Whether file is a named pipe pipe = false + ## Method used to watch for file updates. Can be either "inotify" or "poll". + # watch_method = "inotify" + ## Data format to consume. ## Each data format has its own unique set of configuration options, read ## more about them here: diff --git a/plugins/inputs/tail/tail.go b/plugins/inputs/tail/tail.go index 7812cf8c1..86534da5f 100644 --- a/plugins/inputs/tail/tail.go +++ b/plugins/inputs/tail/tail.go @@ -15,10 +15,15 @@ import ( "github.com/influxdata/telegraf/plugins/parsers" ) +const ( + defaultWatchMethod = "inotify" +) + type Tail struct { Files []string FromBeginning bool Pipe bool + WatchMethod string tailers []*tail.Tail parser parsers.Parser @@ -50,6 +55,9 @@ const sampleConfig = ` ## Whether file is a named pipe pipe = false + ## Method used to watch for file updates. Can be either "inotify" or "poll". + # watch_method = "inotify" + ## Data format to consume. ## Each data format has its own unique set of configuration options, read ## more about them here: @@ -83,6 +91,11 @@ func (t *Tail) Start(acc telegraf.Accumulator) error { } } + var poll bool + if t.WatchMethod == "poll" { + poll = true + } + // Create a "tailer" for each file for _, filepath := range t.Files { g, err := globpath.Compile(filepath) @@ -96,6 +109,7 @@ func (t *Tail) Start(acc telegraf.Accumulator) error { Follow: true, Location: seek, MustExist: true, + Poll: poll, Pipe: t.Pipe, Logger: tail.DiscardingLogger, })