Add file input plugin and grok parser (#4332)

plugins/inputs/file/README.md

@@ -0,0 +1,25 @@
+# File Input Plugin
+
+The file plugin updates a list of files every interval and parses the contents
+using the selected [input data format](/docs/DATA_FORMATS_INPUT.md).
+
+Files will always be read in their entirety, if you wish to tail/follow a file
+use the [tail input plugin](/plugins/inputs/tail) instead.
+
+### Configuration:
+```toml
+[[inputs.file]]
+  ## Files to parse each interval.
+  ## These accept standard unix glob matching rules, but with the addition of
+  ## ** as a "super asterisk". ie:
+  ##   /var/log/**.log     -> recursively find all .log files in /var/log
+  ##   /var/log/*/*.log    -> find all .log files with a parent dir in /var/log
+  ##   /var/log/apache.log -> only tail the apache log file
+  files = ["/var/log/apache/access.log"]
+
+  ## Data format to consume.
+  ## Each data format has its own unique set of configuration options, read
+  ## more about them here:
+  ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md
+  data_format = "influx"
+```

plugins/inputs/file/dev/docker-compose.yml

@@ -0,0 +1,13 @@
+version: '3'
+
+services:
+  telegraf:
+      image: glinton/scratch
+      volumes:
+        - ./telegraf.conf:/telegraf.conf
+        - ../../../../telegraf:/telegraf
+        - ./json_a.log:/var/log/test.log
+      entrypoint:
+        - /telegraf
+        - --config
+        - /telegraf.conf

plugins/inputs/file/dev/json_a.log

@@ -0,0 +1,14 @@
+{
+"parent": {
+	"child": 3.0,
+	"ignored_child": "hi"
+},
+"ignored_null": null,
+"integer": 4,
+"list": [3, 4],
+"ignored_parent": {
+	"another_ignored_null": null,
+	"ignored_string": "hello, world!"
+},
+"another_list": [4]
+}

plugins/inputs/file/dev/telegraf.conf

@@ -0,0 +1,7 @@
+[[inputs.file]]
+  files = ["/var/log/test.log"]
+  data_format = "json"
+  name_override = "json_file"
+
+[[outputs.file]]
+  files = ["stdout"]

plugins/inputs/file/file.go

@@ -0,0 +1,102 @@
+package file
+
+import (
+	"fmt"
+	"io/ioutil"
+
+	"github.com/influxdata/telegraf"
+	"github.com/influxdata/telegraf/internal/globpath"
+	"github.com/influxdata/telegraf/plugins/inputs"
+	"github.com/influxdata/telegraf/plugins/parsers"
+)
+
+type File struct {
+	Files         []string `toml:"files"`
+	FromBeginning bool
+	parser        parsers.Parser
+
+	filenames []string
+}
+
+const sampleConfig = `
+  ## Files to parse each interval.
+  ## These accept standard unix glob matching rules, but with the addition of
+  ## ** as a "super asterisk". ie:
+  ##   /var/log/**.log     -> recursively find all .log files in /var/log
+  ##   /var/log/*/*.log    -> find all .log files with a parent dir in /var/log
+  ##   /var/log/apache.log -> only tail the apache log file
+  files = ["/var/log/apache/access.log"]
+
+  ## The dataformat to be read from files
+  ## Each data format has its own unique set of configuration options, read
+  ## more about them here:
+  ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md
+  data_format = "influx"
+`
+
+// SampleConfig returns the default configuration of the Input
+func (f *File) SampleConfig() string {
+	return sampleConfig
+}
+
+func (f *File) Description() string {
+	return "reload and gather from file[s] on telegraf's interval"
+}
+
+func (f *File) Gather(acc telegraf.Accumulator) error {
+	err := f.refreshFilePaths()
+	if err != nil {
+		return err
+	}
+	for _, k := range f.filenames {
+		metrics, err := f.readMetric(k)
+		if err != nil {
+			return err
+		}
+
+		for _, m := range metrics {
+			acc.AddFields(m.Name(), m.Fields(), m.Tags(), m.Time())
+		}
+	}
+	return nil
+}
+
+func (f *File) SetParser(p parsers.Parser) {
+	f.parser = p
+}
+
+func (f *File) refreshFilePaths() error {
+	var allFiles []string
+	for _, file := range f.Files {
+		g, err := globpath.Compile(file)
+		if err != nil {
+			return fmt.Errorf("could not compile glob %v: %v", file, err)
+		}
+		files := g.Match()
+		if len(files) <= 0 {
+			return fmt.Errorf("could not find file: %v", file)
+		}
+
+		for k := range files {
+			allFiles = append(allFiles, k)
+		}
+	}
+
+	f.filenames = allFiles
+	return nil
+}
+
+func (f *File) readMetric(filename string) ([]telegraf.Metric, error) {
+	fileContents, err := ioutil.ReadFile(filename)
+	if err != nil {
+		return nil, fmt.Errorf("E! Error file: %v could not be read, %s", filename, err)
+	}
+	return f.parser.Parse(fileContents)
+
+}
+
+func init() {
+	inputs.Add("file", func() telegraf.Input {
+		return &File{}
+	})
+}

plugins/inputs/file/file_test.go

@@ -0,0 +1,61 @@
+package file
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/influxdata/telegraf/plugins/parsers"
+	"github.com/influxdata/telegraf/testutil"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestRefreshFilePaths(t *testing.T) {
+	wd, err := os.Getwd()
+	r := File{
+		Files: []string{filepath.Join(wd, "testfiles/**.log")},
+	}
+
+	err = r.refreshFilePaths()
+	require.NoError(t, err)
+	assert.Equal(t, len(r.filenames), 2)
+}
+func TestJSONParserCompile(t *testing.T) {
+	var acc testutil.Accumulator
+	wd, _ := os.Getwd()
+	r := File{
+		Files: []string{filepath.Join(wd, "testfiles/json_a.log")},
+	}
+	parserConfig := parsers.Config{
+		DataFormat: "json",
+		TagKeys:    []string{"parent_ignored_child"},
+	}
+	nParser, err := parsers.NewParser(&parserConfig)
+	r.parser = nParser
+	assert.NoError(t, err)
+
+	r.Gather(&acc)
+	assert.Equal(t, map[string]string{"parent_ignored_child": "hi"}, acc.Metrics[0].Tags)
+	assert.Equal(t, 5, len(acc.Metrics[0].Fields))
+}
+
+func TestGrokParser(t *testing.T) {
+	wd, _ := os.Getwd()
+	var acc testutil.Accumulator
+	r := File{
+		Files: []string{filepath.Join(wd, "testfiles/grok_a.log")},
+	}
+
+	parserConfig := parsers.Config{
+		DataFormat:   "grok",
+		GrokPatterns: []string{"%{COMMON_LOG_FORMAT}"},
+	}
+
+	nParser, err := parsers.NewParser(&parserConfig)
+	r.parser = nParser
+	assert.NoError(t, err)
+
+	err = r.Gather(&acc)
+	assert.Equal(t, 2, len(acc.Metrics))
+}

plugins/inputs/file/testfiles/grok_a.log

@@ -0,0 +1,2 @@
+127.0.0.1 user-identifier frank [10/Oct/2000:13:55:36 -0700] "GET /apache_pb.gif HTTP/1.0" 200 2326
+128.0.0.1 user-identifier tony [10/Oct/2000:13:55:36 -0800] "GET /apache_pb.gif HTTP/1.0" 300 45

plugins/inputs/file/testfiles/json_a.log

@@ -0,0 +1,14 @@
+{
+	"parent": {
+		"child": 3.0,
+		"ignored_child": "hi"
+	},
+	"ignored_null": null,
+	"integer": 4,
+	"list": [3, 4],
+	"ignored_parent": {
+		"another_ignored_null": null,
+		"ignored_string": "hello, world!"
+	},
+	"another_list": [4]
+}