diff --git a/README.md b/README.md index 03e3a8f58..a0f396cad 100644 --- a/README.md +++ b/README.md @@ -158,6 +158,7 @@ configuration options. * [fibaro](./plugins/inputs/fibaro) * [file](./plugins/inputs/file) * [filestat](./plugins/inputs/filestat) +* [filecount](./plugins/inputs/filecount) * [fluentd](./plugins/inputs/fluentd) * [graylog](./plugins/inputs/graylog) * [haproxy](./plugins/inputs/haproxy) diff --git a/plugins/inputs/all/all.go b/plugins/inputs/all/all.go index 8989684e4..4d46a5490 100644 --- a/plugins/inputs/all/all.go +++ b/plugins/inputs/all/all.go @@ -31,6 +31,7 @@ import ( _ "github.com/influxdata/telegraf/plugins/inputs/fail2ban" _ "github.com/influxdata/telegraf/plugins/inputs/fibaro" _ "github.com/influxdata/telegraf/plugins/inputs/file" + _ "github.com/influxdata/telegraf/plugins/inputs/filecount" _ "github.com/influxdata/telegraf/plugins/inputs/filestat" _ "github.com/influxdata/telegraf/plugins/inputs/fluentd" _ "github.com/influxdata/telegraf/plugins/inputs/graylog" diff --git a/plugins/inputs/filecount/README.md b/plugins/inputs/filecount/README.md new file mode 100644 index 000000000..ccec532aa --- /dev/null +++ b/plugins/inputs/filecount/README.md @@ -0,0 +1,49 @@ +# filecount Input Plugin + +Counts files in directories that match certain criteria. + +### Configuration: + +```toml +# Count files in a directory +[[inputs.filecount]] + ## Directory to gather stats about. + directory = "/var/cache/apt/archives" + + ## Only count files that match the name pattern. Defaults to "*". + name = "*.deb" + + ## Count files in subdirectories. Defaults to true. + recursive = false + + ## Only count regular files. Defaults to true. + regular_only = true + + ## Only count files that are at least this size in bytes. If size is + ## a negative number, only count files that are smaller than the + ## absolute value of size. Defaults to 0. + size = 0 + + ## Only count files that have not been touched for at least this + ## duration. If mtime is negative, only count files that have been + ## touched in this duration. Defaults to "0s". + mtime = "0s" +``` + +### Measurements & Fields: + +- filecount + - count (int) + +### Tags: + +- All measurements have the following tags: + - directory (the directory path, as specified in the config) + +### Example Output: + +``` +$ telegraf --config /etc/telegraf/telegraf.conf --input-filter filecount --test +> filecount,directory=/var/cache/apt,host=czernobog count=7i 1530034445000000000 +> filecount,directory=/tmp,host=czernobog count=17i 1530034445000000000 +``` diff --git a/plugins/inputs/filecount/filecount.go b/plugins/inputs/filecount/filecount.go new file mode 100644 index 000000000..6041ec7b5 --- /dev/null +++ b/plugins/inputs/filecount/filecount.go @@ -0,0 +1,215 @@ +package filecount + +import ( + "os" + "path/filepath" + "time" + + "github.com/influxdata/telegraf" + "github.com/influxdata/telegraf/internal" + "github.com/influxdata/telegraf/plugins/inputs" +) + +const sampleConfig = ` + ## Directory to gather stats about. + directory = "/var/cache/apt/archives" + + ## Only count files that match the name pattern. Defaults to "*". + name = "*.deb" + + ## Count files in subdirectories. Defaults to true. + recursive = false + + ## Only count regular files. Defaults to true. + regular_only = true + + ## Only count files that are at least this size in bytes. If size is + ## a negative number, only count files that are smaller than the + ## absolute value of size. Defaults to 0. + size = 0 + + ## Only count files that have not been touched for at least this + ## duration. If mtime is negative, only count files that have been + ## touched in this duration. Defaults to "0s". + mtime = "0s" +` + +type FileCount struct { + Directory string + Name string + Recursive bool + RegularOnly bool + Size int64 + MTime internal.Duration `toml:"mtime"` + fileFilters []fileFilterFunc +} + +type countFunc func(os.FileInfo) +type fileFilterFunc func(os.FileInfo) (bool, error) + +func (_ *FileCount) Description() string { + return "Count files in a directory" +} + +func (_ *FileCount) SampleConfig() string { return sampleConfig } + +func rejectNilFilters(filters []fileFilterFunc) []fileFilterFunc { + filtered := make([]fileFilterFunc, 0, len(filters)) + for _, f := range filters { + if f != nil { + filtered = append(filtered, f) + } + } + return filtered +} + +func (fc *FileCount) nameFilter() fileFilterFunc { + if fc.Name == "*" { + return nil + } + + return func(f os.FileInfo) (bool, error) { + match, err := filepath.Match(fc.Name, f.Name()) + if err != nil { + return false, err + } + return match, nil + } +} + +func (fc *FileCount) regularOnlyFilter() fileFilterFunc { + if !fc.RegularOnly { + return nil + } + + return func(f os.FileInfo) (bool, error) { + return f.Mode().IsRegular(), nil + } +} + +func (fc *FileCount) sizeFilter() fileFilterFunc { + if fc.Size == 0 { + return nil + } + + return func(f os.FileInfo) (bool, error) { + if !f.Mode().IsRegular() { + return false, nil + } + if fc.Size < 0 { + return f.Size() < -fc.Size, nil + } + return f.Size() >= fc.Size, nil + } +} + +func (fc *FileCount) mtimeFilter() fileFilterFunc { + if fc.MTime.Duration == 0 { + return nil + } + + return func(f os.FileInfo) (bool, error) { + age := absDuration(fc.MTime.Duration) + mtime := time.Now().Add(-age) + if fc.MTime.Duration < 0 { + return f.ModTime().After(mtime), nil + } + return f.ModTime().Before(mtime), nil + } +} + +func absDuration(x time.Duration) time.Duration { + if x < 0 { + return -x + } + return x +} + +func count(basedir string, recursive bool, countFn countFunc) error { + walkFn := func(path string, file os.FileInfo, err error) error { + if path == basedir { + return nil + } + countFn(file) + if !recursive && file.IsDir() { + return filepath.SkipDir + } + return nil + } + return filepath.Walk(basedir, walkFn) +} + +func (fc *FileCount) initFileFilters() { + filters := []fileFilterFunc{ + fc.nameFilter(), + fc.regularOnlyFilter(), + fc.sizeFilter(), + fc.mtimeFilter(), + } + fc.fileFilters = rejectNilFilters(filters) +} + +func (fc *FileCount) filter(file os.FileInfo) (bool, error) { + if fc.fileFilters == nil { + fc.initFileFilters() + } + + for _, fileFilter := range fc.fileFilters { + match, err := fileFilter(file) + if err != nil { + return false, err + } + if !match { + return false, nil + } + } + + return true, nil +} + +func (fc *FileCount) Gather(acc telegraf.Accumulator) error { + numFiles := int64(0) + countFn := func(f os.FileInfo) { + match, err := fc.filter(f) + if err != nil { + acc.AddError(err) + return + } + if !match { + return + } + numFiles++ + } + err := count(fc.Directory, fc.Recursive, countFn) + if err != nil { + acc.AddError(err) + } + + acc.AddFields("filecount", + map[string]interface{}{ + "count": numFiles, + }, + map[string]string{ + "directory": fc.Directory, + }) + + return nil +} + +func NewFileCount() *FileCount { + return &FileCount{ + Directory: "", + Name: "*", + Recursive: true, + RegularOnly: true, + Size: 0, + MTime: internal.Duration{Duration: 0}, + fileFilters: nil, + } +} + +func init() { + inputs.Add("filecount", func() telegraf.Input { + return NewFileCount() + }) +} diff --git a/plugins/inputs/filecount/filecount_test.go b/plugins/inputs/filecount/filecount_test.go new file mode 100644 index 000000000..294a8b965 --- /dev/null +++ b/plugins/inputs/filecount/filecount_test.go @@ -0,0 +1,99 @@ +package filecount + +import ( + "os" + "path/filepath" + "runtime" + "strings" + "testing" + "time" + + "github.com/influxdata/telegraf/internal" + "github.com/influxdata/telegraf/testutil" + "github.com/stretchr/testify/require" +) + +func TestNoFilters(t *testing.T) { + fc := getNoFilterFileCount() + matches := []string{"foo", "bar", "baz", "qux", + "subdir/", "subdir/quux", "subdir/quuz"} + require.True(t, fileCountEquals(fc, len(matches))) +} + +func TestNameFilter(t *testing.T) { + fc := getNoFilterFileCount() + fc.Name = "ba*" + matches := []string{"bar", "baz"} + require.True(t, fileCountEquals(fc, len(matches))) +} + +func TestNonRecursive(t *testing.T) { + fc := getNoFilterFileCount() + fc.Recursive = false + matches := []string{"foo", "bar", "baz", "qux", "subdir"} + require.True(t, fileCountEquals(fc, len(matches))) +} + +func TestRegularOnlyFilter(t *testing.T) { + fc := getNoFilterFileCount() + fc.RegularOnly = true + matches := []string{ + "foo", "bar", "baz", "qux", "subdir/quux", "subdir/quuz", + } + require.True(t, fileCountEquals(fc, len(matches))) +} + +func TestSizeFilter(t *testing.T) { + fc := getNoFilterFileCount() + fc.Size = -100 + matches := []string{"foo", "bar", "baz", + "subdir/quux", "subdir/quuz"} + require.True(t, fileCountEquals(fc, len(matches))) + + fc.Size = 100 + matches = []string{"qux"} + require.True(t, fileCountEquals(fc, len(matches))) +} + +func TestMTimeFilter(t *testing.T) { + oldFile := filepath.Join(getTestdataDir(), "baz") + mtime := time.Date(1979, time.December, 14, 18, 25, 5, 0, time.UTC) + if err := os.Chtimes(oldFile, mtime, mtime); err != nil { + t.Skip("skipping mtime filter test.") + } + fileAge := time.Since(mtime) - (60 * time.Second) + + fc := getNoFilterFileCount() + fc.MTime = internal.Duration{Duration: -fileAge} + matches := []string{"foo", "bar", "qux", + "subdir/", "subdir/quux", "subdir/quuz"} + require.True(t, fileCountEquals(fc, len(matches))) + + fc.MTime = internal.Duration{Duration: fileAge} + matches = []string{"baz"} + require.True(t, fileCountEquals(fc, len(matches))) +} + +func getNoFilterFileCount() FileCount { + return FileCount{ + Directory: getTestdataDir(), + Name: "*", + Recursive: true, + RegularOnly: false, + Size: 0, + MTime: internal.Duration{Duration: 0}, + fileFilters: nil, + } +} + +func getTestdataDir() string { + _, filename, _, _ := runtime.Caller(1) + return strings.Replace(filename, "filecount_test.go", "testdata/", 1) +} + +func fileCountEquals(fc FileCount, expectedCount int) bool { + tags := map[string]string{"directory": getTestdataDir()} + acc := testutil.Accumulator{} + acc.GatherError(fc.Gather) + return acc.HasPoint("filecount", tags, "count", int64(expectedCount)) +} diff --git a/plugins/inputs/filecount/testdata/bar b/plugins/inputs/filecount/testdata/bar new file mode 100644 index 000000000..e69de29bb diff --git a/plugins/inputs/filecount/testdata/baz b/plugins/inputs/filecount/testdata/baz new file mode 100644 index 000000000..e69de29bb diff --git a/plugins/inputs/filecount/testdata/foo b/plugins/inputs/filecount/testdata/foo new file mode 100644 index 000000000..e69de29bb diff --git a/plugins/inputs/filecount/testdata/qux b/plugins/inputs/filecount/testdata/qux new file mode 100644 index 000000000..c7288f23d --- /dev/null +++ b/plugins/inputs/filecount/testdata/qux @@ -0,0 +1,7 @@ +Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do +eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad +minim veniam, quis nostrud exercitation ullamco laboris nisi ut +aliquip ex ea commodo consequat. Duis aute irure dolor in +reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla +pariatur. Excepteur sint occaecat cupidatat non proident, sunt in +culpa qui officia deserunt mollit anim id est laborum. diff --git a/plugins/inputs/filecount/testdata/subdir/quux b/plugins/inputs/filecount/testdata/subdir/quux new file mode 100644 index 000000000..e69de29bb diff --git a/plugins/inputs/filecount/testdata/subdir/quuz b/plugins/inputs/filecount/testdata/subdir/quuz new file mode 100644 index 000000000..e69de29bb