Add filecount input plugin (#4363)

README.md

@@ -158,6 +158,7 @@ configuration options.
 * [fibaro](./plugins/inputs/fibaro)
 * [file](./plugins/inputs/file)
 * [filestat](./plugins/inputs/filestat)
+* [filecount](./plugins/inputs/filecount)
 * [fluentd](./plugins/inputs/fluentd)
 * [graylog](./plugins/inputs/graylog)
 * [haproxy](./plugins/inputs/haproxy)

plugins/inputs/all/all.go

@@ -31,6 +31,7 @@ import (
 	_ "github.com/influxdata/telegraf/plugins/inputs/fail2ban"
 	_ "github.com/influxdata/telegraf/plugins/inputs/fibaro"
 	_ "github.com/influxdata/telegraf/plugins/inputs/file"
+	_ "github.com/influxdata/telegraf/plugins/inputs/filecount"
 	_ "github.com/influxdata/telegraf/plugins/inputs/filestat"
 	_ "github.com/influxdata/telegraf/plugins/inputs/fluentd"
 	_ "github.com/influxdata/telegraf/plugins/inputs/graylog"

plugins/inputs/filecount/README.md

@@ -0,0 +1,49 @@
+# filecount Input Plugin
+
+Counts files in directories that match certain criteria.
+
+### Configuration:
+
+```toml
+# Count files in a directory
+[[inputs.filecount]]
+  ## Directory to gather stats about.
+  directory = "/var/cache/apt/archives"
+
+  ## Only count files that match the name pattern. Defaults to "*".
+  name = "*.deb"
+
+  ## Count files in subdirectories. Defaults to true.
+  recursive = false
+
+  ## Only count regular files. Defaults to true.
+  regular_only = true
+
+  ## Only count files that are at least this size in bytes. If size is
+  ## a negative number, only count files that are smaller than the
+  ## absolute value of size. Defaults to 0.
+  size = 0
+
+  ## Only count files that have not been touched for at least this
+  ## duration. If mtime is negative, only count files that have been
+  ## touched in this duration. Defaults to "0s".
+  mtime = "0s"
+```
+
+### Measurements & Fields:
+
+- filecount
+    - count (int)
+
+### Tags:
+
+- All measurements have the following tags:
+    - directory (the directory path, as specified in the config)
+
+### Example Output:
+
+```
+$ telegraf --config /etc/telegraf/telegraf.conf --input-filter filecount --test
+> filecount,directory=/var/cache/apt,host=czernobog count=7i 1530034445000000000
+> filecount,directory=/tmp,host=czernobog count=17i 1530034445000000000
+```

plugins/inputs/filecount/filecount.go

@@ -0,0 +1,215 @@
+package filecount
+
+import (
+	"os"
+	"path/filepath"
+	"time"
+
+	"github.com/influxdata/telegraf"
+	"github.com/influxdata/telegraf/internal"
+	"github.com/influxdata/telegraf/plugins/inputs"
+)
+
+const sampleConfig = `
+  ## Directory to gather stats about.
+  directory = "/var/cache/apt/archives"
+
+  ## Only count files that match the name pattern. Defaults to "*".
+  name = "*.deb"
+
+  ## Count files in subdirectories. Defaults to true.
+  recursive = false
+
+  ## Only count regular files. Defaults to true.
+  regular_only = true
+
+  ## Only count files that are at least this size in bytes. If size is
+  ## a negative number, only count files that are smaller than the
+  ## absolute value of size. Defaults to 0.
+  size = 0
+
+  ## Only count files that have not been touched for at least this
+  ## duration. If mtime is negative, only count files that have been
+  ## touched in this duration. Defaults to "0s".
+  mtime = "0s"
+`
+
+type FileCount struct {
+	Directory   string
+	Name        string
+	Recursive   bool
+	RegularOnly bool
+	Size        int64
+	MTime       internal.Duration `toml:"mtime"`
+	fileFilters []fileFilterFunc
+}
+
+type countFunc func(os.FileInfo)
+type fileFilterFunc func(os.FileInfo) (bool, error)
+
+func (_ *FileCount) Description() string {
+	return "Count files in a directory"
+}
+
+func (_ *FileCount) SampleConfig() string { return sampleConfig }
+
+func rejectNilFilters(filters []fileFilterFunc) []fileFilterFunc {
+	filtered := make([]fileFilterFunc, 0, len(filters))
+	for _, f := range filters {
+		if f != nil {
+			filtered = append(filtered, f)
+		}
+	}
+	return filtered
+}
+
+func (fc *FileCount) nameFilter() fileFilterFunc {
+	if fc.Name == "*" {
+		return nil
+	}
+
+	return func(f os.FileInfo) (bool, error) {
+		match, err := filepath.Match(fc.Name, f.Name())
+		if err != nil {
+			return false, err
+		}
+		return match, nil
+	}
+}
+
+func (fc *FileCount) regularOnlyFilter() fileFilterFunc {
+	if !fc.RegularOnly {
+		return nil
+	}
+
+	return func(f os.FileInfo) (bool, error) {
+		return f.Mode().IsRegular(), nil
+	}
+}
+
+func (fc *FileCount) sizeFilter() fileFilterFunc {
+	if fc.Size == 0 {
+		return nil
+	}
+
+	return func(f os.FileInfo) (bool, error) {
+		if !f.Mode().IsRegular() {
+			return false, nil
+		}
+		if fc.Size < 0 {
+			return f.Size() < -fc.Size, nil
+		}
+		return f.Size() >= fc.Size, nil
+	}
+}
+
+func (fc *FileCount) mtimeFilter() fileFilterFunc {
+	if fc.MTime.Duration == 0 {
+		return nil
+	}
+
+	return func(f os.FileInfo) (bool, error) {
+		age := absDuration(fc.MTime.Duration)
+		mtime := time.Now().Add(-age)
+		if fc.MTime.Duration < 0 {
+			return f.ModTime().After(mtime), nil
+		}
+		return f.ModTime().Before(mtime), nil
+	}
+}
+
+func absDuration(x time.Duration) time.Duration {
+	if x < 0 {
+		return -x
+	}
+	return x
+}
+
+func count(basedir string, recursive bool, countFn countFunc) error {
+	walkFn := func(path string, file os.FileInfo, err error) error {
+		if path == basedir {
+			return nil
+		}
+		countFn(file)
+		if !recursive && file.IsDir() {
+			return filepath.SkipDir
+		}
+		return nil
+	}
+	return filepath.Walk(basedir, walkFn)
+}
+
+func (fc *FileCount) initFileFilters() {
+	filters := []fileFilterFunc{
+		fc.nameFilter(),
+		fc.regularOnlyFilter(),
+		fc.sizeFilter(),
+		fc.mtimeFilter(),
+	}
+	fc.fileFilters = rejectNilFilters(filters)
+}
+
+func (fc *FileCount) filter(file os.FileInfo) (bool, error) {
+	if fc.fileFilters == nil {
+		fc.initFileFilters()
+	}
+
+	for _, fileFilter := range fc.fileFilters {
+		match, err := fileFilter(file)
+		if err != nil {
+			return false, err
+		}
+		if !match {
+			return false, nil
+		}
+	}
+
+	return true, nil
+}
+
+func (fc *FileCount) Gather(acc telegraf.Accumulator) error {
+	numFiles := int64(0)
+	countFn := func(f os.FileInfo) {
+		match, err := fc.filter(f)
+		if err != nil {
+			acc.AddError(err)
+			return
+		}
+		if !match {
+			return
+		}
+		numFiles++
+	}
+	err := count(fc.Directory, fc.Recursive, countFn)
+	if err != nil {
+		acc.AddError(err)
+	}
+
+	acc.AddFields("filecount",
+		map[string]interface{}{
+			"count": numFiles,
+		},
+		map[string]string{
+			"directory": fc.Directory,
+		})
+
+	return nil
+}
+
+func NewFileCount() *FileCount {
+	return &FileCount{
+		Directory:   "",
+		Name:        "*",
+		Recursive:   true,
+		RegularOnly: true,
+		Size:        0,
+		MTime:       internal.Duration{Duration: 0},
+		fileFilters: nil,
+	}
+}
+
+func init() {
+	inputs.Add("filecount", func() telegraf.Input {
+		return NewFileCount()
+	})
+}

plugins/inputs/filecount/filecount_test.go

@@ -0,0 +1,99 @@
+package filecount
+
+import (
+	"os"
+	"path/filepath"
+	"runtime"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/influxdata/telegraf/internal"
+	"github.com/influxdata/telegraf/testutil"
+	"github.com/stretchr/testify/require"
+)
+
+func TestNoFilters(t *testing.T) {
+	fc := getNoFilterFileCount()
+	matches := []string{"foo", "bar", "baz", "qux",
+		"subdir/", "subdir/quux", "subdir/quuz"}
+	require.True(t, fileCountEquals(fc, len(matches)))
+}
+
+func TestNameFilter(t *testing.T) {
+	fc := getNoFilterFileCount()
+	fc.Name = "ba*"
+	matches := []string{"bar", "baz"}
+	require.True(t, fileCountEquals(fc, len(matches)))
+}
+
+func TestNonRecursive(t *testing.T) {
+	fc := getNoFilterFileCount()
+	fc.Recursive = false
+	matches := []string{"foo", "bar", "baz", "qux", "subdir"}
+	require.True(t, fileCountEquals(fc, len(matches)))
+}
+
+func TestRegularOnlyFilter(t *testing.T) {
+	fc := getNoFilterFileCount()
+	fc.RegularOnly = true
+	matches := []string{
+		"foo", "bar", "baz", "qux", "subdir/quux", "subdir/quuz",
+	}
+	require.True(t, fileCountEquals(fc, len(matches)))
+}
+
+func TestSizeFilter(t *testing.T) {
+	fc := getNoFilterFileCount()
+	fc.Size = -100
+	matches := []string{"foo", "bar", "baz",
+		"subdir/quux", "subdir/quuz"}
+	require.True(t, fileCountEquals(fc, len(matches)))
+
+	fc.Size = 100
+	matches = []string{"qux"}
+	require.True(t, fileCountEquals(fc, len(matches)))
+}
+
+func TestMTimeFilter(t *testing.T) {
+	oldFile := filepath.Join(getTestdataDir(), "baz")
+	mtime := time.Date(1979, time.December, 14, 18, 25, 5, 0, time.UTC)
+	if err := os.Chtimes(oldFile, mtime, mtime); err != nil {
+		t.Skip("skipping mtime filter test.")
+	}
+	fileAge := time.Since(mtime) - (60 * time.Second)
+
+	fc := getNoFilterFileCount()
+	fc.MTime = internal.Duration{Duration: -fileAge}
+	matches := []string{"foo", "bar", "qux",
+		"subdir/", "subdir/quux", "subdir/quuz"}
+	require.True(t, fileCountEquals(fc, len(matches)))
+
+	fc.MTime = internal.Duration{Duration: fileAge}
+	matches = []string{"baz"}
+	require.True(t, fileCountEquals(fc, len(matches)))
+}
+
+func getNoFilterFileCount() FileCount {
+	return FileCount{
+		Directory:   getTestdataDir(),
+		Name:        "*",
+		Recursive:   true,
+		RegularOnly: false,
+		Size:        0,
+		MTime:       internal.Duration{Duration: 0},
+		fileFilters: nil,
+	}
+}
+
+func getTestdataDir() string {
+	_, filename, _, _ := runtime.Caller(1)
+	return strings.Replace(filename, "filecount_test.go", "testdata/", 1)
+}
+
+func fileCountEquals(fc FileCount, expectedCount int) bool {
+	tags := map[string]string{"directory": getTestdataDir()}
+	acc := testutil.Accumulator{}
+	acc.GatherError(fc.Gather)
+	return acc.HasPoint("filecount", tags, "count", int64(expectedCount))
+}

plugins/inputs/filecount/testdata/qux

@@ -0,0 +1,7 @@
+Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do
+eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad
+minim veniam, quis nostrud exercitation ullamco laboris nisi ut
+aliquip ex ea commodo consequat. Duis aute irure dolor in
+reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla
+pariatur. Excepteur sint occaecat cupidatat non proident, sunt in
+culpa qui officia deserunt mollit anim id est laborum.