Add per-directory file counts in the filecount input (#4752)

plugins/inputs/filecount/filecount.go

@@ -1,19 +1,30 @@
 package filecount
 
 import (
+	"fmt"
 	"os"
 	"path/filepath"
 	"time"
 
 	"github.com/influxdata/telegraf"
 	"github.com/influxdata/telegraf/internal"
+	"github.com/influxdata/telegraf/internal/globpath"
 	"github.com/influxdata/telegraf/plugins/inputs"
 )
 
 const sampleConfig = `
   ## Directory to gather stats about.
+  ##   deprecated in 1.9; use the directories option
   directory = "/var/cache/apt/archives"
 
+  ## Directories to gather stats about.
+  ## This accept standard unit glob matching rules, but with the addition of
+  ## ** as a "super asterisk". ie:
+  ##   /var/log/**    -> recursively find all directories in /var/log and count files in each directories
+  ##   /var/log/*/*   -> find all directories with a parent dir in /var/log and count files in each directories
+  ##   /var/log       -> count all files in /var/log and all of its subdirectories
+  directories = ["/var/cache/apt/archives"]
+
   ## Only count files that match the name pattern. Defaults to "*".
   name = "*.deb"
 
@@ -35,7 +46,8 @@ const sampleConfig = `
 `
 
 type FileCount struct {
-	Directory   string
+	Directory   string // deprecated in 1.9
+	Directories []string
 	Name        string
 	Recursive   bool
 	RegularOnly bool
@@ -44,7 +56,6 @@ type FileCount struct {
 	fileFilters []fileFilterFunc
 }
 
-type countFunc func(os.FileInfo)
 type fileFilterFunc func(os.FileInfo) (bool, error)
 
 func (_ *FileCount) Description() string {
@@ -125,18 +136,40 @@ func absDuration(x time.Duration) time.Duration {
 	return x
 }
 
-func count(basedir string, recursive bool, countFn countFunc) error {
+func (fc *FileCount) count(acc telegraf.Accumulator, basedir string, recursive bool) {
+	numFiles := int64(0)
 	walkFn := func(path string, file os.FileInfo, err error) error {
 		if path == basedir {
 			return nil
 		}
-		countFn(file)
+		match, err := fc.filter(file)
+		if err != nil {
+			acc.AddError(err)
+			return nil
+		}
+		if match {
+			numFiles++
+		}
 		if !recursive && file.IsDir() {
 			return filepath.SkipDir
 		}
 		return nil
 	}
-	return filepath.Walk(basedir, walkFn)
+
+	err := filepath.Walk(basedir, walkFn)
+	if err != nil {
+		acc.AddError(err)
+		return
+	}
+
+	acc.AddFields("filecount",
+		map[string]interface{}{
+			"count": numFiles,
+		},
+		map[string]string{
+			"directory": basedir,
+		},
+	)
 }
 
 func (fc *FileCount) initFileFilters() {
@@ -168,37 +201,53 @@ func (fc *FileCount) filter(file os.FileInfo) (bool, error) {
 }
 
 func (fc *FileCount) Gather(acc telegraf.Accumulator) error {
-	numFiles := int64(0)
+	globDirs := fc.getDirs()
-	countFn := func(f os.FileInfo) {
+	dirs, err := getCompiledDirs(globDirs)
-		match, err := fc.filter(f)
-		if err != nil {
-			acc.AddError(err)
-			return
-		}
-		if !match {
-			return
-		}
-		numFiles++
-	}
-	err := count(fc.Directory, fc.Recursive, countFn)
 	if err != nil {
-		acc.AddError(err)
+		return err
 	}
 
-	acc.AddFields("filecount",
+	for _, dir := range dirs {
-		map[string]interface{}{
+		fc.count(acc, dir, fc.Recursive)
-			"count": numFiles,
+	}
-		},
-		map[string]string{
-			"directory": fc.Directory,
-		})
 
 	return nil
 }
 
+func (fc *FileCount) getDirs() []string {
+	dirs := make([]string, len(fc.Directories))
+	for i, dir := range fc.Directories {
+		dirs[i] = dir
+	}
+
+	if fc.Directory != "" {
+		dirs = append(dirs, fc.Directory)
+	}
+
+	return dirs
+}
+
+func getCompiledDirs(dirs []string) ([]string, error) {
+	compiledDirs := []string{}
+	for _, dir := range dirs {
+		g, err := globpath.Compile(dir)
+		if err != nil {
+			return nil, fmt.Errorf("could not compile glob %v: %v", dir, err)
+		}
+
+		for path, file := range g.Match() {
+			if file.IsDir() {
+				compiledDirs = append(compiledDirs, path)
+			}
+		}
+	}
+	return compiledDirs, nil
+}
+
 func NewFileCount() *FileCount {
 	return &FileCount{
 		Directory:   "",
+		Directories: []string{},
 		Name:        "*",
 		Recursive:   true,
 		RegularOnly: true,

plugins/inputs/filecount/filecount_test.go

@@ -14,69 +14,108 @@ import (
 )
 
 func TestNoFilters(t *testing.T) {
-	fc := getNoFilterFileCount()
+	fc := getNoFilterFileCount("*")
-	matches := []string{"foo", "bar", "baz", "qux",
+	matches := []string{"foo", "bar", "baz", "qux", "subdir/", "subdir/quux", "subdir/quuz"}
-		"subdir/", "subdir/quux", "subdir/quuz"}
+
-	require.True(t, fileCountEquals(fc, len(matches)))
+	acc := testutil.Accumulator{}
+	acc.GatherError(fc.Gather)
+
+	require.True(t, assertFileCount(&acc, "testdata", len(matches)))
+}
+
+func TestNoFiltersOnChildDir(t *testing.T) {
+	fc := getNoFilterFileCount("testdata/*")
+	matches := []string{"subdir/quux", "subdir/quuz"}
+
+	acc := testutil.Accumulator{}
+	acc.GatherError(fc.Gather)
+
+	require.True(t, assertFileCount(&acc, "testdata/subdir", len(matches)))
 }
 
 func TestNameFilter(t *testing.T) {
-	fc := getNoFilterFileCount()
+	fc := getNoFilterFileCount("testdata")
 	fc.Name = "ba*"
 	matches := []string{"bar", "baz"}
-	require.True(t, fileCountEquals(fc, len(matches)))
+
+	acc := testutil.Accumulator{}
+	acc.GatherError(fc.Gather)
+
+	require.True(t, assertFileCount(&acc, "testdata", len(matches)))
 }
 
 func TestNonRecursive(t *testing.T) {
-	fc := getNoFilterFileCount()
+	fc := getNoFilterFileCount("testdata")
 	fc.Recursive = false
 	matches := []string{"foo", "bar", "baz", "qux", "subdir"}
-	require.True(t, fileCountEquals(fc, len(matches)))
+
+	acc := testutil.Accumulator{}
+	acc.GatherError(fc.Gather)
+
+	require.True(t, assertFileCount(&acc, "testdata", len(matches)))
 }
 
 func TestRegularOnlyFilter(t *testing.T) {
-	fc := getNoFilterFileCount()
+	fc := getNoFilterFileCount("testdata")
 	fc.RegularOnly = true
 	matches := []string{
 		"foo", "bar", "baz", "qux", "subdir/quux", "subdir/quuz",
 	}
-	require.True(t, fileCountEquals(fc, len(matches)))
+
+	acc := testutil.Accumulator{}
+	acc.GatherError(fc.Gather)
+
+	require.True(t, assertFileCount(&acc, "testdata", len(matches)))
 }
 
 func TestSizeFilter(t *testing.T) {
-	fc := getNoFilterFileCount()
+	fc := getNoFilterFileCount("testdata")
 	fc.Size = -100
-	matches := []string{"foo", "bar", "baz",
+	matches := []string{"foo", "bar", "baz", "subdir/quux", "subdir/quuz"}
-		"subdir/quux", "subdir/quuz"}
+
-	require.True(t, fileCountEquals(fc, len(matches)))
+	acc := testutil.Accumulator{}
+	acc.GatherError(fc.Gather)
+
+	require.True(t, assertFileCount(&acc, "testdata", len(matches)))
 
 	fc.Size = 100
 	matches = []string{"qux"}
-	require.True(t, fileCountEquals(fc, len(matches)))
+
+	acc = testutil.Accumulator{}
+	acc.GatherError(fc.Gather)
+
+	require.True(t, assertFileCount(&acc, "testdata", len(matches)))
 }
 
 func TestMTimeFilter(t *testing.T) {
-	oldFile := filepath.Join(getTestdataDir(), "baz")
+	oldFile := filepath.Join(getTestdataDir("testdata"), "baz")
 	mtime := time.Date(1979, time.December, 14, 18, 25, 5, 0, time.UTC)
 	if err := os.Chtimes(oldFile, mtime, mtime); err != nil {
 		t.Skip("skipping mtime filter test.")
 	}
 	fileAge := time.Since(mtime) - (60 * time.Second)
 
-	fc := getNoFilterFileCount()
+	fc := getNoFilterFileCount("testdata")
 	fc.MTime = internal.Duration{Duration: -fileAge}
-	matches := []string{"foo", "bar", "qux",
+	matches := []string{"foo", "bar", "qux", "subdir/", "subdir/quux", "subdir/quuz"}
-		"subdir/", "subdir/quux", "subdir/quuz"}
+
-	require.True(t, fileCountEquals(fc, len(matches)))
+	acc := testutil.Accumulator{}
+	acc.GatherError(fc.Gather)
+
+	require.True(t, assertFileCount(&acc, "testdata", len(matches)))
 
 	fc.MTime = internal.Duration{Duration: fileAge}
 	matches = []string{"baz"}
-	require.True(t, fileCountEquals(fc, len(matches)))
+
+	acc = testutil.Accumulator{}
+	acc.GatherError(fc.Gather)
+
+	require.True(t, assertFileCount(&acc, "testdata", len(matches)))
 }
 
-func getNoFilterFileCount() FileCount {
+func getNoFilterFileCount(dir string) FileCount {
 	return FileCount{
-		Directory:   getTestdataDir(),
+		Directories: []string{getTestdataDir(dir)},
 		Name:        "*",
 		Recursive:   true,
 		RegularOnly: false,
@@ -86,14 +125,12 @@ func getNoFilterFileCount() FileCount {
 	}
 }
 
-func getTestdataDir() string {
+func getTestdataDir(dir string) string {
 	_, filename, _, _ := runtime.Caller(1)
-	return strings.Replace(filename, "filecount_test.go", "testdata/", 1)
+	return strings.Replace(filename, "filecount_test.go", dir, 1)
 }
 
-func fileCountEquals(fc FileCount, expectedCount int) bool {
+func assertFileCount(acc *testutil.Accumulator, expectedDir string, expectedCount int) bool {
-	tags := map[string]string{"directory": getTestdataDir()}
+	tags := map[string]string{"directory": getTestdataDir(expectedDir)}
-	acc := testutil.Accumulator{}
-	acc.GatherError(fc.Gather)
 	return acc.HasPoint("filecount", tags, "count", int64(expectedCount))
 }