From 030f94450562ab2d3823d298df5c6eeecbe3a649 Mon Sep 17 00:00:00 2001 From: Lee Jaeyong Date: Sat, 6 Oct 2018 04:55:23 +0900 Subject: [PATCH] Add per-directory file counts in the filecount input (#4752) --- plugins/inputs/filecount/filecount.go | 101 +++++++++++++++------ plugins/inputs/filecount/filecount_test.go | 95 +++++++++++++------ 2 files changed, 141 insertions(+), 55 deletions(-) diff --git a/plugins/inputs/filecount/filecount.go b/plugins/inputs/filecount/filecount.go index 6041ec7b5..a0dcd2cb4 100644 --- a/plugins/inputs/filecount/filecount.go +++ b/plugins/inputs/filecount/filecount.go @@ -1,19 +1,30 @@ package filecount import ( + "fmt" "os" "path/filepath" "time" "github.com/influxdata/telegraf" "github.com/influxdata/telegraf/internal" + "github.com/influxdata/telegraf/internal/globpath" "github.com/influxdata/telegraf/plugins/inputs" ) const sampleConfig = ` ## Directory to gather stats about. + ## deprecated in 1.9; use the directories option directory = "/var/cache/apt/archives" + ## Directories to gather stats about. + ## This accept standard unit glob matching rules, but with the addition of + ## ** as a "super asterisk". ie: + ## /var/log/** -> recursively find all directories in /var/log and count files in each directories + ## /var/log/*/* -> find all directories with a parent dir in /var/log and count files in each directories + ## /var/log -> count all files in /var/log and all of its subdirectories + directories = ["/var/cache/apt/archives"] + ## Only count files that match the name pattern. Defaults to "*". name = "*.deb" @@ -35,7 +46,8 @@ const sampleConfig = ` ` type FileCount struct { - Directory string + Directory string // deprecated in 1.9 + Directories []string Name string Recursive bool RegularOnly bool @@ -44,7 +56,6 @@ type FileCount struct { fileFilters []fileFilterFunc } -type countFunc func(os.FileInfo) type fileFilterFunc func(os.FileInfo) (bool, error) func (_ *FileCount) Description() string { @@ -125,18 +136,40 @@ func absDuration(x time.Duration) time.Duration { return x } -func count(basedir string, recursive bool, countFn countFunc) error { +func (fc *FileCount) count(acc telegraf.Accumulator, basedir string, recursive bool) { + numFiles := int64(0) walkFn := func(path string, file os.FileInfo, err error) error { if path == basedir { return nil } - countFn(file) + match, err := fc.filter(file) + if err != nil { + acc.AddError(err) + return nil + } + if match { + numFiles++ + } if !recursive && file.IsDir() { return filepath.SkipDir } return nil } - return filepath.Walk(basedir, walkFn) + + err := filepath.Walk(basedir, walkFn) + if err != nil { + acc.AddError(err) + return + } + + acc.AddFields("filecount", + map[string]interface{}{ + "count": numFiles, + }, + map[string]string{ + "directory": basedir, + }, + ) } func (fc *FileCount) initFileFilters() { @@ -168,37 +201,53 @@ func (fc *FileCount) filter(file os.FileInfo) (bool, error) { } func (fc *FileCount) Gather(acc telegraf.Accumulator) error { - numFiles := int64(0) - countFn := func(f os.FileInfo) { - match, err := fc.filter(f) - if err != nil { - acc.AddError(err) - return - } - if !match { - return - } - numFiles++ - } - err := count(fc.Directory, fc.Recursive, countFn) + globDirs := fc.getDirs() + dirs, err := getCompiledDirs(globDirs) if err != nil { - acc.AddError(err) + return err } - acc.AddFields("filecount", - map[string]interface{}{ - "count": numFiles, - }, - map[string]string{ - "directory": fc.Directory, - }) + for _, dir := range dirs { + fc.count(acc, dir, fc.Recursive) + } return nil } +func (fc *FileCount) getDirs() []string { + dirs := make([]string, len(fc.Directories)) + for i, dir := range fc.Directories { + dirs[i] = dir + } + + if fc.Directory != "" { + dirs = append(dirs, fc.Directory) + } + + return dirs +} + +func getCompiledDirs(dirs []string) ([]string, error) { + compiledDirs := []string{} + for _, dir := range dirs { + g, err := globpath.Compile(dir) + if err != nil { + return nil, fmt.Errorf("could not compile glob %v: %v", dir, err) + } + + for path, file := range g.Match() { + if file.IsDir() { + compiledDirs = append(compiledDirs, path) + } + } + } + return compiledDirs, nil +} + func NewFileCount() *FileCount { return &FileCount{ Directory: "", + Directories: []string{}, Name: "*", Recursive: true, RegularOnly: true, diff --git a/plugins/inputs/filecount/filecount_test.go b/plugins/inputs/filecount/filecount_test.go index 294a8b965..16bb83de5 100644 --- a/plugins/inputs/filecount/filecount_test.go +++ b/plugins/inputs/filecount/filecount_test.go @@ -14,69 +14,108 @@ import ( ) func TestNoFilters(t *testing.T) { - fc := getNoFilterFileCount() - matches := []string{"foo", "bar", "baz", "qux", - "subdir/", "subdir/quux", "subdir/quuz"} - require.True(t, fileCountEquals(fc, len(matches))) + fc := getNoFilterFileCount("*") + matches := []string{"foo", "bar", "baz", "qux", "subdir/", "subdir/quux", "subdir/quuz"} + + acc := testutil.Accumulator{} + acc.GatherError(fc.Gather) + + require.True(t, assertFileCount(&acc, "testdata", len(matches))) +} + +func TestNoFiltersOnChildDir(t *testing.T) { + fc := getNoFilterFileCount("testdata/*") + matches := []string{"subdir/quux", "subdir/quuz"} + + acc := testutil.Accumulator{} + acc.GatherError(fc.Gather) + + require.True(t, assertFileCount(&acc, "testdata/subdir", len(matches))) } func TestNameFilter(t *testing.T) { - fc := getNoFilterFileCount() + fc := getNoFilterFileCount("testdata") fc.Name = "ba*" matches := []string{"bar", "baz"} - require.True(t, fileCountEquals(fc, len(matches))) + + acc := testutil.Accumulator{} + acc.GatherError(fc.Gather) + + require.True(t, assertFileCount(&acc, "testdata", len(matches))) } func TestNonRecursive(t *testing.T) { - fc := getNoFilterFileCount() + fc := getNoFilterFileCount("testdata") fc.Recursive = false matches := []string{"foo", "bar", "baz", "qux", "subdir"} - require.True(t, fileCountEquals(fc, len(matches))) + + acc := testutil.Accumulator{} + acc.GatherError(fc.Gather) + + require.True(t, assertFileCount(&acc, "testdata", len(matches))) } func TestRegularOnlyFilter(t *testing.T) { - fc := getNoFilterFileCount() + fc := getNoFilterFileCount("testdata") fc.RegularOnly = true matches := []string{ "foo", "bar", "baz", "qux", "subdir/quux", "subdir/quuz", } - require.True(t, fileCountEquals(fc, len(matches))) + + acc := testutil.Accumulator{} + acc.GatherError(fc.Gather) + + require.True(t, assertFileCount(&acc, "testdata", len(matches))) } func TestSizeFilter(t *testing.T) { - fc := getNoFilterFileCount() + fc := getNoFilterFileCount("testdata") fc.Size = -100 - matches := []string{"foo", "bar", "baz", - "subdir/quux", "subdir/quuz"} - require.True(t, fileCountEquals(fc, len(matches))) + matches := []string{"foo", "bar", "baz", "subdir/quux", "subdir/quuz"} + + acc := testutil.Accumulator{} + acc.GatherError(fc.Gather) + + require.True(t, assertFileCount(&acc, "testdata", len(matches))) fc.Size = 100 matches = []string{"qux"} - require.True(t, fileCountEquals(fc, len(matches))) + + acc = testutil.Accumulator{} + acc.GatherError(fc.Gather) + + require.True(t, assertFileCount(&acc, "testdata", len(matches))) } func TestMTimeFilter(t *testing.T) { - oldFile := filepath.Join(getTestdataDir(), "baz") + oldFile := filepath.Join(getTestdataDir("testdata"), "baz") mtime := time.Date(1979, time.December, 14, 18, 25, 5, 0, time.UTC) if err := os.Chtimes(oldFile, mtime, mtime); err != nil { t.Skip("skipping mtime filter test.") } fileAge := time.Since(mtime) - (60 * time.Second) - fc := getNoFilterFileCount() + fc := getNoFilterFileCount("testdata") fc.MTime = internal.Duration{Duration: -fileAge} - matches := []string{"foo", "bar", "qux", - "subdir/", "subdir/quux", "subdir/quuz"} - require.True(t, fileCountEquals(fc, len(matches))) + matches := []string{"foo", "bar", "qux", "subdir/", "subdir/quux", "subdir/quuz"} + + acc := testutil.Accumulator{} + acc.GatherError(fc.Gather) + + require.True(t, assertFileCount(&acc, "testdata", len(matches))) fc.MTime = internal.Duration{Duration: fileAge} matches = []string{"baz"} - require.True(t, fileCountEquals(fc, len(matches))) + + acc = testutil.Accumulator{} + acc.GatherError(fc.Gather) + + require.True(t, assertFileCount(&acc, "testdata", len(matches))) } -func getNoFilterFileCount() FileCount { +func getNoFilterFileCount(dir string) FileCount { return FileCount{ - Directory: getTestdataDir(), + Directories: []string{getTestdataDir(dir)}, Name: "*", Recursive: true, RegularOnly: false, @@ -86,14 +125,12 @@ func getNoFilterFileCount() FileCount { } } -func getTestdataDir() string { +func getTestdataDir(dir string) string { _, filename, _, _ := runtime.Caller(1) - return strings.Replace(filename, "filecount_test.go", "testdata/", 1) + return strings.Replace(filename, "filecount_test.go", dir, 1) } -func fileCountEquals(fc FileCount, expectedCount int) bool { - tags := map[string]string{"directory": getTestdataDir()} - acc := testutil.Accumulator{} - acc.GatherError(fc.Gather) +func assertFileCount(acc *testutil.Accumulator, expectedDir string, expectedCount int) bool { + tags := map[string]string{"directory": getTestdataDir(expectedDir)} return acc.HasPoint("filecount", tags, "count", int64(expectedCount)) }