Add per-directory file counts in the filecount input (#4752)

This commit is contained in:
Lee Jaeyong 2018-10-06 04:55:23 +09:00 committed by Daniel Nelson
parent f81696b6b5
commit 030f944505
2 changed files with 141 additions and 55 deletions

View File

@ -1,19 +1,30 @@
package filecount package filecount
import ( import (
"fmt"
"os" "os"
"path/filepath" "path/filepath"
"time" "time"
"github.com/influxdata/telegraf" "github.com/influxdata/telegraf"
"github.com/influxdata/telegraf/internal" "github.com/influxdata/telegraf/internal"
"github.com/influxdata/telegraf/internal/globpath"
"github.com/influxdata/telegraf/plugins/inputs" "github.com/influxdata/telegraf/plugins/inputs"
) )
const sampleConfig = ` const sampleConfig = `
## Directory to gather stats about. ## Directory to gather stats about.
## deprecated in 1.9; use the directories option
directory = "/var/cache/apt/archives" directory = "/var/cache/apt/archives"
## Directories to gather stats about.
## This accept standard unit glob matching rules, but with the addition of
## ** as a "super asterisk". ie:
## /var/log/** -> recursively find all directories in /var/log and count files in each directories
## /var/log/*/* -> find all directories with a parent dir in /var/log and count files in each directories
## /var/log -> count all files in /var/log and all of its subdirectories
directories = ["/var/cache/apt/archives"]
## Only count files that match the name pattern. Defaults to "*". ## Only count files that match the name pattern. Defaults to "*".
name = "*.deb" name = "*.deb"
@ -35,7 +46,8 @@ const sampleConfig = `
` `
type FileCount struct { type FileCount struct {
Directory string Directory string // deprecated in 1.9
Directories []string
Name string Name string
Recursive bool Recursive bool
RegularOnly bool RegularOnly bool
@ -44,7 +56,6 @@ type FileCount struct {
fileFilters []fileFilterFunc fileFilters []fileFilterFunc
} }
type countFunc func(os.FileInfo)
type fileFilterFunc func(os.FileInfo) (bool, error) type fileFilterFunc func(os.FileInfo) (bool, error)
func (_ *FileCount) Description() string { func (_ *FileCount) Description() string {
@ -125,18 +136,40 @@ func absDuration(x time.Duration) time.Duration {
return x return x
} }
func count(basedir string, recursive bool, countFn countFunc) error { func (fc *FileCount) count(acc telegraf.Accumulator, basedir string, recursive bool) {
numFiles := int64(0)
walkFn := func(path string, file os.FileInfo, err error) error { walkFn := func(path string, file os.FileInfo, err error) error {
if path == basedir { if path == basedir {
return nil return nil
} }
countFn(file) match, err := fc.filter(file)
if err != nil {
acc.AddError(err)
return nil
}
if match {
numFiles++
}
if !recursive && file.IsDir() { if !recursive && file.IsDir() {
return filepath.SkipDir return filepath.SkipDir
} }
return nil return nil
} }
return filepath.Walk(basedir, walkFn)
err := filepath.Walk(basedir, walkFn)
if err != nil {
acc.AddError(err)
return
}
acc.AddFields("filecount",
map[string]interface{}{
"count": numFiles,
},
map[string]string{
"directory": basedir,
},
)
} }
func (fc *FileCount) initFileFilters() { func (fc *FileCount) initFileFilters() {
@ -168,37 +201,53 @@ func (fc *FileCount) filter(file os.FileInfo) (bool, error) {
} }
func (fc *FileCount) Gather(acc telegraf.Accumulator) error { func (fc *FileCount) Gather(acc telegraf.Accumulator) error {
numFiles := int64(0) globDirs := fc.getDirs()
countFn := func(f os.FileInfo) { dirs, err := getCompiledDirs(globDirs)
match, err := fc.filter(f)
if err != nil {
acc.AddError(err)
return
}
if !match {
return
}
numFiles++
}
err := count(fc.Directory, fc.Recursive, countFn)
if err != nil { if err != nil {
acc.AddError(err) return err
} }
acc.AddFields("filecount", for _, dir := range dirs {
map[string]interface{}{ fc.count(acc, dir, fc.Recursive)
"count": numFiles, }
},
map[string]string{
"directory": fc.Directory,
})
return nil return nil
} }
func (fc *FileCount) getDirs() []string {
dirs := make([]string, len(fc.Directories))
for i, dir := range fc.Directories {
dirs[i] = dir
}
if fc.Directory != "" {
dirs = append(dirs, fc.Directory)
}
return dirs
}
func getCompiledDirs(dirs []string) ([]string, error) {
compiledDirs := []string{}
for _, dir := range dirs {
g, err := globpath.Compile(dir)
if err != nil {
return nil, fmt.Errorf("could not compile glob %v: %v", dir, err)
}
for path, file := range g.Match() {
if file.IsDir() {
compiledDirs = append(compiledDirs, path)
}
}
}
return compiledDirs, nil
}
func NewFileCount() *FileCount { func NewFileCount() *FileCount {
return &FileCount{ return &FileCount{
Directory: "", Directory: "",
Directories: []string{},
Name: "*", Name: "*",
Recursive: true, Recursive: true,
RegularOnly: true, RegularOnly: true,

View File

@ -14,69 +14,108 @@ import (
) )
func TestNoFilters(t *testing.T) { func TestNoFilters(t *testing.T) {
fc := getNoFilterFileCount() fc := getNoFilterFileCount("*")
matches := []string{"foo", "bar", "baz", "qux", matches := []string{"foo", "bar", "baz", "qux", "subdir/", "subdir/quux", "subdir/quuz"}
"subdir/", "subdir/quux", "subdir/quuz"}
require.True(t, fileCountEquals(fc, len(matches))) acc := testutil.Accumulator{}
acc.GatherError(fc.Gather)
require.True(t, assertFileCount(&acc, "testdata", len(matches)))
}
func TestNoFiltersOnChildDir(t *testing.T) {
fc := getNoFilterFileCount("testdata/*")
matches := []string{"subdir/quux", "subdir/quuz"}
acc := testutil.Accumulator{}
acc.GatherError(fc.Gather)
require.True(t, assertFileCount(&acc, "testdata/subdir", len(matches)))
} }
func TestNameFilter(t *testing.T) { func TestNameFilter(t *testing.T) {
fc := getNoFilterFileCount() fc := getNoFilterFileCount("testdata")
fc.Name = "ba*" fc.Name = "ba*"
matches := []string{"bar", "baz"} matches := []string{"bar", "baz"}
require.True(t, fileCountEquals(fc, len(matches)))
acc := testutil.Accumulator{}
acc.GatherError(fc.Gather)
require.True(t, assertFileCount(&acc, "testdata", len(matches)))
} }
func TestNonRecursive(t *testing.T) { func TestNonRecursive(t *testing.T) {
fc := getNoFilterFileCount() fc := getNoFilterFileCount("testdata")
fc.Recursive = false fc.Recursive = false
matches := []string{"foo", "bar", "baz", "qux", "subdir"} matches := []string{"foo", "bar", "baz", "qux", "subdir"}
require.True(t, fileCountEquals(fc, len(matches)))
acc := testutil.Accumulator{}
acc.GatherError(fc.Gather)
require.True(t, assertFileCount(&acc, "testdata", len(matches)))
} }
func TestRegularOnlyFilter(t *testing.T) { func TestRegularOnlyFilter(t *testing.T) {
fc := getNoFilterFileCount() fc := getNoFilterFileCount("testdata")
fc.RegularOnly = true fc.RegularOnly = true
matches := []string{ matches := []string{
"foo", "bar", "baz", "qux", "subdir/quux", "subdir/quuz", "foo", "bar", "baz", "qux", "subdir/quux", "subdir/quuz",
} }
require.True(t, fileCountEquals(fc, len(matches)))
acc := testutil.Accumulator{}
acc.GatherError(fc.Gather)
require.True(t, assertFileCount(&acc, "testdata", len(matches)))
} }
func TestSizeFilter(t *testing.T) { func TestSizeFilter(t *testing.T) {
fc := getNoFilterFileCount() fc := getNoFilterFileCount("testdata")
fc.Size = -100 fc.Size = -100
matches := []string{"foo", "bar", "baz", matches := []string{"foo", "bar", "baz", "subdir/quux", "subdir/quuz"}
"subdir/quux", "subdir/quuz"}
require.True(t, fileCountEquals(fc, len(matches))) acc := testutil.Accumulator{}
acc.GatherError(fc.Gather)
require.True(t, assertFileCount(&acc, "testdata", len(matches)))
fc.Size = 100 fc.Size = 100
matches = []string{"qux"} matches = []string{"qux"}
require.True(t, fileCountEquals(fc, len(matches)))
acc = testutil.Accumulator{}
acc.GatherError(fc.Gather)
require.True(t, assertFileCount(&acc, "testdata", len(matches)))
} }
func TestMTimeFilter(t *testing.T) { func TestMTimeFilter(t *testing.T) {
oldFile := filepath.Join(getTestdataDir(), "baz") oldFile := filepath.Join(getTestdataDir("testdata"), "baz")
mtime := time.Date(1979, time.December, 14, 18, 25, 5, 0, time.UTC) mtime := time.Date(1979, time.December, 14, 18, 25, 5, 0, time.UTC)
if err := os.Chtimes(oldFile, mtime, mtime); err != nil { if err := os.Chtimes(oldFile, mtime, mtime); err != nil {
t.Skip("skipping mtime filter test.") t.Skip("skipping mtime filter test.")
} }
fileAge := time.Since(mtime) - (60 * time.Second) fileAge := time.Since(mtime) - (60 * time.Second)
fc := getNoFilterFileCount() fc := getNoFilterFileCount("testdata")
fc.MTime = internal.Duration{Duration: -fileAge} fc.MTime = internal.Duration{Duration: -fileAge}
matches := []string{"foo", "bar", "qux", matches := []string{"foo", "bar", "qux", "subdir/", "subdir/quux", "subdir/quuz"}
"subdir/", "subdir/quux", "subdir/quuz"}
require.True(t, fileCountEquals(fc, len(matches))) acc := testutil.Accumulator{}
acc.GatherError(fc.Gather)
require.True(t, assertFileCount(&acc, "testdata", len(matches)))
fc.MTime = internal.Duration{Duration: fileAge} fc.MTime = internal.Duration{Duration: fileAge}
matches = []string{"baz"} matches = []string{"baz"}
require.True(t, fileCountEquals(fc, len(matches)))
acc = testutil.Accumulator{}
acc.GatherError(fc.Gather)
require.True(t, assertFileCount(&acc, "testdata", len(matches)))
} }
func getNoFilterFileCount() FileCount { func getNoFilterFileCount(dir string) FileCount {
return FileCount{ return FileCount{
Directory: getTestdataDir(), Directories: []string{getTestdataDir(dir)},
Name: "*", Name: "*",
Recursive: true, Recursive: true,
RegularOnly: false, RegularOnly: false,
@ -86,14 +125,12 @@ func getNoFilterFileCount() FileCount {
} }
} }
func getTestdataDir() string { func getTestdataDir(dir string) string {
_, filename, _, _ := runtime.Caller(1) _, filename, _, _ := runtime.Caller(1)
return strings.Replace(filename, "filecount_test.go", "testdata/", 1) return strings.Replace(filename, "filecount_test.go", dir, 1)
} }
func fileCountEquals(fc FileCount, expectedCount int) bool { func assertFileCount(acc *testutil.Accumulator, expectedDir string, expectedCount int) bool {
tags := map[string]string{"directory": getTestdataDir()} tags := map[string]string{"directory": getTestdataDir(expectedDir)}
acc := testutil.Accumulator{}
acc.GatherError(fc.Gather)
return acc.HasPoint("filecount", tags, "count", int64(expectedCount)) return acc.HasPoint("filecount", tags, "count", int64(expectedCount))
} }