dom
/
telegraf
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Add per-directory file counts in the filecount input (#4752)

This commit is contained in:
Lee Jaeyong 2018-10-06 04:55:23 +09:00 committed by Daniel Nelson
parent f81696b6b5
commit 030f944505
2 changed files with 141 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

101
plugins/inputs/filecount/filecount.go
View File

@ -1,19 +1,30 @@
package filecount
import (
"fmt"
"os"
"path/filepath"
"time"
"github.com/influxdata/telegraf"
"github.com/influxdata/telegraf/internal"
"github.com/influxdata/telegraf/internal/globpath"
"github.com/influxdata/telegraf/plugins/inputs"
)
const sampleConfig = `
## Directory to gather stats about.
## deprecated in 1.9; use the directories option
directory = "/var/cache/apt/archives"
## Directories to gather stats about.
## This accept standard unit glob matching rules, but with the addition of
## ** as a "super asterisk". ie:
## /var/log/** -> recursively find all directories in /var/log and count files in each directories
## /var/log/*/* -> find all directories with a parent dir in /var/log and count files in each directories
## /var/log -> count all files in /var/log and all of its subdirectories
directories = ["/var/cache/apt/archives"]
## Only count files that match the name pattern. Defaults to "*".
name = "*.deb"
@ -35,7 +46,8 @@ const sampleConfig = `
`
type FileCount struct {
Directory string
Directory string // deprecated in 1.9
Directories []string
Name string
Recursive bool
RegularOnly bool
@ -44,7 +56,6 @@ type FileCount struct {
fileFilters []fileFilterFunc
}
type countFunc func(os.FileInfo)
type fileFilterFunc func(os.FileInfo) (bool, error)
func (_ *FileCount) Description() string {
@ -125,18 +136,40 @@ func absDuration(x time.Duration) time.Duration {
return x
}
func count(basedir string, recursive bool, countFn countFunc) error {
func (fc *FileCount) count(acc telegraf.Accumulator, basedir string, recursive bool) {
numFiles := int64(0)
walkFn := func(path string, file os.FileInfo, err error) error {
if path == basedir {
return nil
}
countFn(file)
match, err := fc.filter(file)
if err != nil {
acc.AddError(err)
return nil
}
if match {
numFiles++
}
if !recursive && file.IsDir() {
return filepath.SkipDir
}
return nil
}
return filepath.Walk(basedir, walkFn)
err := filepath.Walk(basedir, walkFn)
if err != nil {
acc.AddError(err)
return
}
acc.AddFields("filecount",
map[string]interface{}{
"count": numFiles,
},
map[string]string{
"directory": basedir,
},
)
}
func (fc *FileCount) initFileFilters() {
@ -168,37 +201,53 @@ func (fc *FileCount) filter(file os.FileInfo) (bool, error) {
}
func (fc *FileCount) Gather(acc telegraf.Accumulator) error {
numFiles := int64(0)
countFn := func(f os.FileInfo) {
match, err := fc.filter(f)
globDirs := fc.getDirs()
dirs, err := getCompiledDirs(globDirs)
if err != nil {
acc.AddError(err)
return
}
if !match {
return
}
numFiles++
}
err := count(fc.Directory, fc.Recursive, countFn)
if err != nil {
acc.AddError(err)
return err
}
acc.AddFields("filecount",
map[string]interface{}{
"count": numFiles,
},
map[string]string{
"directory": fc.Directory,
})
for _, dir := range dirs {
fc.count(acc, dir, fc.Recursive)
}
return nil
}
func (fc *FileCount) getDirs() []string {
dirs := make([]string, len(fc.Directories))
for i, dir := range fc.Directories {
dirs[i] = dir
}
if fc.Directory != "" {
dirs = append(dirs, fc.Directory)
}
return dirs
}
func getCompiledDirs(dirs []string) ([]string, error) {
compiledDirs := []string{}
for _, dir := range dirs {
g, err := globpath.Compile(dir)
if err != nil {
return nil, fmt.Errorf("could not compile glob %v: %v", dir, err)
}
for path, file := range g.Match() {
if file.IsDir() {
compiledDirs = append(compiledDirs, path)
}
}
}
return compiledDirs, nil
}
func NewFileCount() *FileCount {
return &FileCount{
Directory: "",
Directories: []string{},
Name: "*",
Recursive: true,
RegularOnly: true,

95
plugins/inputs/filecount/filecount_test.go
View File

@ -14,69 +14,108 @@ import (
)
func TestNoFilters(t *testing.T) {
fc := getNoFilterFileCount()
matches := []string{"foo", "bar", "baz", "qux",
"subdir/", "subdir/quux", "subdir/quuz"}
require.True(t, fileCountEquals(fc, len(matches)))
fc := getNoFilterFileCount("*")
matches := []string{"foo", "bar", "baz", "qux", "subdir/", "subdir/quux", "subdir/quuz"}
acc := testutil.Accumulator{}
acc.GatherError(fc.Gather)
require.True(t, assertFileCount(&acc, "testdata", len(matches)))
}
func TestNoFiltersOnChildDir(t *testing.T) {
fc := getNoFilterFileCount("testdata/*")
matches := []string{"subdir/quux", "subdir/quuz"}
acc := testutil.Accumulator{}
acc.GatherError(fc.Gather)
require.True(t, assertFileCount(&acc, "testdata/subdir", len(matches)))
}
func TestNameFilter(t *testing.T) {
fc := getNoFilterFileCount()
fc := getNoFilterFileCount("testdata")
fc.Name = "ba*"
matches := []string{"bar", "baz"}
require.True(t, fileCountEquals(fc, len(matches)))
acc := testutil.Accumulator{}
acc.GatherError(fc.Gather)
require.True(t, assertFileCount(&acc, "testdata", len(matches)))
}
func TestNonRecursive(t *testing.T) {
fc := getNoFilterFileCount()
fc := getNoFilterFileCount("testdata")
fc.Recursive = false
matches := []string{"foo", "bar", "baz", "qux", "subdir"}
require.True(t, fileCountEquals(fc, len(matches)))
acc := testutil.Accumulator{}
acc.GatherError(fc.Gather)
require.True(t, assertFileCount(&acc, "testdata", len(matches)))
}
func TestRegularOnlyFilter(t *testing.T) {
fc := getNoFilterFileCount()
fc := getNoFilterFileCount("testdata")
fc.RegularOnly = true
matches := []string{
"foo", "bar", "baz", "qux", "subdir/quux", "subdir/quuz",
}
require.True(t, fileCountEquals(fc, len(matches)))
acc := testutil.Accumulator{}
acc.GatherError(fc.Gather)
require.True(t, assertFileCount(&acc, "testdata", len(matches)))
}
func TestSizeFilter(t *testing.T) {
fc := getNoFilterFileCount()
fc := getNoFilterFileCount("testdata")
fc.Size = -100
matches := []string{"foo", "bar", "baz",
"subdir/quux", "subdir/quuz"}
require.True(t, fileCountEquals(fc, len(matches)))
matches := []string{"foo", "bar", "baz", "subdir/quux", "subdir/quuz"}
acc := testutil.Accumulator{}
acc.GatherError(fc.Gather)
require.True(t, assertFileCount(&acc, "testdata", len(matches)))
fc.Size = 100
matches = []string{"qux"}
require.True(t, fileCountEquals(fc, len(matches)))
acc = testutil.Accumulator{}
acc.GatherError(fc.Gather)
require.True(t, assertFileCount(&acc, "testdata", len(matches)))
}
func TestMTimeFilter(t *testing.T) {
oldFile := filepath.Join(getTestdataDir(), "baz")
oldFile := filepath.Join(getTestdataDir("testdata"), "baz")
mtime := time.Date(1979, time.December, 14, 18, 25, 5, 0, time.UTC)
if err := os.Chtimes(oldFile, mtime, mtime); err != nil {
t.Skip("skipping mtime filter test.")
}
fileAge := time.Since(mtime) - (60 * time.Second)
fc := getNoFilterFileCount()
fc := getNoFilterFileCount("testdata")
fc.MTime = internal.Duration{Duration: -fileAge}
matches := []string{"foo", "bar", "qux",
"subdir/", "subdir/quux", "subdir/quuz"}
require.True(t, fileCountEquals(fc, len(matches)))
matches := []string{"foo", "bar", "qux", "subdir/", "subdir/quux", "subdir/quuz"}
acc := testutil.Accumulator{}
acc.GatherError(fc.Gather)
require.True(t, assertFileCount(&acc, "testdata", len(matches)))
fc.MTime = internal.Duration{Duration: fileAge}
matches = []string{"baz"}
require.True(t, fileCountEquals(fc, len(matches)))
acc = testutil.Accumulator{}
acc.GatherError(fc.Gather)
require.True(t, assertFileCount(&acc, "testdata", len(matches)))
}
func getNoFilterFileCount() FileCount {
func getNoFilterFileCount(dir string) FileCount {
return FileCount{
Directory: getTestdataDir(),
Directories: []string{getTestdataDir(dir)},
Name: "*",
Recursive: true,
RegularOnly: false,
@ -86,14 +125,12 @@ func getNoFilterFileCount() FileCount {
}
}
func getTestdataDir() string {
func getTestdataDir(dir string) string {
_, filename, _, _ := runtime.Caller(1)
return strings.Replace(filename, "filecount_test.go", "testdata/", 1)
return strings.Replace(filename, "filecount_test.go", dir, 1)
}
func fileCountEquals(fc FileCount, expectedCount int) bool {
tags := map[string]string{"directory": getTestdataDir()}
acc := testutil.Accumulator{}
acc.GatherError(fc.Gather)
func assertFileCount(acc *testutil.Accumulator, expectedDir string, expectedCount int) bool {
tags := map[string]string{"directory": getTestdataDir(expectedDir)}
return acc.HasPoint("filecount", tags, "count", int64(expectedCount))
}