diff --git a/plugins/inputs/filecount/filecount.go b/plugins/inputs/filecount/filecount.go index c0072e0d8..929ec66a7 100644 --- a/plugins/inputs/filecount/filecount.go +++ b/plugins/inputs/filecount/filecount.go @@ -58,6 +58,7 @@ type FileCount struct { MTime internal.Duration `toml:"mtime"` fileFilters []fileFilterFunc globPaths []globpath.GlobPath + Fs fileSystem } func (_ *FileCount) Description() string { @@ -159,7 +160,7 @@ func (fc *FileCount) count(acc telegraf.Accumulator, basedir string, glob globpa if err == nil && rel == "." { return nil } - file, err := os.Stat(path) + file, err := fc.Fs.Stat(path) if err != nil { if os.IsNotExist(err) { return nil @@ -244,7 +245,7 @@ func (fc *FileCount) Gather(acc telegraf.Accumulator) error { } for _, glob := range fc.globPaths { - for _, dir := range onlyDirectories(glob.GetRoots()) { + for _, dir := range fc.onlyDirectories(glob.GetRoots()) { fc.count(acc, dir, glob) } } @@ -252,10 +253,10 @@ func (fc *FileCount) Gather(acc telegraf.Accumulator) error { return nil } -func onlyDirectories(directories []string) []string { +func (fc *FileCount) onlyDirectories(directories []string) []string { out := make([]string, 0) for _, path := range directories { - info, err := os.Stat(path) + info, err := fc.Fs.Stat(path) if err == nil && info.IsDir() { out = append(out, path) } @@ -286,6 +287,7 @@ func (fc *FileCount) initGlobPaths(acc telegraf.Accumulator) { fc.globPaths = append(fc.globPaths, *glob) } } + } func NewFileCount() *FileCount { @@ -298,6 +300,7 @@ func NewFileCount() *FileCount { Size: internal.Size{Size: 0}, MTime: internal.Duration{Duration: 0}, fileFilters: nil, + Fs: osFS{}, } } diff --git a/plugins/inputs/filecount/filecount_test.go b/plugins/inputs/filecount/filecount_test.go index 2294e8ce6..99213104b 100644 --- a/plugins/inputs/filecount/filecount_test.go +++ b/plugins/inputs/filecount/filecount_test.go @@ -2,7 +2,6 @@ package filecount import ( "os" - "path/filepath" "runtime" "strings" "testing" @@ -18,7 +17,7 @@ func TestNoFilters(t *testing.T) { matches := []string{"foo", "bar", "baz", "qux", "subdir/", "subdir/quux", "subdir/quuz", "subdir/nested2", "subdir/nested2/qux"} - fileCountEquals(t, fc, len(matches), 9084) + fileCountEquals(t, fc, len(matches), 5096) } func TestNoFiltersOnChildDir(t *testing.T) { @@ -30,9 +29,8 @@ func TestNoFiltersOnChildDir(t *testing.T) { tags := map[string]string{"directory": getTestdataDir() + "/subdir"} acc := testutil.Accumulator{} acc.GatherError(fc.Gather) - require.True(t, acc.HasPoint("filecount", tags, "count", int64(len(matches)))) - require.True(t, acc.HasPoint("filecount", tags, "size_bytes", int64(4542))) + require.True(t, acc.HasPoint("filecount", tags, "size_bytes", int64(600))) } func TestNoRecursiveButSuperMeta(t *testing.T) { @@ -46,7 +44,7 @@ func TestNoRecursiveButSuperMeta(t *testing.T) { acc.GatherError(fc.Gather) require.True(t, acc.HasPoint("filecount", tags, "count", int64(len(matches)))) - require.True(t, acc.HasPoint("filecount", tags, "size_bytes", int64(4096))) + require.True(t, acc.HasPoint("filecount", tags, "size_bytes", int64(200))) } func TestNameFilter(t *testing.T) { @@ -60,20 +58,22 @@ func TestNonRecursive(t *testing.T) { fc := getNoFilterFileCount() fc.Recursive = false matches := []string{"foo", "bar", "baz", "qux", "subdir"} - fileCountEquals(t, fc, len(matches), 4542) + + fileCountEquals(t, fc, len(matches), 4496) } func TestDoubleAndSimpleStar(t *testing.T) { fc := getNoFilterFileCount() fc.Directories = []string{getTestdataDir() + "/**/*"} matches := []string{"qux"} + tags := map[string]string{"directory": getTestdataDir() + "/subdir/nested2"} acc := testutil.Accumulator{} acc.GatherError(fc.Gather) require.True(t, acc.HasPoint("filecount", tags, "count", int64(len(matches)))) - require.True(t, acc.HasPoint("filecount", tags, "size_bytes", int64(446))) + require.True(t, acc.HasPoint("filecount", tags, "size_bytes", int64(400))) } func TestRegularOnlyFilter(t *testing.T) { @@ -82,7 +82,8 @@ func TestRegularOnlyFilter(t *testing.T) { matches := []string{ "foo", "bar", "baz", "qux", "subdir/quux", "subdir/quuz", "subdir/nested2/qux"} - fileCountEquals(t, fc, len(matches), 892) + + fileCountEquals(t, fc, len(matches), 800) } func TestSizeFilter(t *testing.T) { @@ -94,23 +95,22 @@ func TestSizeFilter(t *testing.T) { fc.Size = internal.Size{Size: 100} matches = []string{"qux", "subdir/nested2//qux"} - fileCountEquals(t, fc, len(matches), 892) + + fileCountEquals(t, fc, len(matches), 800) } func TestMTimeFilter(t *testing.T) { - oldFile := filepath.Join(getTestdataDir(), "baz") - mtime := time.Date(1979, time.December, 14, 18, 25, 5, 0, time.UTC) - if err := os.Chtimes(oldFile, mtime, mtime); err != nil { - t.Skip("skipping mtime filter test.") - } + + mtime := time.Date(2011, time.December, 14, 18, 25, 5, 0, time.UTC) fileAge := time.Since(mtime) - (60 * time.Second) fc := getNoFilterFileCount() fc.MTime = internal.Duration{Duration: -fileAge} matches := []string{"foo", "bar", "qux", "subdir/", "subdir/quux", "subdir/quuz", - "sbudir/nested2", "subdir/nested2/qux"} - fileCountEquals(t, fc, len(matches), 9084) + "subdir/nested2", "subdir/nested2/qux"} + + fileCountEquals(t, fc, len(matches), 5096) fc.MTime = internal.Duration{Duration: fileAge} matches = []string{"baz"} @@ -126,12 +126,60 @@ func getNoFilterFileCount() FileCount { Size: internal.Size{Size: 0}, MTime: internal.Duration{Duration: 0}, fileFilters: nil, + Fs: getFakeFileSystem(getTestdataDir()), } } func getTestdataDir() string { - _, filename, _, _ := runtime.Caller(1) - return strings.Replace(filename, "filecount_test.go", "testdata", 1) + dir, err := os.Getwd() + if err != nil { + // if we cannot even establish the test directory, further progress is meaningless + panic(err) + } + + var chunks []string + var testDirectory string + + if runtime.GOOS == "windows" { + chunks = strings.Split(dir, "\\") + testDirectory = strings.Join(chunks[:], "\\") + "\\testdata" + } else { + chunks = strings.Split(dir, "/") + testDirectory = strings.Join(chunks[:], "/") + "/testdata" + } + return testDirectory +} + +func getFakeFileSystem(basePath string) fakeFileSystem { + // create our desired "filesystem" object, complete with an internal map allowing our funcs to return meta data as requested + + mtime := time.Date(2015, time.December, 14, 18, 25, 5, 0, time.UTC) + olderMtime := time.Date(2010, time.December, 14, 18, 25, 5, 0, time.UTC) + + // set file permisions + var fmask uint32 = 0666 + var dmask uint32 = 0666 + + // set directory bit + dmask |= (1 << uint(32-1)) + + // create a lookup map for getting "files" from the "filesystem" + fileList := map[string]fakeFileInfo{ + basePath: {name: "testdata", size: int64(4096), filemode: uint32(dmask), modtime: mtime, isdir: true}, + basePath + "/foo": {name: "foo", filemode: uint32(fmask), modtime: mtime}, + basePath + "/bar": {name: "bar", filemode: uint32(fmask), modtime: mtime}, + basePath + "/baz": {name: "baz", filemode: uint32(fmask), modtime: olderMtime}, + basePath + "/qux": {name: "qux", size: int64(400), filemode: uint32(fmask), modtime: mtime}, + basePath + "/subdir": {name: "subdir", size: int64(4096), filemode: uint32(dmask), modtime: mtime, isdir: true}, + basePath + "/subdir/quux": {name: "quux", filemode: uint32(fmask), modtime: mtime}, + basePath + "/subdir/quuz": {name: "quuz", filemode: uint32(fmask), modtime: mtime}, + basePath + "/subdir/nested2": {name: "nested2", size: int64(200), filemode: uint32(dmask), modtime: mtime, isdir: true}, + basePath + "/subdir/nested2/qux": {name: "qux", filemode: uint32(fmask), modtime: mtime, size: int64(400)}, + } + + fs := fakeFileSystem{files: fileList} + return fs + } func fileCountEquals(t *testing.T, fc FileCount, expectedCount int, expectedSize int) { diff --git a/plugins/inputs/filecount/filesystem_helpers.go b/plugins/inputs/filecount/filesystem_helpers.go new file mode 100644 index 000000000..2bd6c0951 --- /dev/null +++ b/plugins/inputs/filecount/filesystem_helpers.go @@ -0,0 +1,73 @@ +package filecount + +import ( + "errors" + "io" + "os" + "time" +) + +/* + The code below is lifted from numerous articles and originates from Andrew Gerrand's 10 things you (probably) don't know about Go. + it allows for mocking a filesystem; this allows for consistent testing of this code across platforms (directory sizes reported + differently by different platforms, for example), while preserving the rest of the functionality as-is, without modification. +*/ + +type fileSystem interface { + Open(name string) (file, error) + Stat(name string) (os.FileInfo, error) +} + +type file interface { + io.Closer + io.Reader + io.ReaderAt + io.Seeker + Stat() (os.FileInfo, error) +} + +// osFS implements fileSystem using the local disk +type osFS struct{} + +func (osFS) Open(name string) (file, error) { return os.Open(name) } +func (osFS) Stat(name string) (os.FileInfo, error) { return os.Stat(name) } + +/* + The following are for mocking the filesystem - this allows us to mock Stat() files. This means that we can set file attributes, and know that they + will be the same regardless of the platform sitting underneath our tests (directory sizes vary see https://github.com/influxdata/telegraf/issues/6011) + + NOTE: still need the on-disk file structure to mirror this because the 3rd party library ("github.com/karrick/godirwalk") uses its own + walk functions, that we cannot mock from here. +*/ + +type fakeFileSystem struct { + files map[string]fakeFileInfo +} + +type fakeFileInfo struct { + name string + size int64 + filemode uint32 + modtime time.Time + isdir bool + sys interface{} +} + +func (f fakeFileInfo) Name() string { return f.name } +func (f fakeFileInfo) Size() int64 { return f.size } +func (f fakeFileInfo) Mode() os.FileMode { return os.FileMode(f.filemode) } +func (f fakeFileInfo) ModTime() time.Time { return f.modtime } +func (f fakeFileInfo) IsDir() bool { return f.isdir } +func (f fakeFileInfo) Sys() interface{} { return f.sys } + +func (f fakeFileSystem) Open(name string) (file, error) { + return nil, &os.PathError{Op: "Open", Path: name, Err: errors.New("Not implemented by fake filesystem")} +} + +func (f fakeFileSystem) Stat(name string) (os.FileInfo, error) { + if fakeInfo, found := f.files[name]; found { + return fakeInfo, nil + } + return nil, &os.PathError{Op: "Stat", Path: name, Err: errors.New("No such file or directory")} + +} diff --git a/plugins/inputs/filecount/filesystem_helpers_test.go b/plugins/inputs/filecount/filesystem_helpers_test.go new file mode 100644 index 000000000..de028dcab --- /dev/null +++ b/plugins/inputs/filecount/filesystem_helpers_test.go @@ -0,0 +1,90 @@ +package filecount + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestMTime(t *testing.T) { + //this is the time our foo file should have + mtime := time.Date(2015, time.December, 14, 18, 25, 5, 0, time.UTC) + + fs := getTestFileSystem() + fileInfo, err := fs.Stat("/testdata/foo") + require.Nil(t, err) + require.Equal(t, mtime, fileInfo.ModTime()) +} + +func TestSize(t *testing.T) { + //this is the time our foo file should have + size := int64(4096) + fs := getTestFileSystem() + fileInfo, err := fs.Stat("/testdata") + require.Nil(t, err) + require.Equal(t, size, fileInfo.Size()) +} + +func TestIsDir(t *testing.T) { + //this is the time our foo file should have + dir := true + fs := getTestFileSystem() + fileInfo, err := fs.Stat("/testdata") + require.Nil(t, err) + require.Equal(t, dir, fileInfo.IsDir()) +} + +func TestRealFS(t *testing.T) { + //test that the default (non-test) empty FS causes expected behaviour + var fs fileSystem = osFS{} + //the following file exists on disk - and not in our fake fs + fileInfo, err := fs.Stat(getTestdataDir() + "/qux") + require.Nil(t, err) + require.Equal(t, false, fileInfo.IsDir()) + require.Equal(t, int64(446), fileInfo.Size()) + + // now swap out real, for fake filesystem + fs = getTestFileSystem() + // now, the same test as above will return an error as the file doesn't exist in our fake fs + expectedError := "Stat " + getTestdataDir() + "/qux: No such file or directory" + fileInfo, err = fs.Stat(getTestdataDir() + "/qux") + require.Equal(t, expectedError, err.Error()) + // and verify that what we DO expect to find, we do + fileInfo, err = fs.Stat("/testdata/foo") + require.Nil(t, err) +} + +func getTestFileSystem() fakeFileSystem { + /* + create our desired "filesystem" object, complete with an internal map allowing our funcs to return meta data as requested + + type FileInfo interface { + Name() string // base name of the file + Size() int64 // length in bytes of file + Mode() FileMode // file mode bits + ModTime() time.Time // modification time + IsDir() bool // returns bool indicating if a Dir or not + Sys() interface{} // underlying data source. always nil (in this case) + } + + */ + + mtime := time.Date(2015, time.December, 14, 18, 25, 5, 0, time.UTC) + + // set file permisions + var fmask uint32 = 0666 + var dmask uint32 = 0666 + + // set directory bit + dmask |= (1 << uint(32-1)) + + fileList := map[string]fakeFileInfo{ + "/testdata": {name: "testdata", size: int64(4096), filemode: uint32(dmask), modtime: mtime, isdir: true}, + "/testdata/foo": {name: "foo", filemode: uint32(fmask), modtime: mtime}, + } + + fs := fakeFileSystem{files: fileList} + return fs + +}