Fix filecount plugin size tests (#6038)

plugins/inputs/filecount/filecount.go

@@ -58,6 +58,7 @@ type FileCount struct {
 	MTime       internal.Duration `toml:"mtime"`
 	fileFilters []fileFilterFunc
 	globPaths   []globpath.GlobPath
+	Fs          fileSystem
 }
 
 func (_ *FileCount) Description() string {
@@ -159,7 +160,7 @@ func (fc *FileCount) count(acc telegraf.Accumulator, basedir string, glob globpa
 		if err == nil && rel == "." {
 			return nil
 		}
-		file, err := os.Stat(path)
+		file, err := fc.Fs.Stat(path)
 		if err != nil {
 			if os.IsNotExist(err) {
 				return nil
@@ -244,7 +245,7 @@ func (fc *FileCount) Gather(acc telegraf.Accumulator) error {
 	}
 
 	for _, glob := range fc.globPaths {
-		for _, dir := range onlyDirectories(glob.GetRoots()) {
+		for _, dir := range fc.onlyDirectories(glob.GetRoots()) {
 			fc.count(acc, dir, glob)
 		}
 	}
@@ -252,10 +253,10 @@ func (fc *FileCount) Gather(acc telegraf.Accumulator) error {
 	return nil
 }
 
-func onlyDirectories(directories []string) []string {
+func (fc *FileCount) onlyDirectories(directories []string) []string {
 	out := make([]string, 0)
 	for _, path := range directories {
-		info, err := os.Stat(path)
+		info, err := fc.Fs.Stat(path)
 		if err == nil && info.IsDir() {
 			out = append(out, path)
 		}
@@ -286,6 +287,7 @@ func (fc *FileCount) initGlobPaths(acc telegraf.Accumulator) {
 			fc.globPaths = append(fc.globPaths, *glob)
 		}
 	}
+
 }
 
 func NewFileCount() *FileCount {
@@ -298,6 +300,7 @@ func NewFileCount() *FileCount {
 		Size:        internal.Size{Size: 0},
 		MTime:       internal.Duration{Duration: 0},
 		fileFilters: nil,
+		Fs:          osFS{},
 	}
 }
 

plugins/inputs/filecount/filecount_test.go

@@ -2,7 +2,6 @@ package filecount
 
 import (
 	"os"
-	"path/filepath"
 	"runtime"
 	"strings"
 	"testing"
@@ -18,7 +17,7 @@ func TestNoFilters(t *testing.T) {
 	matches := []string{"foo", "bar", "baz", "qux",
 		"subdir/", "subdir/quux", "subdir/quuz",
 		"subdir/nested2", "subdir/nested2/qux"}
-	fileCountEquals(t, fc, len(matches), 9084)
+	fileCountEquals(t, fc, len(matches), 5096)
 }
 
 func TestNoFiltersOnChildDir(t *testing.T) {
@@ -30,9 +29,8 @@ func TestNoFiltersOnChildDir(t *testing.T) {
 	tags := map[string]string{"directory": getTestdataDir() + "/subdir"}
 	acc := testutil.Accumulator{}
 	acc.GatherError(fc.Gather)
-
 	require.True(t, acc.HasPoint("filecount", tags, "count", int64(len(matches))))
-	require.True(t, acc.HasPoint("filecount", tags, "size_bytes", int64(4542)))
+	require.True(t, acc.HasPoint("filecount", tags, "size_bytes", int64(600)))
 }
 
 func TestNoRecursiveButSuperMeta(t *testing.T) {
@@ -46,7 +44,7 @@ func TestNoRecursiveButSuperMeta(t *testing.T) {
 	acc.GatherError(fc.Gather)
 
 	require.True(t, acc.HasPoint("filecount", tags, "count", int64(len(matches))))
-	require.True(t, acc.HasPoint("filecount", tags, "size_bytes", int64(4096)))
+	require.True(t, acc.HasPoint("filecount", tags, "size_bytes", int64(200)))
 }
 
 func TestNameFilter(t *testing.T) {
@@ -60,20 +58,22 @@ func TestNonRecursive(t *testing.T) {
 	fc := getNoFilterFileCount()
 	fc.Recursive = false
 	matches := []string{"foo", "bar", "baz", "qux", "subdir"}
-	fileCountEquals(t, fc, len(matches), 4542)
+
+	fileCountEquals(t, fc, len(matches), 4496)
 }
 
 func TestDoubleAndSimpleStar(t *testing.T) {
 	fc := getNoFilterFileCount()
 	fc.Directories = []string{getTestdataDir() + "/**/*"}
 	matches := []string{"qux"}
+
 	tags := map[string]string{"directory": getTestdataDir() + "/subdir/nested2"}
 
 	acc := testutil.Accumulator{}
 	acc.GatherError(fc.Gather)
 
 	require.True(t, acc.HasPoint("filecount", tags, "count", int64(len(matches))))
-	require.True(t, acc.HasPoint("filecount", tags, "size_bytes", int64(446)))
+	require.True(t, acc.HasPoint("filecount", tags, "size_bytes", int64(400)))
 }
 
 func TestRegularOnlyFilter(t *testing.T) {
@@ -82,7 +82,8 @@ func TestRegularOnlyFilter(t *testing.T) {
 	matches := []string{
 		"foo", "bar", "baz", "qux", "subdir/quux", "subdir/quuz",
 		"subdir/nested2/qux"}
-	fileCountEquals(t, fc, len(matches), 892)
+
+	fileCountEquals(t, fc, len(matches), 800)
 }
 
 func TestSizeFilter(t *testing.T) {
@@ -94,23 +95,22 @@ func TestSizeFilter(t *testing.T) {
 
 	fc.Size = internal.Size{Size: 100}
 	matches = []string{"qux", "subdir/nested2//qux"}
-	fileCountEquals(t, fc, len(matches), 892)
+
+	fileCountEquals(t, fc, len(matches), 800)
 }
 
 func TestMTimeFilter(t *testing.T) {
-	oldFile := filepath.Join(getTestdataDir(), "baz")
+
-	mtime := time.Date(1979, time.December, 14, 18, 25, 5, 0, time.UTC)
+	mtime := time.Date(2011, time.December, 14, 18, 25, 5, 0, time.UTC)
-	if err := os.Chtimes(oldFile, mtime, mtime); err != nil {
-		t.Skip("skipping mtime filter test.")
-	}
 	fileAge := time.Since(mtime) - (60 * time.Second)
 
 	fc := getNoFilterFileCount()
 	fc.MTime = internal.Duration{Duration: -fileAge}
 	matches := []string{"foo", "bar", "qux",
 		"subdir/", "subdir/quux", "subdir/quuz",
-		"sbudir/nested2", "subdir/nested2/qux"}
+		"subdir/nested2", "subdir/nested2/qux"}
-	fileCountEquals(t, fc, len(matches), 9084)
+
+	fileCountEquals(t, fc, len(matches), 5096)
 
 	fc.MTime = internal.Duration{Duration: fileAge}
 	matches = []string{"baz"}
@@ -126,12 +126,60 @@ func getNoFilterFileCount() FileCount {
 		Size:        internal.Size{Size: 0},
 		MTime:       internal.Duration{Duration: 0},
 		fileFilters: nil,
+		Fs:          getFakeFileSystem(getTestdataDir()),
 	}
 }
 
 func getTestdataDir() string {
-	_, filename, _, _ := runtime.Caller(1)
+	dir, err := os.Getwd()
-	return strings.Replace(filename, "filecount_test.go", "testdata", 1)
+	if err != nil {
+		// if we cannot even establish the test directory, further progress is meaningless
+		panic(err)
+	}
+
+	var chunks []string
+	var testDirectory string
+
+	if runtime.GOOS == "windows" {
+		chunks = strings.Split(dir, "\\")
+		testDirectory = strings.Join(chunks[:], "\\") + "\\testdata"
+	} else {
+		chunks = strings.Split(dir, "/")
+		testDirectory = strings.Join(chunks[:], "/") + "/testdata"
+	}
+	return testDirectory
+}
+
+func getFakeFileSystem(basePath string) fakeFileSystem {
+	// create our desired "filesystem" object, complete with an internal map allowing our funcs to return meta data as requested
+
+	mtime := time.Date(2015, time.December, 14, 18, 25, 5, 0, time.UTC)
+	olderMtime := time.Date(2010, time.December, 14, 18, 25, 5, 0, time.UTC)
+
+	// set file permisions
+	var fmask uint32 = 0666
+	var dmask uint32 = 0666
+
+	// set directory bit
+	dmask |= (1 << uint(32-1))
+
+	// create a lookup map for getting "files" from the "filesystem"
+	fileList := map[string]fakeFileInfo{
+		basePath:                         {name: "testdata", size: int64(4096), filemode: uint32(dmask), modtime: mtime, isdir: true},
+		basePath + "/foo":                {name: "foo", filemode: uint32(fmask), modtime: mtime},
+		basePath + "/bar":                {name: "bar", filemode: uint32(fmask), modtime: mtime},
+		basePath + "/baz":                {name: "baz", filemode: uint32(fmask), modtime: olderMtime},
+		basePath + "/qux":                {name: "qux", size: int64(400), filemode: uint32(fmask), modtime: mtime},
+		basePath + "/subdir":             {name: "subdir", size: int64(4096), filemode: uint32(dmask), modtime: mtime, isdir: true},
+		basePath + "/subdir/quux":        {name: "quux", filemode: uint32(fmask), modtime: mtime},
+		basePath + "/subdir/quuz":        {name: "quuz", filemode: uint32(fmask), modtime: mtime},
+		basePath + "/subdir/nested2":     {name: "nested2", size: int64(200), filemode: uint32(dmask), modtime: mtime, isdir: true},
+		basePath + "/subdir/nested2/qux": {name: "qux", filemode: uint32(fmask), modtime: mtime, size: int64(400)},
+	}
+
+	fs := fakeFileSystem{files: fileList}
+	return fs
+
 }
 
 func fileCountEquals(t *testing.T, fc FileCount, expectedCount int, expectedSize int) {

plugins/inputs/filecount/filesystem_helpers.go

@@ -0,0 +1,73 @@
+package filecount
+
+import (
+	"errors"
+	"io"
+	"os"
+	"time"
+)
+
+/*
+	The code below is lifted from numerous articles and originates from Andrew Gerrand's 10 things you (probably) don't know about Go.
+	it allows for mocking a filesystem; this allows for consistent testing of this code across platforms (directory sizes reported
+	differently by different platforms, for example), while preserving the rest of the functionality as-is, without modification.
+*/
+
+type fileSystem interface {
+	Open(name string) (file, error)
+	Stat(name string) (os.FileInfo, error)
+}
+
+type file interface {
+	io.Closer
+	io.Reader
+	io.ReaderAt
+	io.Seeker
+	Stat() (os.FileInfo, error)
+}
+
+// osFS implements fileSystem using the local disk
+type osFS struct{}
+
+func (osFS) Open(name string) (file, error)        { return os.Open(name) }
+func (osFS) Stat(name string) (os.FileInfo, error) { return os.Stat(name) }
+
+/*
+	The following are for mocking the filesystem - this allows us to mock Stat() files. This means that we can set file attributes, and know that they
+	will be the same regardless of the platform sitting underneath our tests (directory sizes vary see https://github.com/influxdata/telegraf/issues/6011)
+
+	NOTE: still need the on-disk file structure to mirror this because the 3rd party library ("github.com/karrick/godirwalk") uses its own
+	walk functions, that we cannot mock from here.
+*/
+
+type fakeFileSystem struct {
+	files map[string]fakeFileInfo
+}
+
+type fakeFileInfo struct {
+	name     string
+	size     int64
+	filemode uint32
+	modtime  time.Time
+	isdir    bool
+	sys      interface{}
+}
+
+func (f fakeFileInfo) Name() string       { return f.name }
+func (f fakeFileInfo) Size() int64        { return f.size }
+func (f fakeFileInfo) Mode() os.FileMode  { return os.FileMode(f.filemode) }
+func (f fakeFileInfo) ModTime() time.Time { return f.modtime }
+func (f fakeFileInfo) IsDir() bool        { return f.isdir }
+func (f fakeFileInfo) Sys() interface{}   { return f.sys }
+
+func (f fakeFileSystem) Open(name string) (file, error) {
+	return nil, &os.PathError{Op: "Open", Path: name, Err: errors.New("Not implemented by fake filesystem")}
+}
+
+func (f fakeFileSystem) Stat(name string) (os.FileInfo, error) {
+	if fakeInfo, found := f.files[name]; found {
+		return fakeInfo, nil
+	}
+	return nil, &os.PathError{Op: "Stat", Path: name, Err: errors.New("No such file or directory")}
+
+}

plugins/inputs/filecount/filesystem_helpers_test.go

@@ -0,0 +1,90 @@
+package filecount
+
+import (
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestMTime(t *testing.T) {
+	//this is the time our foo file should have
+	mtime := time.Date(2015, time.December, 14, 18, 25, 5, 0, time.UTC)
+
+	fs := getTestFileSystem()
+	fileInfo, err := fs.Stat("/testdata/foo")
+	require.Nil(t, err)
+	require.Equal(t, mtime, fileInfo.ModTime())
+}
+
+func TestSize(t *testing.T) {
+	//this is the time our foo file should have
+	size := int64(4096)
+	fs := getTestFileSystem()
+	fileInfo, err := fs.Stat("/testdata")
+	require.Nil(t, err)
+	require.Equal(t, size, fileInfo.Size())
+}
+
+func TestIsDir(t *testing.T) {
+	//this is the time our foo file should have
+	dir := true
+	fs := getTestFileSystem()
+	fileInfo, err := fs.Stat("/testdata")
+	require.Nil(t, err)
+	require.Equal(t, dir, fileInfo.IsDir())
+}
+
+func TestRealFS(t *testing.T) {
+	//test that the default (non-test) empty FS causes expected behaviour
+	var fs fileSystem = osFS{}
+	//the following file exists on disk - and not in our fake fs
+	fileInfo, err := fs.Stat(getTestdataDir() + "/qux")
+	require.Nil(t, err)
+	require.Equal(t, false, fileInfo.IsDir())
+	require.Equal(t, int64(446), fileInfo.Size())
+
+	// now swap out real, for fake filesystem
+	fs = getTestFileSystem()
+	// now, the same test as above will return an error as the file doesn't exist in our fake fs
+	expectedError := "Stat " + getTestdataDir() + "/qux: No such file or directory"
+	fileInfo, err = fs.Stat(getTestdataDir() + "/qux")
+	require.Equal(t, expectedError, err.Error())
+	// and verify that what we DO expect to find, we do
+	fileInfo, err = fs.Stat("/testdata/foo")
+	require.Nil(t, err)
+}
+
+func getTestFileSystem() fakeFileSystem {
+	/*
+		create our desired "filesystem" object, complete with an internal map allowing our funcs to return meta data as requested
+
+		type FileInfo interface {
+			Name() string       // base name of the file
+			Size() int64        // length in bytes of file
+			Mode() FileMode     // file mode bits
+			ModTime() time.Time // modification time
+			IsDir() bool        // returns bool indicating if a Dir or not
+			Sys() interface{}   // underlying data source. always nil (in this case)
+		}
+
+	*/
+
+	mtime := time.Date(2015, time.December, 14, 18, 25, 5, 0, time.UTC)
+
+	// set file permisions
+	var fmask uint32 = 0666
+	var dmask uint32 = 0666
+
+	// set directory bit
+	dmask |= (1 << uint(32-1))
+
+	fileList := map[string]fakeFileInfo{
+		"/testdata":     {name: "testdata", size: int64(4096), filemode: uint32(dmask), modtime: mtime, isdir: true},
+		"/testdata/foo": {name: "foo", filemode: uint32(fmask), modtime: mtime},
+	}
+
+	fs := fakeFileSystem{files: fileList}
+	return fs
+
+}