dom
/
telegraf
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Fix filecount plugin size tests (#6038)

This commit is contained in:
Nic Grobler 2019-06-24 20:03:05 +02:00 committed by Daniel Nelson
parent aa84011dc3
commit bd9ddd8cb1
4 changed files with 236 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
plugins/inputs/filecount/filecount.go
View File

@ -58,6 +58,7 @@ type FileCount struct {
MTime internal.Duration `toml:"mtime"`
fileFilters []fileFilterFunc
globPaths []globpath.GlobPath
Fs fileSystem
}
func (_ *FileCount) Description() string {
@ -159,7 +160,7 @@ func (fc *FileCount) count(acc telegraf.Accumulator, basedir string, glob globpa
if err == nil && rel == "." {
return nil
}
file, err := os.Stat(path)
file, err := fc.Fs.Stat(path)
if err != nil {
if os.IsNotExist(err) {
return nil
@ -244,7 +245,7 @@ func (fc *FileCount) Gather(acc telegraf.Accumulator) error {
}
for _, glob := range fc.globPaths {
for _, dir := range onlyDirectories(glob.GetRoots()) {
for _, dir := range fc.onlyDirectories(glob.GetRoots()) {
fc.count(acc, dir, glob)
}
}
@ -252,10 +253,10 @@ func (fc *FileCount) Gather(acc telegraf.Accumulator) error {
return nil
}
func onlyDirectories(directories []string) []string {
func (fc *FileCount) onlyDirectories(directories []string) []string {
out := make([]string, 0)
for _, path := range directories {
info, err := os.Stat(path)
info, err := fc.Fs.Stat(path)
if err == nil && info.IsDir() {
out = append(out, path)
}
@ -286,6 +287,7 @@ func (fc *FileCount) initGlobPaths(acc telegraf.Accumulator) {
fc.globPaths = append(fc.globPaths, *glob)
}
}
}
func NewFileCount() *FileCount {
@ -298,6 +300,7 @@ func NewFileCount() *FileCount {
Size: internal.Size{Size: 0},
MTime: internal.Duration{Duration: 0},
fileFilters: nil,
Fs: osFS{},
}
}

84
plugins/inputs/filecount/filecount_test.go
View File

@ -2,7 +2,6 @@ package filecount
import (
"os"
"path/filepath"
"runtime"
"strings"
"testing"
@ -18,7 +17,7 @@ func TestNoFilters(t *testing.T) {
matches := []string{"foo", "bar", "baz", "qux",
"subdir/", "subdir/quux", "subdir/quuz",
"subdir/nested2", "subdir/nested2/qux"}
fileCountEquals(t, fc, len(matches), 9084)
fileCountEquals(t, fc, len(matches), 5096)
}
func TestNoFiltersOnChildDir(t *testing.T) {
@ -30,9 +29,8 @@ func TestNoFiltersOnChildDir(t *testing.T) {
tags := map[string]string{"directory": getTestdataDir() + "/subdir"}
acc := testutil.Accumulator{}
acc.GatherError(fc.Gather)
require.True(t, acc.HasPoint("filecount", tags, "count", int64(len(matches))))
require.True(t, acc.HasPoint("filecount", tags, "size_bytes", int64(4542)))
require.True(t, acc.HasPoint("filecount", tags, "size_bytes", int64(600)))
}
func TestNoRecursiveButSuperMeta(t *testing.T) {
@ -46,7 +44,7 @@ func TestNoRecursiveButSuperMeta(t *testing.T) {
acc.GatherError(fc.Gather)
require.True(t, acc.HasPoint("filecount", tags, "count", int64(len(matches))))
require.True(t, acc.HasPoint("filecount", tags, "size_bytes", int64(4096)))
require.True(t, acc.HasPoint("filecount", tags, "size_bytes", int64(200)))
}
func TestNameFilter(t *testing.T) {
@ -60,20 +58,22 @@ func TestNonRecursive(t *testing.T) {
fc := getNoFilterFileCount()
fc.Recursive = false
matches := []string{"foo", "bar", "baz", "qux", "subdir"}
fileCountEquals(t, fc, len(matches), 4542)
fileCountEquals(t, fc, len(matches), 4496)
}
func TestDoubleAndSimpleStar(t *testing.T) {
fc := getNoFilterFileCount()
fc.Directories = []string{getTestdataDir() + "/**/*"}
matches := []string{"qux"}
tags := map[string]string{"directory": getTestdataDir() + "/subdir/nested2"}
acc := testutil.Accumulator{}
acc.GatherError(fc.Gather)
require.True(t, acc.HasPoint("filecount", tags, "count", int64(len(matches))))
require.True(t, acc.HasPoint("filecount", tags, "size_bytes", int64(446)))
require.True(t, acc.HasPoint("filecount", tags, "size_bytes", int64(400)))
}
func TestRegularOnlyFilter(t *testing.T) {
@ -82,7 +82,8 @@ func TestRegularOnlyFilter(t *testing.T) {
matches := []string{
"foo", "bar", "baz", "qux", "subdir/quux", "subdir/quuz",
"subdir/nested2/qux"}
fileCountEquals(t, fc, len(matches), 892)
fileCountEquals(t, fc, len(matches), 800)
}
func TestSizeFilter(t *testing.T) {
@ -94,23 +95,22 @@ func TestSizeFilter(t *testing.T) {
fc.Size = internal.Size{Size: 100}
matches = []string{"qux", "subdir/nested2//qux"}
fileCountEquals(t, fc, len(matches), 892)
fileCountEquals(t, fc, len(matches), 800)
}
func TestMTimeFilter(t *testing.T) {
oldFile := filepath.Join(getTestdataDir(), "baz")
mtime := time.Date(1979, time.December, 14, 18, 25, 5, 0, time.UTC)
if err := os.Chtimes(oldFile, mtime, mtime); err != nil {
t.Skip("skipping mtime filter test.")
}
mtime := time.Date(2011, time.December, 14, 18, 25, 5, 0, time.UTC)
fileAge := time.Since(mtime) - (60 * time.Second)
fc := getNoFilterFileCount()
fc.MTime = internal.Duration{Duration: -fileAge}
matches := []string{"foo", "bar", "qux",
"subdir/", "subdir/quux", "subdir/quuz",
"sbudir/nested2", "subdir/nested2/qux"}
fileCountEquals(t, fc, len(matches), 9084)
"subdir/nested2", "subdir/nested2/qux"}
fileCountEquals(t, fc, len(matches), 5096)
fc.MTime = internal.Duration{Duration: fileAge}
matches = []string{"baz"}
@ -126,12 +126,60 @@ func getNoFilterFileCount() FileCount {
Size: internal.Size{Size: 0},
MTime: internal.Duration{Duration: 0},
fileFilters: nil,
Fs: getFakeFileSystem(getTestdataDir()),
}
}
func getTestdataDir() string {
_, filename, _, _ := runtime.Caller(1)
return strings.Replace(filename, "filecount_test.go", "testdata", 1)
dir, err := os.Getwd()
if err != nil {
// if we cannot even establish the test directory, further progress is meaningless
panic(err)
}
var chunks []string
var testDirectory string
if runtime.GOOS == "windows" {
chunks = strings.Split(dir, "\\")
testDirectory = strings.Join(chunks[:], "\\") + "\\testdata"
} else {
chunks = strings.Split(dir, "/")
testDirectory = strings.Join(chunks[:], "/") + "/testdata"
}
return testDirectory
}
func getFakeFileSystem(basePath string) fakeFileSystem {
// create our desired "filesystem" object, complete with an internal map allowing our funcs to return meta data as requested
mtime := time.Date(2015, time.December, 14, 18, 25, 5, 0, time.UTC)
olderMtime := time.Date(2010, time.December, 14, 18, 25, 5, 0, time.UTC)
// set file permisions
var fmask uint32 = 0666
var dmask uint32 = 0666
// set directory bit
dmask |= (1 << uint(32-1))
// create a lookup map for getting "files" from the "filesystem"
fileList := map[string]fakeFileInfo{
basePath: {name: "testdata", size: int64(4096), filemode: uint32(dmask), modtime: mtime, isdir: true},
basePath + "/foo": {name: "foo", filemode: uint32(fmask), modtime: mtime},
basePath + "/bar": {name: "bar", filemode: uint32(fmask), modtime: mtime},
basePath + "/baz": {name: "baz", filemode: uint32(fmask), modtime: olderMtime},
basePath + "/qux": {name: "qux", size: int64(400), filemode: uint32(fmask), modtime: mtime},
basePath + "/subdir": {name: "subdir", size: int64(4096), filemode: uint32(dmask), modtime: mtime, isdir: true},
basePath + "/subdir/quux": {name: "quux", filemode: uint32(fmask), modtime: mtime},
basePath + "/subdir/quuz": {name: "quuz", filemode: uint32(fmask), modtime: mtime},
basePath + "/subdir/nested2": {name: "nested2", size: int64(200), filemode: uint32(dmask), modtime: mtime, isdir: true},
basePath + "/subdir/nested2/qux": {name: "qux", filemode: uint32(fmask), modtime: mtime, size: int64(400)},
}
fs := fakeFileSystem{files: fileList}
return fs
}
func fileCountEquals(t *testing.T, fc FileCount, expectedCount int, expectedSize int) {

73
plugins/inputs/filecount/filesystem_helpers.go Normal file
View File

@ -0,0 +1,73 @@
package filecount
import (
"errors"
"io"
"os"
"time"
)
/*
The code below is lifted from numerous articles and originates from Andrew Gerrand's 10 things you (probably) don't know about Go.
it allows for mocking a filesystem; this allows for consistent testing of this code across platforms (directory sizes reported
differently by different platforms, for example), while preserving the rest of the functionality as-is, without modification.
*/
type fileSystem interface {
Open(name string) (file, error)
Stat(name string) (os.FileInfo, error)
}
type file interface {
io.Closer
io.Reader
io.ReaderAt
io.Seeker
Stat() (os.FileInfo, error)
}
// osFS implements fileSystem using the local disk
type osFS struct{}
func (osFS) Open(name string) (file, error) { return os.Open(name) }
func (osFS) Stat(name string) (os.FileInfo, error) { return os.Stat(name) }
/*
The following are for mocking the filesystem - this allows us to mock Stat() files. This means that we can set file attributes, and know that they
will be the same regardless of the platform sitting underneath our tests (directory sizes vary see https://github.com/influxdata/telegraf/issues/6011)
NOTE: still need the on-disk file structure to mirror this because the 3rd party library ("github.com/karrick/godirwalk") uses its own
walk functions, that we cannot mock from here.
*/
type fakeFileSystem struct {
files map[string]fakeFileInfo
}
type fakeFileInfo struct {
name string
size int64
filemode uint32
modtime time.Time
isdir bool
sys interface{}
}
func (f fakeFileInfo) Name() string { return f.name }
func (f fakeFileInfo) Size() int64 { return f.size }
func (f fakeFileInfo) Mode() os.FileMode { return os.FileMode(f.filemode) }
func (f fakeFileInfo) ModTime() time.Time { return f.modtime }
func (f fakeFileInfo) IsDir() bool { return f.isdir }
func (f fakeFileInfo) Sys() interface{} { return f.sys }
func (f fakeFileSystem) Open(name string) (file, error) {
return nil, &os.PathError{Op: "Open", Path: name, Err: errors.New("Not implemented by fake filesystem")}
}
func (f fakeFileSystem) Stat(name string) (os.FileInfo, error) {
if fakeInfo, found := f.files[name]; found {
return fakeInfo, nil
}
return nil, &os.PathError{Op: "Stat", Path: name, Err: errors.New("No such file or directory")}
}

90
plugins/inputs/filecount/filesystem_helpers_test.go Normal file
View File

@ -0,0 +1,90 @@
package filecount
import (
"testing"
"time"
"github.com/stretchr/testify/require"
)
func TestMTime(t *testing.T) {
//this is the time our foo file should have
mtime := time.Date(2015, time.December, 14, 18, 25, 5, 0, time.UTC)
fs := getTestFileSystem()
fileInfo, err := fs.Stat("/testdata/foo")
require.Nil(t, err)
require.Equal(t, mtime, fileInfo.ModTime())
}
func TestSize(t *testing.T) {
//this is the time our foo file should have
size := int64(4096)
fs := getTestFileSystem()
fileInfo, err := fs.Stat("/testdata")
require.Nil(t, err)
require.Equal(t, size, fileInfo.Size())
}
func TestIsDir(t *testing.T) {
//this is the time our foo file should have
dir := true
fs := getTestFileSystem()
fileInfo, err := fs.Stat("/testdata")
require.Nil(t, err)
require.Equal(t, dir, fileInfo.IsDir())
}
func TestRealFS(t *testing.T) {
//test that the default (non-test) empty FS causes expected behaviour
var fs fileSystem = osFS{}
//the following file exists on disk - and not in our fake fs
fileInfo, err := fs.Stat(getTestdataDir() + "/qux")
require.Nil(t, err)
require.Equal(t, false, fileInfo.IsDir())
require.Equal(t, int64(446), fileInfo.Size())
// now swap out real, for fake filesystem
fs = getTestFileSystem()
// now, the same test as above will return an error as the file doesn't exist in our fake fs
expectedError := "Stat " + getTestdataDir() + "/qux: No such file or directory"
fileInfo, err = fs.Stat(getTestdataDir() + "/qux")
require.Equal(t, expectedError, err.Error())
// and verify that what we DO expect to find, we do
fileInfo, err = fs.Stat("/testdata/foo")
require.Nil(t, err)
}
func getTestFileSystem() fakeFileSystem {
/*
create our desired "filesystem" object, complete with an internal map allowing our funcs to return meta data as requested
type FileInfo interface {
Name() string // base name of the file
Size() int64 // length in bytes of file
Mode() FileMode // file mode bits
ModTime() time.Time // modification time
IsDir() bool // returns bool indicating if a Dir or not
Sys() interface{} // underlying data source. always nil (in this case)
}
*/
mtime := time.Date(2015, time.December, 14, 18, 25, 5, 0, time.UTC)
// set file permisions
var fmask uint32 = 0666
var dmask uint32 = 0666
// set directory bit
dmask |= (1 << uint(32-1))
fileList := map[string]fakeFileInfo{
"/testdata": {name: "testdata", size: int64(4096), filemode: uint32(dmask), modtime: mtime, isdir: true},
"/testdata/foo": {name: "foo", filemode: uint32(fmask), modtime: mtime},
}
fs := fakeFileSystem{files: fileList}
return fs
}