telegraf/plugins/inputs/filecount/filecount.go

312 lines
7.0 KiB
Go
Raw Normal View History

2018-07-31 22:05:55 +00:00
package filecount
import (
"os"
"path/filepath"
"time"
"github.com/influxdata/telegraf"
"github.com/influxdata/telegraf/internal"
"github.com/influxdata/telegraf/internal/globpath"
2018-07-31 22:05:55 +00:00
"github.com/influxdata/telegraf/plugins/inputs"
"github.com/karrick/godirwalk"
"github.com/pkg/errors"
2018-07-31 22:05:55 +00:00
)
const sampleConfig = `
## Directory to gather stats about.
## deprecated in 1.9; use the directories option
# directory = "/var/cache/apt/archives"
2018-07-31 22:05:55 +00:00
## Directories to gather stats about.
## This accept standard unit glob matching rules, but with the addition of
## ** as a "super asterisk". ie:
## /var/log/** -> recursively find all directories in /var/log and count files in each directories
## /var/log/*/* -> find all directories with a parent dir in /var/log and count files in each directories
## /var/log -> count all files in /var/log and all of its subdirectories
directories = ["/var/cache/apt/archives"]
2018-07-31 22:05:55 +00:00
## Only count files that match the name pattern. Defaults to "*".
name = "*.deb"
## Count files in subdirectories. Defaults to true.
recursive = false
## Only count regular files. Defaults to true.
regular_only = true
## Only count files that are at least this size. If size is
2018-07-31 22:05:55 +00:00
## a negative number, only count files that are smaller than the
## absolute value of size. Acceptable units are B, KiB, MiB, KB, ...
## Without quotes and units, interpreted as size in bytes.
size = "0B"
2018-07-31 22:05:55 +00:00
## Only count files that have not been touched for at least this
## duration. If mtime is negative, only count files that have been
## touched in this duration. Defaults to "0s".
mtime = "0s"
`
type FileCount struct {
Directory string // deprecated in 1.9
Directories []string
2018-07-31 22:05:55 +00:00
Name string
Recursive bool
RegularOnly bool
Size internal.Size
2018-07-31 22:05:55 +00:00
MTime internal.Duration `toml:"mtime"`
fileFilters []fileFilterFunc
2018-12-13 20:25:49 +00:00
globPaths []globpath.GlobPath
Fs fileSystem
Log telegraf.Logger
2018-07-31 22:05:55 +00:00
}
func (_ *FileCount) Description() string {
return "Count files in a directory"
}
func (_ *FileCount) SampleConfig() string { return sampleConfig }
2018-12-13 20:25:49 +00:00
type fileFilterFunc func(os.FileInfo) (bool, error)
2018-07-31 22:05:55 +00:00
func rejectNilFilters(filters []fileFilterFunc) []fileFilterFunc {
filtered := make([]fileFilterFunc, 0, len(filters))
for _, f := range filters {
if f != nil {
filtered = append(filtered, f)
}
}
return filtered
}
func (fc *FileCount) nameFilter() fileFilterFunc {
if fc.Name == "*" {
return nil
}
return func(f os.FileInfo) (bool, error) {
match, err := filepath.Match(fc.Name, f.Name())
if err != nil {
return false, err
}
return match, nil
}
}
func (fc *FileCount) regularOnlyFilter() fileFilterFunc {
if !fc.RegularOnly {
return nil
}
return func(f os.FileInfo) (bool, error) {
return f.Mode().IsRegular(), nil
}
}
func (fc *FileCount) sizeFilter() fileFilterFunc {
if fc.Size.Size == 0 {
2018-07-31 22:05:55 +00:00
return nil
}
return func(f os.FileInfo) (bool, error) {
if !f.Mode().IsRegular() {
return false, nil
}
if fc.Size.Size < 0 {
return f.Size() < -fc.Size.Size, nil
2018-07-31 22:05:55 +00:00
}
return f.Size() >= fc.Size.Size, nil
2018-07-31 22:05:55 +00:00
}
}
func (fc *FileCount) mtimeFilter() fileFilterFunc {
if fc.MTime.Duration == 0 {
return nil
}
return func(f os.FileInfo) (bool, error) {
age := absDuration(fc.MTime.Duration)
mtime := time.Now().Add(-age)
if fc.MTime.Duration < 0 {
return f.ModTime().After(mtime), nil
}
return f.ModTime().Before(mtime), nil
}
}
func absDuration(x time.Duration) time.Duration {
if x < 0 {
return -x
}
return x
}
2018-12-13 20:25:49 +00:00
func (fc *FileCount) initFileFilters() {
filters := []fileFilterFunc{
fc.nameFilter(),
fc.regularOnlyFilter(),
fc.sizeFilter(),
fc.mtimeFilter(),
}
fc.fileFilters = rejectNilFilters(filters)
}
func (fc *FileCount) count(acc telegraf.Accumulator, basedir string, glob globpath.GlobPath) {
childCount := make(map[string]int64)
childSize := make(map[string]int64)
2018-12-13 20:25:49 +00:00
walkFn := func(path string, de *godirwalk.Dirent) error {
rel, err := filepath.Rel(basedir, path)
if err == nil && rel == "." {
2018-12-13 20:25:49 +00:00
return nil
}
file, err := fc.Fs.Stat(path)
if err != nil {
if os.IsNotExist(err) {
return nil
}
return err
}
match, err := fc.filter(file)
if err != nil {
acc.AddError(err)
return nil
}
if match {
parent := filepath.Dir(path)
2018-12-13 20:25:49 +00:00
childCount[parent]++
childSize[parent] += file.Size()
}
2018-12-13 20:25:49 +00:00
if file.IsDir() && !fc.Recursive && !glob.HasSuperMeta {
2018-07-31 22:05:55 +00:00
return filepath.SkipDir
}
return nil
}
2018-12-13 20:25:49 +00:00
postChildrenFn := func(path string, de *godirwalk.Dirent) error {
if glob.MatchString(path) {
gauge := map[string]interface{}{
"count": childCount[path],
"size_bytes": childSize[path],
}
acc.AddGauge("filecount", gauge,
map[string]string{
"directory": path,
})
}
parent := filepath.Dir(path)
2018-12-13 20:25:49 +00:00
if fc.Recursive {
childCount[parent] += childCount[path]
childSize[parent] += childSize[path]
}
delete(childCount, path)
delete(childSize, path)
return nil
}
2018-12-13 20:25:49 +00:00
err := godirwalk.Walk(basedir, &godirwalk.Options{
Callback: walkFn,
PostChildrenCallback: postChildrenFn,
Unsorted: true,
ErrorCallback: func(osPathname string, err error) godirwalk.ErrorAction {
if os.IsPermission(errors.Cause(err)) {
fc.Log.Debug(err)
return godirwalk.SkipNode
}
return godirwalk.Halt
},
2018-12-13 20:25:49 +00:00
})
if err != nil {
acc.AddError(err)
}
2018-07-31 22:05:55 +00:00
}
func (fc *FileCount) filter(file os.FileInfo) (bool, error) {
if fc.fileFilters == nil {
fc.initFileFilters()
}
for _, fileFilter := range fc.fileFilters {
match, err := fileFilter(file)
if err != nil {
return false, err
}
if !match {
return false, nil
}
}
return true, nil
}
func (fc *FileCount) Gather(acc telegraf.Accumulator) error {
2018-12-13 20:25:49 +00:00
if fc.globPaths == nil {
fc.initGlobPaths(acc)
2018-07-31 22:05:55 +00:00
}
2018-12-13 20:25:49 +00:00
for _, glob := range fc.globPaths {
for _, dir := range fc.onlyDirectories(glob.GetRoots()) {
2018-12-13 20:25:49 +00:00
fc.count(acc, dir, glob)
}
}
2018-07-31 22:05:55 +00:00
return nil
}
func (fc *FileCount) onlyDirectories(directories []string) []string {
2018-12-13 20:25:49 +00:00
out := make([]string, 0)
for _, path := range directories {
info, err := fc.Fs.Stat(path)
2018-12-13 20:25:49 +00:00
if err == nil && info.IsDir() {
out = append(out, path)
}
}
return out
}
func (fc *FileCount) getDirs() []string {
dirs := make([]string, len(fc.Directories))
for i, dir := range fc.Directories {
dirs[i] = filepath.Clean(dir)
}
if fc.Directory != "" {
dirs = append(dirs, filepath.Clean(fc.Directory))
}
return dirs
}
2018-12-13 20:25:49 +00:00
func (fc *FileCount) initGlobPaths(acc telegraf.Accumulator) {
fc.globPaths = []globpath.GlobPath{}
for _, directory := range fc.getDirs() {
glob, err := globpath.Compile(directory)
if err != nil {
2018-12-13 20:25:49 +00:00
acc.AddError(err)
} else {
fc.globPaths = append(fc.globPaths, *glob)
}
}
}
2018-07-31 22:05:55 +00:00
func NewFileCount() *FileCount {
return &FileCount{
Directory: "",
Directories: []string{},
2018-07-31 22:05:55 +00:00
Name: "*",
Recursive: true,
RegularOnly: true,
Size: internal.Size{Size: 0},
2018-07-31 22:05:55 +00:00
MTime: internal.Duration{Duration: 0},
fileFilters: nil,
Fs: osFS{},
2018-07-31 22:05:55 +00:00
}
}
func init() {
inputs.Add("filecount", func() telegraf.Input {
return NewFileCount()
})
}