Improve performance of globpath with some patterns (#4836)

This commit is contained in:
Samuel-BF 2018-10-12 23:48:11 +02:00 committed by Daniel Nelson
parent a2ac9115b3
commit f259229a35
2 changed files with 29 additions and 55 deletions

View File

@ -15,8 +15,8 @@ type GlobPath struct {
path string path string
hasMeta bool hasMeta bool
hasSuperMeta bool hasSuperMeta bool
rootGlob string
g glob.Glob g glob.Glob
root string
} }
func Compile(path string) (*GlobPath, error) { func Compile(path string) (*GlobPath, error) {
@ -27,23 +27,25 @@ func Compile(path string) (*GlobPath, error) {
} }
// if there are no glob meta characters in the path, don't bother compiling // if there are no glob meta characters in the path, don't bother compiling
// a glob object or finding the root directory. (see short-circuit in Match) // a glob object
if !out.hasMeta || !out.hasSuperMeta { if !out.hasMeta || !out.hasSuperMeta {
return &out, nil return &out, nil
} }
// find the root elements of the object path, the entry point for recursion
// when you have a super-meta in your path (which are :
// glob(/your/expression/until/first/star/of/super-meta))
out.rootGlob = path[:strings.Index(path, "**")+1]
var err error var err error
if out.g, err = glob.Compile(path, os.PathSeparator); err != nil { if out.g, err = glob.Compile(path, os.PathSeparator); err != nil {
return nil, err return nil, err
} }
// Get the root directory for this filepath
out.root = findRootDir(path)
return &out, nil return &out, nil
} }
func (g *GlobPath) Match() map[string]os.FileInfo { func (g *GlobPath) Match() map[string]os.FileInfo {
if !g.hasMeta {
out := make(map[string]os.FileInfo) out := make(map[string]os.FileInfo)
if !g.hasMeta {
info, err := os.Stat(g.path) info, err := os.Stat(g.path)
if err == nil { if err == nil {
out[g.path] = info out[g.path] = info
@ -51,7 +53,6 @@ func (g *GlobPath) Match() map[string]os.FileInfo {
return out return out
} }
if !g.hasSuperMeta { if !g.hasSuperMeta {
out := make(map[string]os.FileInfo)
files, _ := filepath.Glob(g.path) files, _ := filepath.Glob(g.path)
for _, file := range files { for _, file := range files {
info, err := os.Stat(file) info, err := os.Stat(file)
@ -61,46 +62,19 @@ func (g *GlobPath) Match() map[string]os.FileInfo {
} }
return out return out
} }
return walkFilePath(g.root, g.g) roots, err := filepath.Glob(g.rootGlob)
if err != nil {
return out
} }
// walk the filepath from the given root and return a list of files that match
// the given glob.
func walkFilePath(root string, g glob.Glob) map[string]os.FileInfo {
matchedFiles := make(map[string]os.FileInfo)
walkfn := func(path string, info os.FileInfo, _ error) error { walkfn := func(path string, info os.FileInfo, _ error) error {
if g.Match(path) { if g.g.Match(path) {
matchedFiles[path] = info out[path] = info
} }
return nil return nil
}
filepath.Walk(root, walkfn)
return matchedFiles
}
// find the root dir of the given path (could include globs).
// ie:
// /var/log/telegraf.conf -> /var/log
// /home/** -> /home
// /home/*/** -> /home
// /lib/share/*/*/**.txt -> /lib/share
func findRootDir(path string) string {
pathItems := strings.Split(path, sepStr)
out := sepStr
for i, item := range pathItems {
if i == len(pathItems)-1 {
break
} }
if item == "" { for _, root := range roots {
continue filepath.Walk(root, walkfn)
}
if hasMeta(item) {
break
}
out += item + sepStr
}
if out != "/" {
out = strings.TrimSuffix(out, "/")
} }
return out return out
} }

View File

@ -6,7 +6,6 @@ import (
"strings" "strings"
"testing" "testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
) )
@ -29,31 +28,32 @@ func TestCompileAndMatch(t *testing.T) {
require.NoError(t, err) require.NoError(t, err)
matches := g1.Match() matches := g1.Match()
assert.Len(t, matches, 6) require.Len(t, matches, 6)
matches = g2.Match() matches = g2.Match()
assert.Len(t, matches, 2) require.Len(t, matches, 2)
matches = g3.Match() matches = g3.Match()
assert.Len(t, matches, 1) require.Len(t, matches, 1)
matches = g4.Match() matches = g4.Match()
assert.Len(t, matches, 0) require.Len(t, matches, 0)
matches = g5.Match() matches = g5.Match()
assert.Len(t, matches, 0) require.Len(t, matches, 0)
} }
func TestFindRootDir(t *testing.T) { func TestRootGlob(t *testing.T) {
dir := getTestdataDir()
tests := []struct { tests := []struct {
input string input string
output string output string
}{ }{
{"/var/log/telegraf.conf", "/var/log"}, {dir + "/**", dir + "/*"},
{"/home/**", "/home"}, {dir + "/nested?/**", dir + "/nested?/*"},
{"/home/*/**", "/home"}, {dir + "/ne**/nest*", dir + "/ne*"},
{"/lib/share/*/*/**.txt", "/lib/share"}, {dir + "/nested?/*", ""},
} }
for _, test := range tests { for _, test := range tests {
actual := findRootDir(test.input) actual, _ := Compile(test.input)
assert.Equal(t, test.output, actual) require.Equal(t, actual.rootGlob, test.output)
} }
} }
@ -64,7 +64,7 @@ func TestFindNestedTextFile(t *testing.T) {
require.NoError(t, err) require.NoError(t, err)
matches := g1.Match() matches := g1.Match()
assert.Len(t, matches, 1) require.Len(t, matches, 1)
} }
func getTestdataDir() string { func getTestdataDir() string {