Improve performance of globpath with some patterns (#4836)

This commit is contained in:
Samuel-BF 2018-10-12 23:48:11 +02:00 committed by Daniel Nelson
parent a2ac9115b3
commit f259229a35
2 changed files with 29 additions and 55 deletions

View File

@ -15,8 +15,8 @@ type GlobPath struct {
path string
hasMeta bool
hasSuperMeta bool
rootGlob string
g glob.Glob
root string
}
func Compile(path string) (*GlobPath, error) {
@ -27,23 +27,25 @@ func Compile(path string) (*GlobPath, error) {
}
// if there are no glob meta characters in the path, don't bother compiling
// a glob object or finding the root directory. (see short-circuit in Match)
// a glob object
if !out.hasMeta || !out.hasSuperMeta {
return &out, nil
}
// find the root elements of the object path, the entry point for recursion
// when you have a super-meta in your path (which are :
// glob(/your/expression/until/first/star/of/super-meta))
out.rootGlob = path[:strings.Index(path, "**")+1]
var err error
if out.g, err = glob.Compile(path, os.PathSeparator); err != nil {
return nil, err
}
// Get the root directory for this filepath
out.root = findRootDir(path)
return &out, nil
}
func (g *GlobPath) Match() map[string]os.FileInfo {
if !g.hasMeta {
out := make(map[string]os.FileInfo)
if !g.hasMeta {
info, err := os.Stat(g.path)
if err == nil {
out[g.path] = info
@ -51,7 +53,6 @@ func (g *GlobPath) Match() map[string]os.FileInfo {
return out
}
if !g.hasSuperMeta {
out := make(map[string]os.FileInfo)
files, _ := filepath.Glob(g.path)
for _, file := range files {
info, err := os.Stat(file)
@ -61,46 +62,19 @@ func (g *GlobPath) Match() map[string]os.FileInfo {
}
return out
}
return walkFilePath(g.root, g.g)
roots, err := filepath.Glob(g.rootGlob)
if err != nil {
return out
}
// walk the filepath from the given root and return a list of files that match
// the given glob.
func walkFilePath(root string, g glob.Glob) map[string]os.FileInfo {
matchedFiles := make(map[string]os.FileInfo)
walkfn := func(path string, info os.FileInfo, _ error) error {
if g.Match(path) {
matchedFiles[path] = info
if g.g.Match(path) {
out[path] = info
}
return nil
}
filepath.Walk(root, walkfn)
return matchedFiles
}
// find the root dir of the given path (could include globs).
// ie:
// /var/log/telegraf.conf -> /var/log
// /home/** -> /home
// /home/*/** -> /home
// /lib/share/*/*/**.txt -> /lib/share
func findRootDir(path string) string {
pathItems := strings.Split(path, sepStr)
out := sepStr
for i, item := range pathItems {
if i == len(pathItems)-1 {
break
}
if item == "" {
continue
}
if hasMeta(item) {
break
}
out += item + sepStr
}
if out != "/" {
out = strings.TrimSuffix(out, "/")
for _, root := range roots {
filepath.Walk(root, walkfn)
}
return out
}

View File

@ -6,7 +6,6 @@ import (
"strings"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
@ -29,31 +28,32 @@ func TestCompileAndMatch(t *testing.T) {
require.NoError(t, err)
matches := g1.Match()
assert.Len(t, matches, 6)
require.Len(t, matches, 6)
matches = g2.Match()
assert.Len(t, matches, 2)
require.Len(t, matches, 2)
matches = g3.Match()
assert.Len(t, matches, 1)
require.Len(t, matches, 1)
matches = g4.Match()
assert.Len(t, matches, 0)
require.Len(t, matches, 0)
matches = g5.Match()
assert.Len(t, matches, 0)
require.Len(t, matches, 0)
}
func TestFindRootDir(t *testing.T) {
func TestRootGlob(t *testing.T) {
dir := getTestdataDir()
tests := []struct {
input string
output string
}{
{"/var/log/telegraf.conf", "/var/log"},
{"/home/**", "/home"},
{"/home/*/**", "/home"},
{"/lib/share/*/*/**.txt", "/lib/share"},
{dir + "/**", dir + "/*"},
{dir + "/nested?/**", dir + "/nested?/*"},
{dir + "/ne**/nest*", dir + "/ne*"},
{dir + "/nested?/*", ""},
}
for _, test := range tests {
actual := findRootDir(test.input)
assert.Equal(t, test.output, actual)
actual, _ := Compile(test.input)
require.Equal(t, actual.rootGlob, test.output)
}
}
@ -64,7 +64,7 @@ func TestFindNestedTextFile(t *testing.T) {
require.NoError(t, err)
matches := g1.Match()
assert.Len(t, matches, 1)
require.Len(t, matches, 1)
}
func getTestdataDir() string {