add grok as a top level parser

This commit is contained in:
Max U 2018-06-21 15:56:20 -07:00
parent 9c4b52256d
commit 4e24a1bbe3
3 changed files with 60 additions and 4 deletions

View File

@ -18,6 +18,12 @@ type Reader struct {
Tags []string Tags []string
Filenames []string Filenames []string
//for grok parser
Patterns []string
namedPatterns []string
CustomPatterns string
CustomPatternFiles []string
} }
const sampleConfig = `## Files to parse. const sampleConfig = `## Files to parse.
@ -63,13 +69,15 @@ func (r *Reader) compileParser() {
log.Printf("E! No data_format specified") log.Printf("E! No data_format specified")
return return
} }
if r.DataFormat == "grok" {
log.Printf("Grok isn't supported yet")
return
}
r.ParserConfig = parsers.Config{ r.ParserConfig = parsers.Config{
DataFormat: r.DataFormat, DataFormat: r.DataFormat,
TagKeys: r.Tags, TagKeys: r.Tags,
//grok settings
Patterns: r.Patterns,
NamedPatterns: r.namedPatterns,
CustomPatterns: r.CustomPatterns,
CustomPatternFiles: r.CustomPatternFiles,
} }
nParser, err := parsers.NewParser(&r.ParserConfig) nParser, err := parsers.NewParser(&r.ParserConfig)
if err != nil { if err != nil {

View File

@ -35,6 +35,23 @@ func TestJSONParserCompile(t *testing.T) {
assert.Equal(t, 5, len(acc.Metrics[0].Fields)) assert.Equal(t, 5, len(acc.Metrics[0].Fields))
} }
func TestGrokParser(t *testing.T) {
testDir := getPluginDir()
var acc testutil.Accumulator
r := Reader{
Filepaths: []string{testDir + "/reader/testfiles/grok_a.log"},
DataFormat: "grok",
Patterns: []string{"%{COMMON_LOG_FORMAT}"},
}
r.compileParser()
err := r.Gather(&acc)
log.Printf("err: %v", err)
log.Printf("metric[0]_tags: %v, metric[0]_fields: %v", acc.Metrics[0].Tags, acc.Metrics[0].Fields)
log.Printf("metric[1]_tags: %v, metric[1]_fields: %v", acc.Metrics[1].Tags, acc.Metrics[1].Fields)
t.Error()
}
func getPluginDir() string { func getPluginDir() string {
_, filename, _, _ := runtime.Caller(1) _, filename, _, _ := runtime.Caller(1)
return strings.Replace(filename, "/reader/reader_test.go", "", 1) return strings.Replace(filename, "/reader/reader_test.go", "", 1)

View File

@ -8,6 +8,7 @@ import (
"github.com/influxdata/telegraf/plugins/parsers/collectd" "github.com/influxdata/telegraf/plugins/parsers/collectd"
"github.com/influxdata/telegraf/plugins/parsers/dropwizard" "github.com/influxdata/telegraf/plugins/parsers/dropwizard"
"github.com/influxdata/telegraf/plugins/parsers/graphite" "github.com/influxdata/telegraf/plugins/parsers/graphite"
"github.com/influxdata/telegraf/plugins/parsers/grok"
"github.com/influxdata/telegraf/plugins/parsers/influx" "github.com/influxdata/telegraf/plugins/parsers/influx"
"github.com/influxdata/telegraf/plugins/parsers/json" "github.com/influxdata/telegraf/plugins/parsers/json"
"github.com/influxdata/telegraf/plugins/parsers/nagios" "github.com/influxdata/telegraf/plugins/parsers/nagios"
@ -87,6 +88,12 @@ type Config struct {
// an optional map containing tag names as keys and json paths to retrieve the tag values from as values // an optional map containing tag names as keys and json paths to retrieve the tag values from as values
// used if TagsPath is empty or doesn't return any tags // used if TagsPath is empty or doesn't return any tags
DropwizardTagPathsMap map[string]string DropwizardTagPathsMap map[string]string
//grok patterns
Patterns []string
NamedPatterns []string
CustomPatterns string
CustomPatternFiles []string
} }
// NewParser returns a Parser interface based on the given config. // NewParser returns a Parser interface based on the given config.
@ -120,12 +127,36 @@ func NewParser(config *Config) (Parser, error) {
config.DefaultTags, config.DefaultTags,
config.Separator, config.Separator,
config.Templates) config.Templates)
case "grok":
parser, err = NewGrokParser(
config.MetricName,
config.Patterns,
config.NamedPatterns,
config.CustomPatterns,
config.CustomPatternFiles)
default: default:
err = fmt.Errorf("Invalid data format: %s", config.DataFormat) err = fmt.Errorf("Invalid data format: %s", config.DataFormat)
} }
return parser, err return parser, err
} }
func NewGrokParser(metricName string,
patterns []string,
nPatterns []string,
cPatterns string,
cPatternFiles []string) (Parser, error) {
parser := grok.Parser{
Measurement: metricName,
Patterns: patterns,
NamedPatterns: nPatterns,
CustomPatterns: cPatterns,
CustomPatternFiles: cPatternFiles,
}
parser.Compile()
return &parser, nil
}
func NewJSONParser( func NewJSONParser(
metricName string, metricName string,
tagKeys []string, tagKeys []string,