still unfinished logparser changes
This commit is contained in:
parent
bfc13a744b
commit
67db14332f
|
@ -3,9 +3,7 @@
|
|||
package logparser
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log"
|
||||
"reflect"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
|
@ -35,9 +33,10 @@ type logEntry struct {
|
|||
|
||||
// LogParserPlugin is the primary struct to implement the interface for logparser plugin
|
||||
type LogParserPlugin struct {
|
||||
Files []string
|
||||
FromBeginning bool
|
||||
WatchMethod string
|
||||
Files []string
|
||||
FromBeginning bool
|
||||
WatchMethod string
|
||||
MeasurementName string `toml:"measurement"`
|
||||
|
||||
tailers map[string]*tail.Tail
|
||||
lines chan logEntry
|
||||
|
@ -48,7 +47,7 @@ type LogParserPlugin struct {
|
|||
|
||||
sync.Mutex
|
||||
|
||||
GrokParser *parsers.Parser `toml:"grok"`
|
||||
GrokParser parsers.Parser `toml:"grok"`
|
||||
|
||||
Patterns []string
|
||||
NamedPatterns []string
|
||||
|
@ -146,38 +145,11 @@ func (l *LogParserPlugin) Start(acc telegraf.Accumulator) error {
|
|||
DataFormat: "grok",
|
||||
}
|
||||
var err error
|
||||
*l.GrokParser, err = parsers.NewParser(config)
|
||||
l.GrokParser, err = parsers.NewParser(config)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
s := reflect.ValueOf(l).Elem()
|
||||
for i := 0; i < s.NumField(); i++ {
|
||||
f := s.Field(i)
|
||||
log.Printf("got field %v: %v", i, f)
|
||||
if !f.CanInterface() {
|
||||
continue
|
||||
}
|
||||
|
||||
if lpPlugin, ok := f.Interface().(parsers.Parser); ok {
|
||||
if reflect.ValueOf(lpPlugin).IsNil() {
|
||||
continue
|
||||
}
|
||||
l.parsers = append(l.parsers, lpPlugin)
|
||||
}
|
||||
}
|
||||
|
||||
if len(l.parsers) == 0 {
|
||||
return fmt.Errorf("logparser input plugin: no parser defined")
|
||||
}
|
||||
|
||||
// //compile log parser patterns:
|
||||
// for _, parser := range l.parsers {
|
||||
// if err := parser.Compile(); err != nil {
|
||||
// return err
|
||||
// }
|
||||
// }
|
||||
|
||||
l.wg.Add(1)
|
||||
go l.parser()
|
||||
|
||||
|
@ -284,18 +256,17 @@ func (l *LogParserPlugin) parser() {
|
|||
continue
|
||||
}
|
||||
}
|
||||
for _, parser := range l.parsers {
|
||||
m, err = parser.ParseLine(entry.line)
|
||||
if err == nil {
|
||||
if m != nil {
|
||||
tags := m.Tags()
|
||||
tags["path"] = entry.path
|
||||
l.acc.AddFields(m.Name(), m.Fields(), tags, m.Time())
|
||||
}
|
||||
} else {
|
||||
log.Println("E! Error parsing log line: " + err.Error())
|
||||
m, err = l.GrokParser.ParseLine(entry.line)
|
||||
if err == nil {
|
||||
if m != nil {
|
||||
tags := m.Tags()
|
||||
tags["path"] = entry.path
|
||||
l.acc.AddFields(l.MeasurementName, m.Fields(), tags, m.Time())
|
||||
}
|
||||
} else {
|
||||
log.Println("E! Error parsing log line: " + err.Error())
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
@ -2,6 +2,7 @@ package logparser
|
|||
|
||||
import (
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"os"
|
||||
"runtime"
|
||||
"strings"
|
||||
|
@ -36,7 +37,7 @@ func TestGrokParseLogFilesNonExistPattern(t *testing.T) {
|
|||
logparser := &LogParserPlugin{
|
||||
FromBeginning: true,
|
||||
Files: []string{thisdir + "grok/testdata/*.log"},
|
||||
GrokParser: &p,
|
||||
GrokParser: p,
|
||||
}
|
||||
|
||||
acc := testutil.Accumulator{}
|
||||
|
@ -46,26 +47,23 @@ func TestGrokParseLogFilesNonExistPattern(t *testing.T) {
|
|||
|
||||
func TestGrokParseLogFiles(t *testing.T) {
|
||||
thisdir := getCurrentDir()
|
||||
c := parsers.Config{
|
||||
Patterns: []string{"%{TEST_LOG_A}", "%{TEST_LOG_B}"},
|
||||
CustomPatternFiles: []string{thisdir + "grok/testdata/test-patterns"},
|
||||
DataFormat: "grok",
|
||||
}
|
||||
p, _ := parsers.NewParser(&c)
|
||||
|
||||
logparser := &LogParserPlugin{
|
||||
FromBeginning: true,
|
||||
Files: []string{thisdir + "grok/testdata/*.log"},
|
||||
GrokParser: &p,
|
||||
Patterns: []string{"%{TEST_LOG_A}", "%{TEST_LOG_B}"},
|
||||
CustomPatternFiles: []string{thisdir + "grok/testdata/test-patterns"},
|
||||
FromBeginning: true,
|
||||
Files: []string{thisdir + "grok/testdata/*.log"},
|
||||
MeasurementName: "logparser_grok",
|
||||
}
|
||||
|
||||
acc := testutil.Accumulator{}
|
||||
assert.NoError(t, logparser.Start(&acc))
|
||||
|
||||
//acc.Wait(2)
|
||||
acc.Wait(2)
|
||||
|
||||
logparser.Stop()
|
||||
|
||||
log.Printf("metric[0] %v, tags: %v, fields: %v", acc.Metrics[0].Measurement, acc.Metrics[0].Tags, acc.Metrics[0].Fields)
|
||||
acc.AssertContainsTaggedFields(t, "logparser_grok",
|
||||
map[string]interface{}{
|
||||
"clientip": "192.168.1.1",
|
||||
|
@ -95,17 +93,13 @@ func TestGrokParseLogFilesAppearLater(t *testing.T) {
|
|||
assert.NoError(t, err)
|
||||
|
||||
thisdir := getCurrentDir()
|
||||
c := &parsers.Config{
|
||||
Patterns: []string{"%{TEST_LOG_A}", "%{TEST_LOG_B}"},
|
||||
CustomPatternFiles: []string{thisdir + "grok/testdata/test-patterns"},
|
||||
DataFormat: "grok",
|
||||
}
|
||||
p, err := parsers.NewParser(c)
|
||||
|
||||
logparser := &LogParserPlugin{
|
||||
FromBeginning: true,
|
||||
Files: []string{emptydir + "/*.log"},
|
||||
GrokParser: &p,
|
||||
FromBeginning: true,
|
||||
Files: []string{emptydir + "/*.log"},
|
||||
Patterns: []string{"%{TEST_LOG_A}", "%{TEST_LOG_B}"},
|
||||
CustomPatternFiles: []string{thisdir + "grok/testdata/test-patterns"},
|
||||
MeasurementName: "logparser_grok",
|
||||
}
|
||||
|
||||
acc := testutil.Accumulator{}
|
||||
|
@ -136,17 +130,13 @@ func TestGrokParseLogFilesAppearLater(t *testing.T) {
|
|||
// pattern available for test_b.log
|
||||
func TestGrokParseLogFilesOneBad(t *testing.T) {
|
||||
thisdir := getCurrentDir()
|
||||
c := &parsers.Config{
|
||||
Patterns: []string{"%{TEST_LOG_A}", "%{TEST_LOG_BAD}"},
|
||||
CustomPatternFiles: []string{thisdir + "grok/testdata/test-patterns"},
|
||||
DataFormat: "grok",
|
||||
}
|
||||
p, _ := parsers.NewParser(c)
|
||||
|
||||
logparser := &LogParserPlugin{
|
||||
FromBeginning: true,
|
||||
Files: []string{thisdir + "grok/testdata/test_a.log"},
|
||||
GrokParser: &p,
|
||||
FromBeginning: true,
|
||||
Files: []string{thisdir + "grok/testdata/test_a.log"},
|
||||
Patterns: []string{"%{TEST_LOG_A}", "%{TEST_LOG_BAD}"},
|
||||
CustomPatternFiles: []string{thisdir + "grok/testdata/test-patterns"},
|
||||
MeasurementName: "logparser_grok",
|
||||
}
|
||||
|
||||
acc := testutil.Accumulator{}
|
||||
|
|
Loading…
Reference in New Issue