561 lines
15 KiB
Go
561 lines
15 KiB
Go
package grok
|
|
|
|
import (
|
|
"bufio"
|
|
"bytes"
|
|
"fmt"
|
|
"log"
|
|
"os"
|
|
"regexp"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/influxdata/telegraf"
|
|
"github.com/influxdata/telegraf/metric"
|
|
"github.com/vjeantet/grok"
|
|
)
|
|
|
|
var timeLayouts = map[string]string{
|
|
"ts-ansic": "Mon Jan _2 15:04:05 2006",
|
|
"ts-unix": "Mon Jan _2 15:04:05 MST 2006",
|
|
"ts-ruby": "Mon Jan 02 15:04:05 -0700 2006",
|
|
"ts-rfc822": "02 Jan 06 15:04 MST",
|
|
"ts-rfc822z": "02 Jan 06 15:04 -0700", // RFC822 with numeric zone
|
|
"ts-rfc850": "Monday, 02-Jan-06 15:04:05 MST",
|
|
"ts-rfc1123": "Mon, 02 Jan 2006 15:04:05 MST",
|
|
"ts-rfc1123z": "Mon, 02 Jan 2006 15:04:05 -0700", // RFC1123 with numeric zone
|
|
"ts-rfc3339": "2006-01-02T15:04:05Z07:00",
|
|
"ts-rfc3339nano": "2006-01-02T15:04:05.999999999Z07:00",
|
|
"ts-httpd": "02/Jan/2006:15:04:05 -0700",
|
|
// These four are not exactly "layouts", but they are special cases that
|
|
// will get handled in the ParseLine function.
|
|
"ts-epoch": "EPOCH",
|
|
"ts-epochnano": "EPOCH_NANO",
|
|
"ts-epochmilli": "EPOCH_MILLI",
|
|
"ts-syslog": "SYSLOG_TIMESTAMP",
|
|
"ts": "GENERIC_TIMESTAMP", // try parsing all known timestamp layouts.
|
|
}
|
|
|
|
const (
|
|
MEASUREMENT = "measurement"
|
|
INT = "int"
|
|
TAG = "tag"
|
|
FLOAT = "float"
|
|
STRING = "string"
|
|
DURATION = "duration"
|
|
DROP = "drop"
|
|
EPOCH = "EPOCH"
|
|
EPOCH_MILLI = "EPOCH_MILLI"
|
|
EPOCH_NANO = "EPOCH_NANO"
|
|
SYSLOG_TIMESTAMP = "SYSLOG_TIMESTAMP"
|
|
GENERIC_TIMESTAMP = "GENERIC_TIMESTAMP"
|
|
)
|
|
|
|
var (
|
|
// matches named captures that contain a modifier.
|
|
// ie,
|
|
// %{NUMBER:bytes:int}
|
|
// %{IPORHOST:clientip:tag}
|
|
// %{HTTPDATE:ts1:ts-http}
|
|
// %{HTTPDATE:ts2:ts-"02 Jan 06 15:04"}
|
|
modifierRe = regexp.MustCompile(`%{\w+:(\w+):(ts-".+"|t?s?-?\w+)}`)
|
|
// matches a plain pattern name. ie, %{NUMBER}
|
|
patternOnlyRe = regexp.MustCompile(`%{(\w+)}`)
|
|
)
|
|
|
|
// Parser is the primary struct to handle and grok-patterns defined in the config toml
|
|
type Parser struct {
|
|
Patterns []string
|
|
// namedPatterns is a list of internally-assigned names to the patterns
|
|
// specified by the user in Patterns.
|
|
// They will look like:
|
|
// GROK_INTERNAL_PATTERN_0, GROK_INTERNAL_PATTERN_1, etc.
|
|
NamedPatterns []string
|
|
CustomPatterns string
|
|
CustomPatternFiles []string
|
|
Measurement string
|
|
DefaultTags map[string]string
|
|
|
|
// Timezone is an optional component to help render log dates to
|
|
// your chosen zone.
|
|
// Default: "" which renders UTC
|
|
// Options are as follows:
|
|
// 1. Local -- interpret based on machine localtime
|
|
// 2. "America/Chicago" -- Unix TZ values like those found in https://en.wikipedia.org/wiki/List_of_tz_database_time_zones
|
|
// 3. UTC -- or blank/unspecified, will return timestamp in UTC
|
|
Timezone string
|
|
loc *time.Location
|
|
|
|
// UniqueTimestamp when set to "disable", timestamp will not incremented if there is a duplicate.
|
|
UniqueTimestamp string
|
|
|
|
// typeMap is a map of patterns -> capture name -> modifier,
|
|
// ie, {
|
|
// "%{TESTLOG}":
|
|
// {
|
|
// "bytes": "int",
|
|
// "clientip": "tag"
|
|
// }
|
|
// }
|
|
typeMap map[string]map[string]string
|
|
// tsMap is a map of patterns -> capture name -> timestamp layout.
|
|
// ie, {
|
|
// "%{TESTLOG}":
|
|
// {
|
|
// "httptime": "02/Jan/2006:15:04:05 -0700"
|
|
// }
|
|
// }
|
|
tsMap map[string]map[string]string
|
|
// patterns is a map of all of the parsed patterns from CustomPatterns
|
|
// and CustomPatternFiles.
|
|
// ie, {
|
|
// "DURATION": "%{NUMBER}[nuµm]?s"
|
|
// "RESPONSE_CODE": "%{NUMBER:rc:tag}"
|
|
// }
|
|
patterns map[string]string
|
|
// foundTsLayouts is a slice of timestamp patterns that have been found
|
|
// in the log lines. This slice gets updated if the user uses the generic
|
|
// 'ts' modifier for timestamps. This slice is checked first for matches,
|
|
// so that previously-matched layouts get priority over all other timestamp
|
|
// layouts.
|
|
foundTsLayouts []string
|
|
|
|
timeFunc func() time.Time
|
|
g *grok.Grok
|
|
tsModder *tsModder
|
|
}
|
|
|
|
// Compile is a bound method to Parser which will process the options for our parser
|
|
func (p *Parser) Compile() error {
|
|
p.typeMap = make(map[string]map[string]string)
|
|
p.tsMap = make(map[string]map[string]string)
|
|
p.patterns = make(map[string]string)
|
|
p.tsModder = &tsModder{}
|
|
var err error
|
|
p.g, err = grok.NewWithConfig(&grok.Config{NamedCapturesOnly: true})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if p.UniqueTimestamp == "" {
|
|
p.UniqueTimestamp = "auto"
|
|
}
|
|
|
|
// Give Patterns fake names so that they can be treated as named
|
|
// "custom patterns"
|
|
p.NamedPatterns = make([]string, 0, len(p.Patterns))
|
|
for i, pattern := range p.Patterns {
|
|
pattern = strings.TrimSpace(pattern)
|
|
if pattern == "" {
|
|
continue
|
|
}
|
|
name := fmt.Sprintf("GROK_INTERNAL_PATTERN_%d", i)
|
|
p.CustomPatterns += "\n" + name + " " + pattern + "\n"
|
|
p.NamedPatterns = append(p.NamedPatterns, "%{"+name+"}")
|
|
}
|
|
|
|
if len(p.NamedPatterns) == 0 {
|
|
return fmt.Errorf("pattern required")
|
|
}
|
|
|
|
// Combine user-supplied CustomPatterns with DEFAULT_PATTERNS and parse
|
|
// them together as the same type of pattern.
|
|
p.CustomPatterns = DEFAULT_PATTERNS + p.CustomPatterns
|
|
if len(p.CustomPatterns) != 0 {
|
|
scanner := bufio.NewScanner(strings.NewReader(p.CustomPatterns))
|
|
p.addCustomPatterns(scanner)
|
|
}
|
|
|
|
// Parse any custom pattern files supplied.
|
|
for _, filename := range p.CustomPatternFiles {
|
|
file, fileErr := os.Open(filename)
|
|
if fileErr != nil {
|
|
return fileErr
|
|
}
|
|
|
|
scanner := bufio.NewScanner(bufio.NewReader(file))
|
|
p.addCustomPatterns(scanner)
|
|
}
|
|
|
|
p.loc, err = time.LoadLocation(p.Timezone)
|
|
if err != nil {
|
|
log.Printf("W! improper timezone supplied (%s), setting loc to UTC", p.Timezone)
|
|
p.loc, _ = time.LoadLocation("UTC")
|
|
}
|
|
|
|
if p.timeFunc == nil {
|
|
p.timeFunc = time.Now
|
|
}
|
|
|
|
return p.compileCustomPatterns()
|
|
}
|
|
|
|
// ParseLine is the primary function to process individual lines, returning the metrics
|
|
func (p *Parser) ParseLine(line string) (telegraf.Metric, error) {
|
|
var err error
|
|
// values are the parsed fields from the log line
|
|
var values map[string]string
|
|
// the matching pattern string
|
|
var patternName string
|
|
for _, pattern := range p.NamedPatterns {
|
|
if values, err = p.g.Parse(pattern, line); err != nil {
|
|
return nil, err
|
|
}
|
|
if len(values) != 0 {
|
|
patternName = pattern
|
|
break
|
|
}
|
|
}
|
|
|
|
if len(values) == 0 {
|
|
log.Printf("D! Grok no match found for: %q", line)
|
|
return nil, nil
|
|
}
|
|
|
|
fields := make(map[string]interface{})
|
|
tags := make(map[string]string)
|
|
|
|
//add default tags
|
|
for k, v := range p.DefaultTags {
|
|
tags[k] = v
|
|
}
|
|
|
|
timestamp := time.Now()
|
|
for k, v := range values {
|
|
if k == "" || v == "" {
|
|
continue
|
|
}
|
|
// t is the modifier of the field
|
|
var t string
|
|
// check if pattern has some modifiers
|
|
if types, ok := p.typeMap[patternName]; ok {
|
|
t = types[k]
|
|
}
|
|
// if we didn't find a modifier, check if we have a timestamp layout
|
|
if t == "" {
|
|
if ts, ok := p.tsMap[patternName]; ok {
|
|
// check if the modifier is a timestamp layout
|
|
if layout, ok := ts[k]; ok {
|
|
t = layout
|
|
}
|
|
}
|
|
}
|
|
// if we didn't find a type OR timestamp modifier, assume string
|
|
if t == "" {
|
|
t = STRING
|
|
}
|
|
|
|
switch t {
|
|
case MEASUREMENT:
|
|
p.Measurement = v
|
|
case INT:
|
|
iv, err := strconv.ParseInt(v, 0, 64)
|
|
if err != nil {
|
|
log.Printf("E! Error parsing %s to int: %s", v, err)
|
|
} else {
|
|
fields[k] = iv
|
|
}
|
|
case FLOAT:
|
|
fv, err := strconv.ParseFloat(v, 64)
|
|
if err != nil {
|
|
log.Printf("E! Error parsing %s to float: %s", v, err)
|
|
} else {
|
|
fields[k] = fv
|
|
}
|
|
case DURATION:
|
|
d, err := time.ParseDuration(v)
|
|
if err != nil {
|
|
log.Printf("E! Error parsing %s to duration: %s", v, err)
|
|
} else {
|
|
fields[k] = int64(d)
|
|
}
|
|
case TAG:
|
|
tags[k] = v
|
|
case STRING:
|
|
fields[k] = v
|
|
case EPOCH:
|
|
parts := strings.SplitN(v, ".", 2)
|
|
if len(parts) == 0 {
|
|
log.Printf("E! Error parsing %s to timestamp: %s", v, err)
|
|
break
|
|
}
|
|
|
|
sec, err := strconv.ParseInt(parts[0], 10, 64)
|
|
if err != nil {
|
|
log.Printf("E! Error parsing %s to timestamp: %s", v, err)
|
|
break
|
|
}
|
|
ts := time.Unix(sec, 0)
|
|
|
|
if len(parts) == 2 {
|
|
padded := fmt.Sprintf("%-9s", parts[1])
|
|
nsString := strings.Replace(padded[:9], " ", "0", -1)
|
|
nanosec, err := strconv.ParseInt(nsString, 10, 64)
|
|
if err != nil {
|
|
log.Printf("E! Error parsing %s to timestamp: %s", v, err)
|
|
break
|
|
}
|
|
ts = ts.Add(time.Duration(nanosec) * time.Nanosecond)
|
|
}
|
|
timestamp = ts
|
|
case EPOCH_MILLI:
|
|
ms, err := strconv.ParseInt(v, 10, 64)
|
|
if err != nil {
|
|
log.Printf("E! Error parsing %s to int: %s", v, err)
|
|
} else {
|
|
timestamp = time.Unix(0, ms*int64(time.Millisecond))
|
|
}
|
|
case EPOCH_NANO:
|
|
iv, err := strconv.ParseInt(v, 10, 64)
|
|
if err != nil {
|
|
log.Printf("E! Error parsing %s to int: %s", v, err)
|
|
} else {
|
|
timestamp = time.Unix(0, iv)
|
|
}
|
|
case SYSLOG_TIMESTAMP:
|
|
ts, err := time.ParseInLocation(time.Stamp, v, p.loc)
|
|
if err == nil {
|
|
if ts.Year() == 0 {
|
|
ts = ts.AddDate(timestamp.Year(), 0, 0)
|
|
}
|
|
timestamp = ts
|
|
} else {
|
|
log.Printf("E! Error parsing %s to time layout [%s]: %s", v, t, err)
|
|
}
|
|
case GENERIC_TIMESTAMP:
|
|
var foundTs bool
|
|
// first try timestamp layouts that we've already found
|
|
for _, layout := range p.foundTsLayouts {
|
|
ts, err := time.ParseInLocation(layout, v, p.loc)
|
|
if err == nil {
|
|
timestamp = ts
|
|
foundTs = true
|
|
break
|
|
}
|
|
}
|
|
// if we haven't found a timestamp layout yet, try all timestamp
|
|
// layouts.
|
|
if !foundTs {
|
|
for _, layout := range timeLayouts {
|
|
ts, err := time.ParseInLocation(layout, v, p.loc)
|
|
if err == nil {
|
|
timestamp = ts
|
|
foundTs = true
|
|
p.foundTsLayouts = append(p.foundTsLayouts, layout)
|
|
break
|
|
}
|
|
}
|
|
}
|
|
// if we still haven't found a timestamp layout, log it and we will
|
|
// just use time.Now()
|
|
if !foundTs {
|
|
log.Printf("E! Error parsing timestamp [%s], could not find any "+
|
|
"suitable time layouts.", v)
|
|
}
|
|
case DROP:
|
|
// goodbye!
|
|
default:
|
|
v = strings.Replace(v, ",", ".", -1)
|
|
ts, err := time.ParseInLocation(t, v, p.loc)
|
|
if err == nil {
|
|
if ts.Year() == 0 {
|
|
ts = ts.AddDate(timestamp.Year(), 0, 0)
|
|
}
|
|
timestamp = ts
|
|
} else {
|
|
log.Printf("E! Error parsing %s to time layout [%s]: %s", v, t, err)
|
|
}
|
|
}
|
|
}
|
|
|
|
if p.UniqueTimestamp != "auto" {
|
|
return metric.New(p.Measurement, tags, fields, timestamp)
|
|
}
|
|
|
|
return metric.New(p.Measurement, tags, fields, p.tsModder.tsMod(timestamp))
|
|
}
|
|
|
|
func (p *Parser) Parse(buf []byte) ([]telegraf.Metric, error) {
|
|
|
|
metrics := make([]telegraf.Metric, 0)
|
|
|
|
scanner := bufio.NewScanner(bytes.NewReader(buf))
|
|
for scanner.Scan() {
|
|
line := scanner.Text()
|
|
m, err := p.ParseLine(line)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
if m == nil {
|
|
continue
|
|
}
|
|
metrics = append(metrics, m)
|
|
}
|
|
|
|
return metrics, nil
|
|
}
|
|
|
|
func (p *Parser) SetDefaultTags(tags map[string]string) {
|
|
p.DefaultTags = tags
|
|
}
|
|
|
|
func (p *Parser) addCustomPatterns(scanner *bufio.Scanner) {
|
|
for scanner.Scan() {
|
|
line := strings.TrimSpace(scanner.Text())
|
|
if len(line) > 0 && line[0] != '#' {
|
|
names := strings.SplitN(line, " ", 2)
|
|
p.patterns[names[0]] = names[1]
|
|
}
|
|
}
|
|
}
|
|
|
|
func (p *Parser) compileCustomPatterns() error {
|
|
var err error
|
|
// check if the pattern contains a subpattern that is already defined
|
|
// replace it with the subpattern for modifier inheritance.
|
|
for i := 0; i < 2; i++ {
|
|
for name, pattern := range p.patterns {
|
|
subNames := patternOnlyRe.FindAllStringSubmatch(pattern, -1)
|
|
for _, subName := range subNames {
|
|
if subPattern, ok := p.patterns[subName[1]]; ok {
|
|
pattern = strings.Replace(pattern, subName[0], subPattern, 1)
|
|
}
|
|
}
|
|
p.patterns[name] = pattern
|
|
}
|
|
}
|
|
|
|
// check if pattern contains modifiers. Parse them out if it does.
|
|
for name, pattern := range p.patterns {
|
|
if modifierRe.MatchString(pattern) {
|
|
// this pattern has modifiers, so parse out the modifiers
|
|
pattern, err = p.parseTypedCaptures(name, pattern)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
p.patterns[name] = pattern
|
|
}
|
|
}
|
|
|
|
return p.g.AddPatternsFromMap(p.patterns)
|
|
}
|
|
|
|
// parseTypedCaptures parses the capture modifiers, and then deletes the
|
|
// modifier from the line so that it is a valid "grok" pattern again.
|
|
// ie,
|
|
// %{NUMBER:bytes:int} => %{NUMBER:bytes} (stores %{NUMBER}->bytes->int)
|
|
// %{IPORHOST:clientip:tag} => %{IPORHOST:clientip} (stores %{IPORHOST}->clientip->tag)
|
|
func (p *Parser) parseTypedCaptures(name, pattern string) (string, error) {
|
|
matches := modifierRe.FindAllStringSubmatch(pattern, -1)
|
|
|
|
// grab the name of the capture pattern
|
|
patternName := "%{" + name + "}"
|
|
// create type map for this pattern
|
|
p.typeMap[patternName] = make(map[string]string)
|
|
p.tsMap[patternName] = make(map[string]string)
|
|
|
|
// boolean to verify that each pattern only has a single ts- data type.
|
|
hasTimestamp := false
|
|
for _, match := range matches {
|
|
// regex capture 1 is the name of the capture
|
|
// regex capture 2 is the modifier of the capture
|
|
if strings.HasPrefix(match[2], "ts") {
|
|
if hasTimestamp {
|
|
return pattern, fmt.Errorf("logparser pattern compile error: "+
|
|
"Each pattern is allowed only one named "+
|
|
"timestamp data type. pattern: %s", pattern)
|
|
}
|
|
if layout, ok := timeLayouts[match[2]]; ok {
|
|
// built-in time format
|
|
p.tsMap[patternName][match[1]] = layout
|
|
} else {
|
|
// custom time format
|
|
p.tsMap[patternName][match[1]] = strings.TrimSuffix(strings.TrimPrefix(match[2], `ts-"`), `"`)
|
|
}
|
|
hasTimestamp = true
|
|
} else {
|
|
p.typeMap[patternName][match[1]] = match[2]
|
|
}
|
|
|
|
// the modifier is not a valid part of a "grok" pattern, so remove it
|
|
// from the pattern.
|
|
pattern = strings.Replace(pattern, ":"+match[2]+"}", "}", 1)
|
|
}
|
|
|
|
return pattern, nil
|
|
}
|
|
|
|
// tsModder is a struct for incrementing identical timestamps of log lines
|
|
// so that we don't push identical metrics that will get overwritten.
|
|
type tsModder struct {
|
|
dupe time.Time
|
|
last time.Time
|
|
incr time.Duration
|
|
incrn time.Duration
|
|
rollover time.Duration
|
|
}
|
|
|
|
// tsMod increments the given timestamp one unit more from the previous
|
|
// duplicate timestamp.
|
|
// the increment unit is determined as the next smallest time unit below the
|
|
// most significant time unit of ts.
|
|
// ie, if the input is at ms precision, it will increment it 1µs.
|
|
func (t *tsModder) tsMod(ts time.Time) time.Time {
|
|
if ts.IsZero() {
|
|
return ts
|
|
}
|
|
defer func() { t.last = ts }()
|
|
// don't mod the time if we don't need to
|
|
if t.last.IsZero() || ts.IsZero() {
|
|
t.incrn = 0
|
|
t.rollover = 0
|
|
return ts
|
|
}
|
|
if !ts.Equal(t.last) && !ts.Equal(t.dupe) {
|
|
t.incr = 0
|
|
t.incrn = 0
|
|
t.rollover = 0
|
|
return ts
|
|
}
|
|
if ts.Equal(t.last) {
|
|
t.dupe = ts
|
|
}
|
|
|
|
if ts.Equal(t.dupe) && t.incr == time.Duration(0) {
|
|
tsNano := ts.UnixNano()
|
|
|
|
d := int64(10)
|
|
counter := 1
|
|
for {
|
|
a := tsNano % d
|
|
if a > 0 {
|
|
break
|
|
}
|
|
d = d * 10
|
|
counter++
|
|
}
|
|
|
|
switch {
|
|
case counter <= 6:
|
|
t.incr = time.Nanosecond
|
|
case counter <= 9:
|
|
t.incr = time.Microsecond
|
|
case counter > 9:
|
|
t.incr = time.Millisecond
|
|
}
|
|
}
|
|
|
|
t.incrn++
|
|
if t.incrn == 999 && t.incr > time.Nanosecond {
|
|
t.rollover = t.incr * t.incrn
|
|
t.incrn = 1
|
|
t.incr = t.incr / 1000
|
|
if t.incr < time.Nanosecond {
|
|
t.incr = time.Nanosecond
|
|
}
|
|
}
|
|
return ts.Add(t.incr*t.incrn + t.rollover)
|
|
}
|