2016-06-02 17:47:15 +00:00
|
|
|
package grok
|
|
|
|
|
|
|
|
|
|
// THIS SHOULD BE KEPT IN-SYNC WITH patterns/influx-patterns
|
|
|
|
|
const DEFAULT_PATTERNS = `
|
|
|
|
|
# Captures are a slightly modified version of logstash "grok" patterns, with
|
|
|
|
|
# the format %{<capture syntax>[:<semantic name>][:<modifier>]}
|
|
|
|
|
# By default all named captures are converted into string fields.
|
|
|
|
|
# Modifiers can be used to convert captures to other types or tags.
|
|
|
|
|
# Timestamp modifiers can be used to convert captures to the timestamp of the
|
|
|
|
|
# parsed metric.
|
|
|
|
|
|
|
|
|
|
# View logstash grok pattern docs here:
|
|
|
|
|
# https://www.elastic.co/guide/en/logstash/current/plugins-filters-grok.html
|
|
|
|
|
# All default logstash patterns are supported, these can be viewed here:
|
|
|
|
|
# https://github.com/logstash-plugins/logstash-patterns-core/blob/master/patterns/grok-patterns
|
|
|
|
|
|
|
|
|
|
# Available modifiers:
|
|
|
|
|
# string (default if nothing is specified)
|
|
|
|
|
# int
|
|
|
|
|
# float
|
|
|
|
|
# duration (ie, 5.23ms gets converted to int nanoseconds)
|
|
|
|
|
# tag (converts the field into a tag)
|
|
|
|
|
# drop (drops the field completely)
|
|
|
|
|
# Timestamp modifiers:
|
|
|
|
|
# ts-ansic ("Mon Jan _2 15:04:05 2006")
|
|
|
|
|
# ts-unix ("Mon Jan _2 15:04:05 MST 2006")
|
|
|
|
|
# ts-ruby ("Mon Jan 02 15:04:05 -0700 2006")
|
|
|
|
|
# ts-rfc822 ("02 Jan 06 15:04 MST")
|
|
|
|
|
# ts-rfc822z ("02 Jan 06 15:04 -0700")
|
|
|
|
|
# ts-rfc850 ("Monday, 02-Jan-06 15:04:05 MST")
|
|
|
|
|
# ts-rfc1123 ("Mon, 02 Jan 2006 15:04:05 MST")
|
|
|
|
|
# ts-rfc1123z ("Mon, 02 Jan 2006 15:04:05 -0700")
|
|
|
|
|
# ts-rfc3339 ("2006-01-02T15:04:05Z07:00")
|
|
|
|
|
# ts-rfc3339nano ("2006-01-02T15:04:05.999999999Z07:00")
|
|
|
|
|
# ts-httpd ("02/Jan/2006:15:04:05 -0700")
|
|
|
|
|
# ts-epoch (seconds since unix epoch)
|
|
|
|
|
# ts-epochnano (nanoseconds since unix epoch)
|
|
|
|
|
# ts-"CUSTOM"
|
|
|
|
|
# CUSTOM time layouts must be within quotes and be the representation of the
|
|
|
|
|
# "reference time", which is Mon Jan 2 15:04:05 -0700 MST 2006
|
|
|
|
|
# See https://golang.org/pkg/time/#Parse for more details.
|
|
|
|
|
|
|
|
|
|
# Example log file pattern, example log looks like this:
|
|
|
|
|
# [04/Jun/2016:12:41:45 +0100] 1.25 200 192.168.1.1 5.432µs
|
|
|
|
|
# Breakdown of the DURATION pattern below:
|
|
|
|
|
# NUMBER is a builtin logstash grok pattern matching float & int numbers.
|
|
|
|
|
# [nuµm]? is a regex specifying 0 or 1 of the characters within brackets.
|
|
|
|
|
# s is also regex, this pattern must end in "s".
|
|
|
|
|
# so DURATION will match something like '5.324ms' or '6.1µs' or '10s'
|
|
|
|
|
DURATION %{NUMBER}[nuµm]?s
|
|
|
|
|
RESPONSE_CODE %{NUMBER:response_code:tag}
|
|
|
|
|
RESPONSE_TIME %{DURATION:response_time_ns:duration}
|
|
|
|
|
EXAMPLE_LOG \[%{HTTPDATE:ts:ts-httpd}\] %{NUMBER:myfloat:float} %{RESPONSE_CODE} %{IPORHOST:clientip} %{RESPONSE_TIME}
|
|
|
|
|
|
|
|
|
|
# Wider-ranging username matching vs. logstash built-in %{USER}
|
|
|
|
|
NGUSERNAME [a-zA-Z\.\@\-\+_%]+
|
|
|
|
|
NGUSER %{NGUSERNAME}
|
|
|
|
|
|
|
|
|
|
##
|
|
|
|
|
## COMMON LOG PATTERNS
|
|
|
|
|
##
|
|
|
|
|
|
|
|
|
|
# InfluxDB log patterns
|
|
|
|
|
CLIENT (?:%{IPORHOST}|%{HOSTPORT}|::1)
|
|
|
|
|
INFLUXDB_HTTPD_LOG \[httpd\] %{COMBINED_LOG_FORMAT} %{UUID:uuid:drop} %{NUMBER:response_time_us:int}
|
|
|
|
|
|
|
|
|
|
# apache & nginx logs, this is also known as the "common log format"
|
|
|
|
|
# see https://en.wikipedia.org/wiki/Common_Log_Format
|
2016-07-18 11:54:33 +00:00
|
|
|
COMMON_LOG_FORMAT %{CLIENT:client_ip} %{NGUSER:ident} %{NGUSER:auth} \[%{HTTPDATE:ts:ts-httpd}\] "(?:%{WORD:verb:tag} %{NOTSPACE:request}(?: HTTP/%{NUMBER:http_version:float})?|%{DATA})" %{NUMBER:resp_code:tag} (?:%{NUMBER:resp_bytes:int}|-)
|
2016-06-02 17:47:15 +00:00
|
|
|
|
|
|
|
|
# Combined log format is the same as the common log format but with the addition
|
|
|
|
|
# of two quoted strings at the end for "referrer" and "agent"
|
|
|
|
|
# See Examples at http://httpd.apache.org/docs/current/mod/mod_log_config.html
|
|
|
|
|
COMBINED_LOG_FORMAT %{COMMON_LOG_FORMAT} %{QS:referrer} %{QS:agent}
|
|
|
|
|
|
|
|
|
|
# HTTPD log formats
|
|
|
|
|
HTTPD20_ERRORLOG \[%{HTTPDERROR_DATE:timestamp}\] \[%{LOGLEVEL:loglevel:tag}\] (?:\[client %{IPORHOST:clientip}\] ){0,1}%{GREEDYDATA:errormsg}
|
|
|
|
|
HTTPD24_ERRORLOG \[%{HTTPDERROR_DATE:timestamp}\] \[%{WORD:module}:%{LOGLEVEL:loglevel:tag}\] \[pid %{POSINT:pid:int}:tid %{NUMBER:tid:int}\]( \(%{POSINT:proxy_errorcode:int}\)%{DATA:proxy_errormessage}:)?( \[client %{IPORHOST:client}:%{POSINT:clientport}\])? %{DATA:errorcode}: %{GREEDYDATA:message}
|
|
|
|
|
HTTPD_ERRORLOG %{HTTPD20_ERRORLOG}|%{HTTPD24_ERRORLOG}
|
|
|
|
|
`
|
