Merge edf1336f8f into 0bc76f094a
This commit is contained in:
KPACHbIuLLIAnO4
commit
7a8d81983d
|
|
@ -23,4 +23,5 @@ import (
|
|||
_ "github.com/influxdb/telegraf/plugins/redis"
|
||||
_ "github.com/influxdb/telegraf/plugins/rethinkdb"
|
||||
_ "github.com/influxdb/telegraf/plugins/system"
|
||||
_ "github.com/influxdb/telegraf/plugins/webservercodes"
|
||||
)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,52 @@
|
|||
A plugin that collects and counts http codes from any webserver that has access log.
|
||||
It parses last several entries of log files and counts http codes.
|
||||
|
||||
Configuration.
|
||||
|
||||
A plugin supports per-virtualhost configuration. Config may contains one or more of this sections (example):
|
||||
|
||||
[[webservercodes.vhosts]]
|
||||
|
||||
host = "defaulthost"
|
||||
|
||||
access_log = "/var/log/apache2/access.log"
|
||||
|
||||
regex_parsestring = `\[(?P<time>[^\]]+)\] ".*?" (?P<code>\d{3})`
|
||||
|
||||
parse_interval = "10s"
|
||||
|
||||
'host' is the name of virtualhost. It's preferred to use main domain as a value.
|
||||
|
||||
'access_log' is the full name of log file for this virtualhost (with full path). File must exists ann be readable by telegraf process owner (usually user 'telegraf').
|
||||
|
||||
If you have apache web server installed on Debian/Ubuntu, you can set file readable by executing shell command:
|
||||
|
||||
`$ sudo adduser telegraf adm`
|
||||
|
||||
'regex_parsestring' is regexp pattern for log parsing. Regexp syntax must conform to goland acceptable syntax (see https://golang.org/s/re2syntax).
|
||||
|
||||
Pattern must include named substrings with labels 'time' and 'code'.
|
||||
|
||||
Example for apache "common" and "combined" log formats, nginx default log format ("combined"):
|
||||
|
||||
`\[(?P<time>[^\]]+)\] ".*?" (?P<code>\d{3})`
|
||||
|
||||
This pattern matches for log entries like (example):
|
||||
|
||||
`127.0.0.1 - - [30/Aug/2015:05:59:36 +0000] "GET / HTTP/1.1" 404 379 "-" "-"`
|
||||
|
||||
If you use another web server, please note that time must conform to apache2 `%t` format, see default configuration and http://httpd.apache.org/docs/2.2/mod/mod_log_config.html
|
||||
|
||||
'parse_interval' is interval that limits amount of log entries readed and parsed by plugin. It allows to examine even extremely large log files.
|
||||
|
||||
Usually this setting may match to global 'interval' parameter, but it can be adjusted in some cases of too high or too low load.
|
||||
|
||||
Parameter must be in time.Duration format (see https://golang.org/pkg/time/#ParseDuration).
|
||||
|
||||
Collected measurements.
|
||||
|
||||
Plugin adds tag 'virtualhost' to standard 'host' tag to all it's output values.
|
||||
|
||||
Output values are:
|
||||
- 'webservercodes_NNN' = (int), where NNN is all http codes defined in https://www.ietf.org/rfc/rfc2616.txt;
|
||||
- 'webservercodes_total' = (int) - total amount of codes collected per one plugin start.
|
||||
|
|
@ -0,0 +1,309 @@
|
|||
package webservercodes
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"strconv"
|
||||
"sync"
|
||||
"time"
|
||||
"os"
|
||||
"io"
|
||||
"regexp"
|
||||
"errors"
|
||||
|
||||
"github.com/rogpeppe/rog-go/reverse"
|
||||
"github.com/influxdb/telegraf/plugins"
|
||||
)
|
||||
|
||||
type Vhost struct {
|
||||
Host string
|
||||
AccessLog string
|
||||
RegexParsestring string
|
||||
ParseInterval string
|
||||
}
|
||||
|
||||
type Webservercodes struct {
|
||||
Vhosts []*Vhost
|
||||
}
|
||||
|
||||
type HttpStats struct {
|
||||
codes map[int]int
|
||||
}
|
||||
|
||||
type CombinedEntry struct {
|
||||
time time.Time
|
||||
code int
|
||||
}
|
||||
|
||||
var sampleConfig = `
|
||||
# List of virtualhosts for http codes collecting
|
||||
# (each section for one virtualhost, none for disable collecting codes)
|
||||
[[webservercodes.vhosts]]
|
||||
# 'host' field should match hostname in appropriate status url
|
||||
host = "defaulthost"
|
||||
|
||||
# Telegraf user must have read permissions to this log file
|
||||
# (shell command 'sudo adduser telegraf adm' for apache on Ubuntu)
|
||||
access_log = "/var/log/apache2/access.log"
|
||||
|
||||
# Regular expression for fetching codes from log file strings.
|
||||
# You can adjust this pattern for your custom log format
|
||||
# Example for apache "common" and "combined" log formats, nginx default log format ("combined"):
|
||||
# '\[(?P<time>[^\]]+)\] ".*?" (?P<code>\d{3})'
|
||||
# This pattern matches for strings like (example):
|
||||
# 127.0.0.1 - - [30/Aug/2015:05:59:36 +0000] "GET / HTTP/1.1" 404 379 "-" "-"
|
||||
regex_parsestring = '\[(?P<time>[^\]]+)\] ".*?" (?P<code>\d{3})'
|
||||
|
||||
# Plugin will parse log for http codes from 'now' till 'now - parse_interval' moments.
|
||||
# parse_interval must be in time.Duration format (see https://golang.org/pkg/time/#ParseDuration)
|
||||
parse_interval = "10s"
|
||||
`
|
||||
|
||||
func (n *Webservercodes) SampleConfig() string {
|
||||
return sampleConfig
|
||||
}
|
||||
|
||||
func (n *Webservercodes) Description() string {
|
||||
return "Read webserver access log files and count http return codes found"
|
||||
}
|
||||
|
||||
func (n *Webservercodes) Gather(acc plugins.Accumulator) error {
|
||||
var wg sync.WaitGroup
|
||||
|
||||
hostStats := map[string]HttpStats{}
|
||||
errChan := make(chan error)
|
||||
successChan := make(chan bool)
|
||||
remainingItems := len(n.Vhosts)
|
||||
|
||||
for _, vhost := range n.Vhosts {
|
||||
if duration, err := time.ParseDuration(vhost.ParseInterval); err == nil {
|
||||
wg.Add(1)
|
||||
go func(host string, logfile string, regex string, duration time.Duration, successChan chan bool, errChan chan error) {
|
||||
defer wg.Done()
|
||||
if stats, err := n.ParseHttpCodes(logfile, regex, duration); err == nil {
|
||||
hostStats[host] = *stats
|
||||
successChan <- true
|
||||
} else {
|
||||
errChan <- err
|
||||
}
|
||||
}(vhost.Host, vhost.AccessLog, vhost.RegexParsestring, duration, successChan, errChan)
|
||||
} else {
|
||||
return err
|
||||
}
|
||||
}
|
||||
for {
|
||||
select {
|
||||
case _ = <-successChan:
|
||||
remainingItems--
|
||||
case err := <-errChan:
|
||||
return err
|
||||
}
|
||||
if remainingItems == 0 {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
|
||||
for vhost, stats := range hostStats {
|
||||
n.gatherCodes(vhost, stats, acc)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func SearchStringInSlice(a string, list []string) bool {
|
||||
for _, b := range list {
|
||||
if b == a {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (n Webservercodes) CombineKeysValues(keys []string, values []string) (*CombinedEntry, error) {
|
||||
|
||||
if len(values) < len(keys) {
|
||||
return nil, errors.New("Not enough substrings")
|
||||
}
|
||||
|
||||
items := map[string]string{}
|
||||
for k, v := range keys {
|
||||
items[v] = values[k]
|
||||
}
|
||||
|
||||
combined := CombinedEntry{}
|
||||
if logDt, ok := items["time"]; ok {
|
||||
if time, err := time.Parse("02/Jan/2006:15:04:05 -0700", logDt); err == nil {
|
||||
combined.time = time
|
||||
} else {
|
||||
return nil, errors.New("Time must be in apache %t format. Example: '02/Jan/2006:15:04:05 -0700'")
|
||||
}
|
||||
} else {
|
||||
return nil, errors.New("Time is absent in log line")
|
||||
}
|
||||
|
||||
if _, ok := items["code"]; ok {
|
||||
code, _ := strconv.Atoi(items["code"])
|
||||
combined.code = code
|
||||
} else {
|
||||
return nil, errors.New("Http code is absent in log line")
|
||||
}
|
||||
|
||||
return &combined, nil
|
||||
}
|
||||
|
||||
func (n Webservercodes) ValidateRegexp(regex string, f io.ReadSeeker) (*regexp.Regexp, []string, error) {
|
||||
|
||||
keys := []string{}
|
||||
var rx *regexp.Regexp
|
||||
var err error
|
||||
|
||||
if rx, err = regexp.Compile(regex); err != nil {
|
||||
// error in case of malformed regexp
|
||||
return nil, keys, err
|
||||
}
|
||||
|
||||
keys = rx.SubexpNames();
|
||||
if !(SearchStringInSlice("time", keys) && SearchStringInSlice("code", keys)) {
|
||||
// error if fields 'time' or 'code' are defined
|
||||
return nil, keys, errors.New("Regexp must define 'time' and 'code' fields")
|
||||
}
|
||||
|
||||
// we will check regexp validity by scan the last log line
|
||||
// and parse it, assuming that other lines will match as well
|
||||
reader := reverse.NewScanner(f)
|
||||
if (!reader.Scan()) {
|
||||
// if not Scanned, file is empty, so we don't need to return regex error
|
||||
return rx, keys, nil
|
||||
}
|
||||
|
||||
strings := rx.FindStringSubmatch(reader.Text())
|
||||
if len(strings) == 0 {
|
||||
// error if regexp mismatch
|
||||
return nil, keys, errors.New("Log entries are not match regexp")
|
||||
}
|
||||
if _, err := n.CombineKeysValues(keys, strings); err != nil {
|
||||
// error if no values for 'time' or 'code' are found in parsed log line
|
||||
return nil, keys, err
|
||||
}
|
||||
|
||||
return rx, keys, nil
|
||||
}
|
||||
|
||||
func (n *Webservercodes) ParseHttpCodes(file string, regex string, duration time.Duration) (*HttpStats, error) {
|
||||
|
||||
stats := HttpStats{codes: make(map[int]int)}
|
||||
|
||||
if f, err := os.Open(file); err == nil {
|
||||
defer f.Close()
|
||||
|
||||
if rx, keys, err := n.ValidateRegexp(regex, f); err == nil {
|
||||
curTime := time.Now()
|
||||
errorsCounter := 0
|
||||
errorsMax := 100 // there is something wrong if more than errorsMax parse errors
|
||||
var vastedLoop bool
|
||||
var strings []string
|
||||
|
||||
reader := reverse.NewScanner(f)
|
||||
for reader.Scan() {
|
||||
vastedLoop = false
|
||||
strings = rx.FindStringSubmatch(reader.Text())
|
||||
if len(strings) > 0 {
|
||||
if parsedLine, err := n.CombineKeysValues(keys, strings); err == nil {
|
||||
if curTime.Sub(parsedLine.time) > duration {
|
||||
break
|
||||
}
|
||||
if _, ok := stats.codes[parsedLine.code]; ok {
|
||||
stats.codes[parsedLine.code]++
|
||||
} else {
|
||||
stats.codes[parsedLine.code] = 1
|
||||
}
|
||||
} else {
|
||||
vastedLoop = true
|
||||
}
|
||||
} else {
|
||||
vastedLoop = true
|
||||
}
|
||||
if vastedLoop {
|
||||
errorsCounter++
|
||||
}
|
||||
if errorsCounter >= errorsMax {
|
||||
break
|
||||
}
|
||||
}
|
||||
if errorsCounter >= errorsMax {
|
||||
return nil, errors.New("Too many entries with wrong format in log file. Check regex_parsestring")
|
||||
}
|
||||
} else {
|
||||
return nil, err
|
||||
}
|
||||
} else {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &stats, nil
|
||||
}
|
||||
|
||||
func (n *Webservercodes) gatherCodes(vhost string, stats HttpStats, acc plugins.Accumulator) {
|
||||
|
||||
tags := map[string]string{"virtualhost" : vhost}
|
||||
total := 0
|
||||
var num int
|
||||
|
||||
for _, i := range []int{http.StatusContinue,
|
||||
http.StatusSwitchingProtocols,
|
||||
http.StatusOK,
|
||||
http.StatusCreated,
|
||||
http.StatusAccepted,
|
||||
http.StatusNonAuthoritativeInfo,
|
||||
http.StatusNoContent,
|
||||
http.StatusResetContent,
|
||||
http.StatusPartialContent,
|
||||
http.StatusMultipleChoices,
|
||||
http.StatusMovedPermanently,
|
||||
http.StatusFound,
|
||||
http.StatusSeeOther,
|
||||
http.StatusNotModified,
|
||||
http.StatusUseProxy,
|
||||
http.StatusTemporaryRedirect,
|
||||
http.StatusBadRequest,
|
||||
http.StatusUnauthorized,
|
||||
http.StatusPaymentRequired,
|
||||
http.StatusForbidden,
|
||||
http.StatusNotFound,
|
||||
http.StatusMethodNotAllowed,
|
||||
http.StatusNotAcceptable,
|
||||
http.StatusProxyAuthRequired,
|
||||
http.StatusRequestTimeout,
|
||||
http.StatusConflict,
|
||||
http.StatusGone,
|
||||
http.StatusLengthRequired,
|
||||
http.StatusPreconditionFailed,
|
||||
http.StatusRequestEntityTooLarge,
|
||||
http.StatusRequestURITooLong,
|
||||
http.StatusUnsupportedMediaType,
|
||||
http.StatusRequestedRangeNotSatisfiable,
|
||||
http.StatusExpectationFailed,
|
||||
http.StatusTeapot,
|
||||
http.StatusInternalServerError,
|
||||
http.StatusNotImplemented,
|
||||
http.StatusBadGateway,
|
||||
http.StatusServiceUnavailable,
|
||||
http.StatusGatewayTimeout,
|
||||
http.StatusHTTPVersionNotSupported} {
|
||||
if _, ok := stats.codes[i]; ok {
|
||||
num = stats.codes[i]
|
||||
} else {
|
||||
num = 0
|
||||
}
|
||||
total += num
|
||||
acc.Add(strconv.Itoa(i), num, tags);
|
||||
}
|
||||
acc.Add("total", total, tags);
|
||||
}
|
||||
|
||||
func init() {
|
||||
plugins.Add("webservercodes", func() plugins.Plugin {
|
||||
return &Webservercodes{}
|
||||
})
|
||||
}
|
||||
Loading…
Reference in New Issue