spaces to tabs
This commit is contained in:
Alexander Oleinik
parent
cc8a29c4b2
commit
96cbd0c5f1
|
|
@ -1,37 +1,37 @@
|
||||||
package webservercodes
|
package webservercodes
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"net/http"
|
"net/http"
|
||||||
"strconv"
|
"strconv"
|
||||||
"sync"
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
"os"
|
"os"
|
||||||
"io"
|
"io"
|
||||||
"regexp"
|
"regexp"
|
||||||
"errors"
|
"errors"
|
||||||
|
|
||||||
"github.com/rogpeppe/rog-go/reverse"
|
"github.com/rogpeppe/rog-go/reverse"
|
||||||
"github.com/influxdb/telegraf/plugins"
|
"github.com/influxdb/telegraf/plugins"
|
||||||
)
|
)
|
||||||
|
|
||||||
type Vhost struct {
|
type Vhost struct {
|
||||||
Host string
|
Host string
|
||||||
AccessLog string
|
AccessLog string
|
||||||
RegexParsestring string
|
RegexParsestring string
|
||||||
ParseInterval string
|
ParseInterval string
|
||||||
}
|
}
|
||||||
|
|
||||||
type Webservercodes struct {
|
type Webservercodes struct {
|
||||||
Vhosts []*Vhost
|
Vhosts []*Vhost
|
||||||
}
|
}
|
||||||
|
|
||||||
type HttpStats struct {
|
type HttpStats struct {
|
||||||
codes map[int]int
|
codes map[int]int
|
||||||
}
|
}
|
||||||
|
|
||||||
type CombinedEntry struct {
|
type CombinedEntry struct {
|
||||||
time time.Time
|
time time.Time
|
||||||
code int
|
code int
|
||||||
}
|
}
|
||||||
|
|
||||||
var sampleConfig = `
|
var sampleConfig = `
|
||||||
|
|
@ -48,7 +48,7 @@ access_log = "/var/log/apache2/access.log"
|
||||||
# Regular expression for fetching codes from log file strings.
|
# Regular expression for fetching codes from log file strings.
|
||||||
# You can adjust this pattern for your custom log format
|
# You can adjust this pattern for your custom log format
|
||||||
# Example for apache "common" and "combined" log formats, nginx default log format ("combined"):
|
# Example for apache "common" and "combined" log formats, nginx default log format ("combined"):
|
||||||
# '\[(?P<time>[^\]]+)\] ".*?" (?P<code>\d{3})'
|
# '\[(?P<time>[^\]]+)\] ".*?" (?P<code>\d{3})'
|
||||||
# This pattern matches for strings like (example):
|
# This pattern matches for strings like (example):
|
||||||
# 127.0.0.1 - - [30/Aug/2015:05:59:36 +0000] "GET / HTTP/1.1" 404 379 "-" "-"
|
# 127.0.0.1 - - [30/Aug/2015:05:59:36 +0000] "GET / HTTP/1.1" 404 379 "-" "-"
|
||||||
regex_parsestring = '\[(?P<time>[^\]]+)\] ".*?" (?P<code>\d{3})'
|
regex_parsestring = '\[(?P<time>[^\]]+)\] ".*?" (?P<code>\d{3})'
|
||||||
|
|
@ -59,251 +59,251 @@ parse_interval = "10s"
|
||||||
`
|
`
|
||||||
|
|
||||||
func (n *Webservercodes) SampleConfig() string {
|
func (n *Webservercodes) SampleConfig() string {
|
||||||
return sampleConfig
|
return sampleConfig
|
||||||
}
|
}
|
||||||
|
|
||||||
func (n *Webservercodes) Description() string {
|
func (n *Webservercodes) Description() string {
|
||||||
return "Read webserver access log files and count http return codes found"
|
return "Read webserver access log files and count http return codes found"
|
||||||
}
|
}
|
||||||
|
|
||||||
func (n *Webservercodes) Gather(acc plugins.Accumulator) error {
|
func (n *Webservercodes) Gather(acc plugins.Accumulator) error {
|
||||||
var wg sync.WaitGroup
|
var wg sync.WaitGroup
|
||||||
|
|
||||||
hostStats := map[string]HttpStats{}
|
hostStats := map[string]HttpStats{}
|
||||||
errChan := make(chan error)
|
errChan := make(chan error)
|
||||||
successChan := make(chan bool)
|
successChan := make(chan bool)
|
||||||
remainingItems := len(n.Vhosts)
|
remainingItems := len(n.Vhosts)
|
||||||
|
|
||||||
for _, vhost := range n.Vhosts {
|
for _, vhost := range n.Vhosts {
|
||||||
if duration, err := time.ParseDuration(vhost.ParseInterval); err == nil {
|
if duration, err := time.ParseDuration(vhost.ParseInterval); err == nil {
|
||||||
wg.Add(1)
|
wg.Add(1)
|
||||||
go func(host string, logfile string, regex string, duration time.Duration, successChan chan bool, errChan chan error) {
|
go func(host string, logfile string, regex string, duration time.Duration, successChan chan bool, errChan chan error) {
|
||||||
defer wg.Done()
|
defer wg.Done()
|
||||||
if stats, err := n.ParseHttpCodes(logfile, regex, duration); err == nil {
|
if stats, err := n.ParseHttpCodes(logfile, regex, duration); err == nil {
|
||||||
hostStats[host] = *stats
|
hostStats[host] = *stats
|
||||||
successChan <- true
|
successChan <- true
|
||||||
} else {
|
} else {
|
||||||
errChan <- err
|
errChan <- err
|
||||||
}
|
}
|
||||||
}(vhost.Host, vhost.AccessLog, vhost.RegexParsestring, duration, successChan, errChan)
|
}(vhost.Host, vhost.AccessLog, vhost.RegexParsestring, duration, successChan, errChan)
|
||||||
} else {
|
} else {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
for {
|
for {
|
||||||
select {
|
select {
|
||||||
case _ = <-successChan:
|
case _ = <-successChan:
|
||||||
remainingItems--
|
remainingItems--
|
||||||
case err := <-errChan:
|
case err := <-errChan:
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
if remainingItems == 0 {
|
if remainingItems == 0 {
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
wg.Wait()
|
wg.Wait()
|
||||||
|
|
||||||
for vhost, stats := range hostStats {
|
for vhost, stats := range hostStats {
|
||||||
n.gatherCodes(vhost, stats, acc)
|
n.gatherCodes(vhost, stats, acc)
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func SearchStringInSlice(a string, list []string) bool {
|
func SearchStringInSlice(a string, list []string) bool {
|
||||||
for _, b := range list {
|
for _, b := range list {
|
||||||
if b == a {
|
if b == a {
|
||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
func (n Webservercodes) CombineKeysValues(keys []string, values []string) (*CombinedEntry, error) {
|
func (n Webservercodes) CombineKeysValues(keys []string, values []string) (*CombinedEntry, error) {
|
||||||
|
|
||||||
if len(values) < len(keys) {
|
if len(values) < len(keys) {
|
||||||
return nil, errors.New("Not enough substrings")
|
return nil, errors.New("Not enough substrings")
|
||||||
}
|
}
|
||||||
|
|
||||||
items := map[string]string{}
|
items := map[string]string{}
|
||||||
for k, v := range keys {
|
for k, v := range keys {
|
||||||
items[v] = values[k]
|
items[v] = values[k]
|
||||||
}
|
}
|
||||||
|
|
||||||
combined := CombinedEntry{}
|
combined := CombinedEntry{}
|
||||||
if logDt, ok := items["time"]; ok {
|
if logDt, ok := items["time"]; ok {
|
||||||
if time, err := time.Parse("02/Jan/2006:15:04:05 -0700", logDt); err == nil {
|
if time, err := time.Parse("02/Jan/2006:15:04:05 -0700", logDt); err == nil {
|
||||||
combined.time = time
|
combined.time = time
|
||||||
} else {
|
} else {
|
||||||
return nil, errors.New("Time must be in apache %t format. Example: '02/Jan/2006:15:04:05 -0700'")
|
return nil, errors.New("Time must be in apache %t format. Example: '02/Jan/2006:15:04:05 -0700'")
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
return nil, errors.New("Time is absent in log line")
|
return nil, errors.New("Time is absent in log line")
|
||||||
}
|
}
|
||||||
|
|
||||||
if _, ok := items["code"]; ok {
|
if _, ok := items["code"]; ok {
|
||||||
code, _ := strconv.Atoi(items["code"])
|
code, _ := strconv.Atoi(items["code"])
|
||||||
combined.code = code
|
combined.code = code
|
||||||
} else {
|
} else {
|
||||||
return nil, errors.New("Http code is absent in log line")
|
return nil, errors.New("Http code is absent in log line")
|
||||||
}
|
}
|
||||||
|
|
||||||
return &combined, nil
|
return &combined, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (n Webservercodes) ValidateRegexp(regex string, f io.ReadSeeker) (*regexp.Regexp, []string, error) {
|
func (n Webservercodes) ValidateRegexp(regex string, f io.ReadSeeker) (*regexp.Regexp, []string, error) {
|
||||||
|
|
||||||
keys := []string{}
|
keys := []string{}
|
||||||
var rx *regexp.Regexp
|
var rx *regexp.Regexp
|
||||||
var err error
|
var err error
|
||||||
|
|
||||||
if rx, err = regexp.Compile(regex); err != nil {
|
if rx, err = regexp.Compile(regex); err != nil {
|
||||||
// error in case of malformed regexp
|
// error in case of malformed regexp
|
||||||
return nil, keys, err
|
return nil, keys, err
|
||||||
}
|
}
|
||||||
|
|
||||||
keys = rx.SubexpNames();
|
keys = rx.SubexpNames();
|
||||||
if !(SearchStringInSlice("time", keys) && SearchStringInSlice("code", keys)) {
|
if !(SearchStringInSlice("time", keys) && SearchStringInSlice("code", keys)) {
|
||||||
// error if fields 'time' or 'code' are defined
|
// error if fields 'time' or 'code' are defined
|
||||||
return nil, keys, errors.New("Regexp must define 'time' and 'code' fields")
|
return nil, keys, errors.New("Regexp must define 'time' and 'code' fields")
|
||||||
}
|
}
|
||||||
|
|
||||||
// we will check regexp validity by scan the last log line
|
// we will check regexp validity by scan the last log line
|
||||||
// and parse it, assuming that other lines will match as well
|
// and parse it, assuming that other lines will match as well
|
||||||
reader := reverse.NewScanner(f)
|
reader := reverse.NewScanner(f)
|
||||||
if (!reader.Scan()) {
|
if (!reader.Scan()) {
|
||||||
// if not Scanned, file is empty, so we don't need to return regex error
|
// if not Scanned, file is empty, so we don't need to return regex error
|
||||||
return rx, keys, nil
|
return rx, keys, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
strings := rx.FindStringSubmatch(reader.Text())
|
strings := rx.FindStringSubmatch(reader.Text())
|
||||||
if len(strings) == 0 {
|
if len(strings) == 0 {
|
||||||
// error if regexp mismatch
|
// error if regexp mismatch
|
||||||
return nil, keys, errors.New("Log entries are not match regexp")
|
return nil, keys, errors.New("Log entries are not match regexp")
|
||||||
}
|
}
|
||||||
if _, err := n.CombineKeysValues(keys, strings); err != nil {
|
if _, err := n.CombineKeysValues(keys, strings); err != nil {
|
||||||
// error if no values for 'time' or 'code' are found in parsed log line
|
// error if no values for 'time' or 'code' are found in parsed log line
|
||||||
return nil, keys, err
|
return nil, keys, err
|
||||||
}
|
}
|
||||||
|
|
||||||
return rx, keys, nil
|
return rx, keys, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (n *Webservercodes) ParseHttpCodes(file string, regex string, duration time.Duration) (*HttpStats, error) {
|
func (n *Webservercodes) ParseHttpCodes(file string, regex string, duration time.Duration) (*HttpStats, error) {
|
||||||
|
|
||||||
stats := HttpStats{codes: make(map[int]int)}
|
stats := HttpStats{codes: make(map[int]int)}
|
||||||
|
|
||||||
if f, err := os.Open(file); err == nil {
|
if f, err := os.Open(file); err == nil {
|
||||||
defer f.Close()
|
defer f.Close()
|
||||||
|
|
||||||
if rx, keys, err := n.ValidateRegexp(regex, f); err == nil {
|
if rx, keys, err := n.ValidateRegexp(regex, f); err == nil {
|
||||||
curTime := time.Now()
|
curTime := time.Now()
|
||||||
errorsCounter := 0
|
errorsCounter := 0
|
||||||
errorsMax := 100 // there is something wrong if more than errorsMax parse errors
|
errorsMax := 100 // there is something wrong if more than errorsMax parse errors
|
||||||
var vastedLoop bool
|
var vastedLoop bool
|
||||||
var strings []string
|
var strings []string
|
||||||
|
|
||||||
reader := reverse.NewScanner(f)
|
reader := reverse.NewScanner(f)
|
||||||
for reader.Scan() {
|
for reader.Scan() {
|
||||||
vastedLoop = false
|
vastedLoop = false
|
||||||
strings = rx.FindStringSubmatch(reader.Text())
|
strings = rx.FindStringSubmatch(reader.Text())
|
||||||
if len(strings) > 0 {
|
if len(strings) > 0 {
|
||||||
if parsedLine, err := n.CombineKeysValues(keys, strings); err == nil {
|
if parsedLine, err := n.CombineKeysValues(keys, strings); err == nil {
|
||||||
if curTime.Sub(parsedLine.time) > duration {
|
if curTime.Sub(parsedLine.time) > duration {
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
if _, ok := stats.codes[parsedLine.code]; ok {
|
if _, ok := stats.codes[parsedLine.code]; ok {
|
||||||
stats.codes[parsedLine.code]++
|
stats.codes[parsedLine.code]++
|
||||||
} else {
|
} else {
|
||||||
stats.codes[parsedLine.code] = 1
|
stats.codes[parsedLine.code] = 1
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
vastedLoop = true
|
vastedLoop = true
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
vastedLoop = true
|
vastedLoop = true
|
||||||
}
|
}
|
||||||
if vastedLoop {
|
if vastedLoop {
|
||||||
errorsCounter++
|
errorsCounter++
|
||||||
}
|
}
|
||||||
if errorsCounter >= errorsMax {
|
if errorsCounter >= errorsMax {
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if errorsCounter >= errorsMax {
|
if errorsCounter >= errorsMax {
|
||||||
return nil, errors.New("Too many entries with wrong format in log file. Check regex_parsestring")
|
return nil, errors.New("Too many entries with wrong format in log file. Check regex_parsestring")
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
return &stats, nil
|
return &stats, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (n *Webservercodes) gatherCodes(vhost string, stats HttpStats, acc plugins.Accumulator) {
|
func (n *Webservercodes) gatherCodes(vhost string, stats HttpStats, acc plugins.Accumulator) {
|
||||||
|
|
||||||
tags := map[string]string{"virtualhost" : vhost}
|
tags := map[string]string{"virtualhost" : vhost}
|
||||||
total := 0
|
total := 0
|
||||||
var num int
|
var num int
|
||||||
|
|
||||||
for _, i := range []int{http.StatusContinue,
|
for _, i := range []int{http.StatusContinue,
|
||||||
http.StatusSwitchingProtocols,
|
http.StatusSwitchingProtocols,
|
||||||
http.StatusOK,
|
http.StatusOK,
|
||||||
http.StatusCreated,
|
http.StatusCreated,
|
||||||
http.StatusAccepted,
|
http.StatusAccepted,
|
||||||
http.StatusNonAuthoritativeInfo,
|
http.StatusNonAuthoritativeInfo,
|
||||||
http.StatusNoContent,
|
http.StatusNoContent,
|
||||||
http.StatusResetContent,
|
http.StatusResetContent,
|
||||||
http.StatusPartialContent,
|
http.StatusPartialContent,
|
||||||
http.StatusMultipleChoices,
|
http.StatusMultipleChoices,
|
||||||
http.StatusMovedPermanently,
|
http.StatusMovedPermanently,
|
||||||
http.StatusFound,
|
http.StatusFound,
|
||||||
http.StatusSeeOther,
|
http.StatusSeeOther,
|
||||||
http.StatusNotModified,
|
http.StatusNotModified,
|
||||||
http.StatusUseProxy,
|
http.StatusUseProxy,
|
||||||
http.StatusTemporaryRedirect,
|
http.StatusTemporaryRedirect,
|
||||||
http.StatusBadRequest,
|
http.StatusBadRequest,
|
||||||
http.StatusUnauthorized,
|
http.StatusUnauthorized,
|
||||||
http.StatusPaymentRequired,
|
http.StatusPaymentRequired,
|
||||||
http.StatusForbidden,
|
http.StatusForbidden,
|
||||||
http.StatusNotFound,
|
http.StatusNotFound,
|
||||||
http.StatusMethodNotAllowed,
|
http.StatusMethodNotAllowed,
|
||||||
http.StatusNotAcceptable,
|
http.StatusNotAcceptable,
|
||||||
http.StatusProxyAuthRequired,
|
http.StatusProxyAuthRequired,
|
||||||
http.StatusRequestTimeout,
|
http.StatusRequestTimeout,
|
||||||
http.StatusConflict,
|
http.StatusConflict,
|
||||||
http.StatusGone,
|
http.StatusGone,
|
||||||
http.StatusLengthRequired,
|
http.StatusLengthRequired,
|
||||||
http.StatusPreconditionFailed,
|
http.StatusPreconditionFailed,
|
||||||
http.StatusRequestEntityTooLarge,
|
http.StatusRequestEntityTooLarge,
|
||||||
http.StatusRequestURITooLong,
|
http.StatusRequestURITooLong,
|
||||||
http.StatusUnsupportedMediaType,
|
http.StatusUnsupportedMediaType,
|
||||||
http.StatusRequestedRangeNotSatisfiable,
|
http.StatusRequestedRangeNotSatisfiable,
|
||||||
http.StatusExpectationFailed,
|
http.StatusExpectationFailed,
|
||||||
http.StatusTeapot,
|
http.StatusTeapot,
|
||||||
http.StatusInternalServerError,
|
http.StatusInternalServerError,
|
||||||
http.StatusNotImplemented,
|
http.StatusNotImplemented,
|
||||||
http.StatusBadGateway,
|
http.StatusBadGateway,
|
||||||
http.StatusServiceUnavailable,
|
http.StatusServiceUnavailable,
|
||||||
http.StatusGatewayTimeout,
|
http.StatusGatewayTimeout,
|
||||||
http.StatusHTTPVersionNotSupported} {
|
http.StatusHTTPVersionNotSupported} {
|
||||||
if _, ok := stats.codes[i]; ok {
|
if _, ok := stats.codes[i]; ok {
|
||||||
num = stats.codes[i]
|
num = stats.codes[i]
|
||||||
} else {
|
} else {
|
||||||
num = 0
|
num = 0
|
||||||
}
|
}
|
||||||
total += num
|
total += num
|
||||||
acc.Add(strconv.Itoa(i), num, tags);
|
acc.Add(strconv.Itoa(i), num, tags);
|
||||||
}
|
}
|
||||||
acc.Add("total", total, tags);
|
acc.Add("total", total, tags);
|
||||||
}
|
}
|
||||||
|
|
||||||
func init() {
|
func init() {
|
||||||
plugins.Add("webservercodes", func() plugins.Plugin {
|
plugins.Add("webservercodes", func() plugins.Plugin {
|
||||||
return &Webservercodes{}
|
return &Webservercodes{}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue