added webservercodes plugin

plugins/all/all.go

@@ -21,4 +21,5 @@ import (
 	_ "github.com/influxdb/telegraf/plugins/redis"
 	_ "github.com/influxdb/telegraf/plugins/rethinkdb"
 	_ "github.com/influxdb/telegraf/plugins/system"
+	_ "github.com/influxdb/telegraf/plugins/webservercodes"
 )

plugins/webservercodes/webserver_codes.go

@@ -0,0 +1,255 @@
+package webservercodes
+
+import (
+	"net/http"
+	"strconv"
+	"sync"
+	"time"
+	"os"
+	"io"
+	"regexp"
+	"errors"
+	
+	"github.com/rogpeppe/rog-go/reverse"
+	"github.com/influxdb/telegraf/plugins"
+)
+
+type Vhost struct {
+    Host string
+    AccessLog string
+    RegexParsestring string
+    Interval string
+}
+
+type Webservercodes struct {
+	Vhosts []*Vhost
+}
+
+type HttpStats struct {
+    codes map[int]int
+}
+
+var sampleConfig = `
+# List of virtualhosts for http codes collecting (each section for one virtualhost, none for disable collecting codes)
+[[webservercodes.vhosts]]
+# 'host' field should match hostname in appropriate status url
+host = "defaulthost"
+
+# telegraf user must have read permissions to this log file (shell command 'sudo adduser telegraf adm' for apache on Ubuntu)
+access_log = "/var/log/apache2/access.log"
+
+# Regular expression for fetching codes from log file strings. You can adjust regex for your custom log format
+# example for apache "common" and "combined" log formats, nginx default log format ("combined"):
+#   '\[(?P<time>[^\]]+)\] ".*?" (?P<code>\d{3})'
+regex_parsestring = '\[(?P<time>[^\]]+)\] ".*?" (?P<code>\d{3})'
+
+# plugin will count http codes in this interval. Interval must be in time.Duration format (see https://golang.org/pkg/time/#ParseDuration)
+interval = "10s"
+`
+
+func (n *Webservercodes) SampleConfig() string {
+	return sampleConfig
+}
+
+func (n *Webservercodes) Description() string {
+	return "Read webserver access log files and collects http return codes stats"
+}
+
+func (n *Webservercodes) Gather(acc plugins.Accumulator) error {
+	var wg sync.WaitGroup
+	
+    hostStats := map[string]HttpStats{}
+    errChan := make(chan error)
+    successChan := make(chan bool)
+    remainingItems := len(n.Vhosts)
+    
+    for _, vhost := range n.Vhosts {
+        if duration, err := time.ParseDuration(vhost.Interval); err == nil {
+            wg.Add(1)
+            go func(host string, logfile string, regex string, duration time.Duration, successChan chan bool, errChan chan error) {
+                defer wg.Done()
+                if stats, err := n.ParseHttpCodes(logfile, regex, duration); err == nil {
+                    hostStats[host] = *stats
+                    successChan <- true
+                } else {
+                    errChan <- err
+                }
+            }(vhost.Host, vhost.AccessLog, vhost.RegexParsestring, duration, successChan, errChan)
+        } else {
+            return err
+        }
+    }
+    for {
+        select {
+            case _ = <-successChan:
+                remainingItems--
+            case err := <-errChan:
+                return err
+        }
+        if remainingItems == 0 {
+            break;
+        }
+    }
+    
+    wg.Wait()
+	
+    for vhost, stats := range hostStats {
+        n.gatherCodes(vhost, stats, acc)
+    }
+	
+	return nil
+}
+
+func SearchStringInSlice(a string, list []string) bool {
+    for _, b := range list {
+        if b == a {
+            return true
+        }
+    }
+    return false
+}
+
+func (n Webservercodes) ParseRegex(regex string, f io.ReadSeeker) (*regexp.Regexp, error) {
+    
+    if rx, err := regexp.Compile(regex); err == nil {
+        
+        keys := rx.SubexpNames();
+        if SearchStringInSlice("time", keys) && SearchStringInSlice("code", keys) {
+            
+            reader := reverse.NewScanner(f)
+            if (reader.Scan()) {
+                // we will check regexp validity by scan the last log line and parse it, assuming that other lines will match as well
+                parsedLine := rx.FindStringSubmatch(reader.Text())
+                if len(parsedLine) >= 3 {
+                    
+                    logDt := parsedLine[1]
+                    if _, err := time.Parse("02/Jan/2006:15:04:05 -0700", logDt); err == nil {
+                        
+                        return rx, nil
+                    } else {
+                        return nil, errors.New("Time must be in apache %t format. Example: '02/Jan/2006:15:04:05 -0700'")
+                    }
+                } else {
+                    return nil, errors.New("Cannot find matches for regex in log line")
+                }
+            } else {
+                // if not Scanned, file is empty, so we don't need to return regex error
+                return rx, nil
+            }
+        } else {
+            return nil, errors.New("Regexp must define 'time' and 'code' fields")
+        }
+    } else {
+        return nil, err
+    }
+}
+
+func (n *Webservercodes) ParseHttpCodes(file string, regex string, duration time.Duration) (*HttpStats, error) {
+    
+    stats := HttpStats{codes: make(map[int]int)}
+    
+    if f, err := os.Open(file); err == nil {
+        
+        defer f.Close()
+        
+        if rx, err := n.ParseRegex(regex, f); err == nil {
+            var text, logDt string
+            var parsedLine []string
+            
+            curTime := time.Now()
+            reader := reverse.NewScanner(f)
+            
+            for reader.Scan() {
+                text = reader.Text()
+                
+                parsedLine = rx.FindStringSubmatch(text)
+                if len(parsedLine) > 0 {
+                    logDt = parsedLine[1]
+                    
+                    if time, err := time.Parse("02/Jan/2006:15:04:05 -0700", logDt); err == nil {
+                        if curTime.Sub(time) > duration {
+                            break
+                        }
+                        if code, err := strconv.Atoi(parsedLine[2]); err == nil {
+                            if _, ok := stats.codes[code]; ok {
+                                stats.codes[code]++
+                            } else {
+                                stats.codes[code] = 1
+                            }
+                        }
+                    }
+                }
+            }
+        } else {
+            return nil, err
+        }
+    } else {
+        return nil, err
+    }
+    
+    return &stats, nil
+}
+
+func (n *Webservercodes) gatherCodes(vhost string, stats HttpStats, acc plugins.Accumulator) {
+    
+    tags := map[string]string{"virtualhost" : vhost}
+    total := 0
+    var num int
+    
+    for _, i := range []int{http.StatusContinue,
+                    http.StatusSwitchingProtocols,
+                    http.StatusOK,
+                    http.StatusCreated,
+                    http.StatusAccepted,
+                    http.StatusNonAuthoritativeInfo,
+                    http.StatusNoContent,
+                    http.StatusResetContent,
+                    http.StatusPartialContent,
+                    http.StatusMultipleChoices,
+                    http.StatusMovedPermanently,
+                    http.StatusFound,
+                    http.StatusSeeOther,
+                    http.StatusNotModified,
+                    http.StatusUseProxy,
+                    http.StatusTemporaryRedirect,
+                    http.StatusBadRequest,
+                    http.StatusUnauthorized,
+                    http.StatusPaymentRequired,
+                    http.StatusForbidden,
+                    http.StatusNotFound,
+                    http.StatusMethodNotAllowed,
+                    http.StatusNotAcceptable,
+                    http.StatusProxyAuthRequired,
+                    http.StatusRequestTimeout,
+                    http.StatusConflict,
+                    http.StatusGone,
+                    http.StatusLengthRequired,
+                    http.StatusPreconditionFailed,
+                    http.StatusRequestEntityTooLarge,
+                    http.StatusRequestURITooLong,
+                    http.StatusUnsupportedMediaType,
+                    http.StatusRequestedRangeNotSatisfiable,
+                    http.StatusExpectationFailed,
+                    http.StatusTeapot,
+                    http.StatusInternalServerError,
+                    http.StatusNotImplemented,
+                    http.StatusBadGateway,
+                    http.StatusServiceUnavailable,
+                    http.StatusGatewayTimeout,
+                    http.StatusHTTPVersionNotSupported} {
+        if _, ok := stats.codes[i]; ok {
+            num = stats.codes[i]
+        } else {
+            num = 0
+        }
+        total += num
+        acc.Add(strconv.Itoa(i), num, tags);
+    }
+    acc.Add("total", total, tags);
+}
+
+func init() {
+	plugins.Add("webservercodes", func() plugins.Plugin {
+		return &Webservercodes{}
+	})
+}