dom
/
telegraf
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

0.3.0 Removing internal parallelism: httpjson and exec

This commit is contained in:
Cameron Sparr 2015-12-19 15:29:07 -07:00
parent f60d846eb3
commit 41374aabcb
3 changed files with 62 additions and 154 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

91
plugins/exec/exec.go
View File

@ -3,13 +3,8 @@ package exec
import ( import (
"bytes" "bytes"
"encoding/json" "encoding/json"
"errors"
"fmt" "fmt"
"math"
"os/exec" "os/exec"
"strings"
"sync"
"time"
"github.com/gonuts/go-shellquote" "github.com/gonuts/go-shellquote"
@ -18,47 +13,28 @@ import (
) )
const sampleConfig = ` const sampleConfig = `
# specify commands via an array of tables
[[plugins.exec.commands]]
# the command to run # the command to run
command = "/usr/bin/mycollector --foo=bar" command = "/usr/bin/mycollector --foo=bar"
# name of the command (used as a prefix for measurements) # name of the command (used as a prefix for measurements)
name = "mycollector" name = "mycollector"
# Only run this command if it has been at least this many
# seconds since it last ran
interval = 10
` `
type Exec struct { type Exec struct {
Commands []*Command
runner Runner
clock Clock
}
type Command struct {
Command string Command string
Name string Name string
Interval int
lastRunAt time.Time runner Runner
} }
type Runner interface { type Runner interface {
Run(*Command) ([]byte, error) Run(*Exec) ([]byte, error)
}
type Clock interface {
Now() time.Time
} }
type CommandRunner struct{} type CommandRunner struct{}
type RealClock struct{} func (c CommandRunner) Run(e *Exec) ([]byte, error) {
split_cmd, err := shellquote.Split(e.Command)
func (c CommandRunner) Run(command *Command) ([]byte, error) {
command.lastRunAt = time.Now()
split_cmd, err := shellquote.Split(command.Command)
if err != nil || len(split_cmd) == 0 { if err != nil || len(split_cmd) == 0 {
return nil, fmt.Errorf("exec: unable to parse command, %s", err) return nil, fmt.Errorf("exec: unable to parse command, %s", err)
} }
@ -68,18 +44,14 @@ func (c CommandRunner) Run(command *Command) ([]byte, error) {
cmd.Stdout = &out cmd.Stdout = &out
if err := cmd.Run(); err != nil { if err := cmd.Run(); err != nil {
return nil, fmt.Errorf("exec: %s for command '%s'", err, command.Command) return nil, fmt.Errorf("exec: %s for command '%s'", err, e.Command)
} }
return out.Bytes(), nil return out.Bytes(), nil
} }
func (c RealClock) Now() time.Time {
return time.Now()
}
func NewExec() *Exec { func NewExec() *Exec {
return &Exec{runner: CommandRunner{}, clock: RealClock{}} return &Exec{runner: CommandRunner{}}
} }
func (e *Exec) SampleConfig() string { func (e *Exec) SampleConfig() string {
@ -91,47 +63,7 @@ func (e *Exec) Description() string {
} }
func (e *Exec) Gather(acc plugins.Accumulator) error { func (e *Exec) Gather(acc plugins.Accumulator) error {
var wg sync.WaitGroup out, err := e.runner.Run(e)
errorChannel := make(chan error, len(e.Commands))
for _, c := range e.Commands {
wg.Add(1)
go func(c *Command, acc plugins.Accumulator) {
defer wg.Done()
err := e.gatherCommand(c, acc)
if err != nil {
errorChannel <- err
}
}(c, acc)
}
wg.Wait()
close(errorChannel)
// Get all errors and return them as one giant error
errorStrings := []string{}
for err := range errorChannel {
errorStrings = append(errorStrings, err.Error())
}
if len(errorStrings) == 0 {
return nil
}
return errors.New(strings.Join(errorStrings, "\n"))
}
func (e *Exec) gatherCommand(c *Command, acc plugins.Accumulator) error {
secondsSinceLastRun := 0.0
if c.lastRunAt.Unix() == 0 { // means time is uninitialized
secondsSinceLastRun = math.Inf(1)
} else {
secondsSinceLastRun = (e.clock.Now().Sub(c.lastRunAt)).Seconds()
}
if secondsSinceLastRun >= float64(c.Interval) {
out, err := e.runner.Run(c)
if err != nil { if err != nil {
return err return err
} }
@ -140,7 +72,7 @@ func (e *Exec) gatherCommand(c *Command, acc plugins.Accumulator) error {
err = json.Unmarshal(out, &jsonOut) err = json.Unmarshal(out, &jsonOut)
if err != nil { if err != nil {
return fmt.Errorf("exec: unable to parse output of '%s' as JSON, %s", return fmt.Errorf("exec: unable to parse output of '%s' as JSON, %s",
c.Command, err) e.Command, err)
} }
f := internal.JSONFlattener{} f := internal.JSONFlattener{}
@ -150,13 +82,12 @@ func (e *Exec) gatherCommand(c *Command, acc plugins.Accumulator) error {
} }
var msrmnt_name string var msrmnt_name string
if c.Name == "" { if e.Name == "" {
msrmnt_name = "exec" msrmnt_name = "exec"
} else { } else {
msrmnt_name = "exec_" + c.Name msrmnt_name = "exec_" + e.Name
} }
acc.AddFields(msrmnt_name, f.Fields, nil) acc.AddFields(msrmnt_name, f.Fields, nil)
}
return nil return nil
} }

44
plugins/httpjson/httpjson.go
View File

@ -15,16 +15,12 @@ import (
) )
type HttpJson struct { type HttpJson struct {
Services []Service
client HTTPClient
}
type Service struct {
Name string Name string
Servers []string Servers []string
Method string Method string
TagKeys []string TagKeys []string
Parameters map[string]string Parameters map[string]string
client HTTPClient
} }
type HTTPClient interface { type HTTPClient interface {
@ -48,9 +44,6 @@ func (c RealHTTPClient) MakeRequest(req *http.Request) (*http.Response, error) {
} }
var sampleConfig = ` var sampleConfig = `
# Specify services via an array of tables
[[plugins.httpjson.services]]
# a name for the service being polled # a name for the service being polled
name = "webserver_stats" name = "webserver_stats"
@ -70,7 +63,7 @@ var sampleConfig = `
# ] # ]
# HTTP parameters (all values must be strings) # HTTP parameters (all values must be strings)
[plugins.httpjson.services.parameters] [plugins.httpjson.parameters]
event_type = "cpu_spike" event_type = "cpu_spike"
threshold = "0.75" threshold = "0.75"
` `
@ -87,22 +80,16 @@ func (h *HttpJson) Description() string {
func (h *HttpJson) Gather(acc plugins.Accumulator) error { func (h *HttpJson) Gather(acc plugins.Accumulator) error {
var wg sync.WaitGroup var wg sync.WaitGroup
totalServers := 0 errorChannel := make(chan error, len(h.Servers))
for _, service := range h.Services {
totalServers += len(service.Servers)
}
errorChannel := make(chan error, totalServers)
for _, service := range h.Services { for _, server := range h.Servers {
for _, server := range service.Servers {
wg.Add(1) wg.Add(1)
go func(service Service, server string) { go func(server string) {
defer wg.Done() defer wg.Done()
if err := h.gatherServer(acc, service, server); err != nil { if err := h.gatherServer(acc, server); err != nil {
errorChannel <- err errorChannel <- err
} }
}(service, server) }(server)
}
} }
wg.Wait() wg.Wait()
@ -130,10 +117,9 @@ func (h *HttpJson) Gather(acc plugins.Accumulator) error {
// error: Any error that may have occurred // error: Any error that may have occurred
func (h *HttpJson) gatherServer( func (h *HttpJson) gatherServer(
acc plugins.Accumulator, acc plugins.Accumulator,
service Service,
serverURL string, serverURL string,
) error { ) error {
resp, err := h.sendRequest(service, serverURL) resp, err := h.sendRequest(serverURL)
if err != nil { if err != nil {
return err return err
} }
@ -147,7 +133,7 @@ func (h *HttpJson) gatherServer(
"server": serverURL, "server": serverURL,
} }
for _, tag := range service.TagKeys { for _, tag := range h.TagKeys {
switch v := jsonOut[tag].(type) { switch v := jsonOut[tag].(type) {
case string: case string:
tags[tag] = v tags[tag] = v
@ -162,10 +148,10 @@ func (h *HttpJson) gatherServer(
} }
var msrmnt_name string var msrmnt_name string
if service.Name == "" { if h.Name == "" {
msrmnt_name = "httpjson" msrmnt_name = "httpjson"
} else { } else {
msrmnt_name = "httpjson_" + service.Name msrmnt_name = "httpjson_" + h.Name
} }
acc.AddFields(msrmnt_name, f.Fields, nil) acc.AddFields(msrmnt_name, f.Fields, nil)
return nil return nil
@ -178,7 +164,7 @@ func (h *HttpJson) gatherServer(
// Returns: // Returns:
// string: body of the response // string: body of the response
// error : Any error that may have occurred // error : Any error that may have occurred
func (h *HttpJson) sendRequest(service Service, serverURL string) (string, error) { func (h *HttpJson) sendRequest(serverURL string) (string, error) {
// Prepare URL // Prepare URL
requestURL, err := url.Parse(serverURL) requestURL, err := url.Parse(serverURL)
if err != nil { if err != nil {
@ -186,21 +172,23 @@ func (h *HttpJson) sendRequest(service Service, serverURL string) (string, error
} }
params := url.Values{} params := url.Values{}
for k, v := range service.Parameters { for k, v := range h.Parameters {
params.Add(k, v) params.Add(k, v)
} }
requestURL.RawQuery = params.Encode() requestURL.RawQuery = params.Encode()
// Create + send request // Create + send request
req, err := http.NewRequest(service.Method, requestURL.String(), nil) req, err := http.NewRequest(h.Method, requestURL.String(), nil)
if err != nil { if err != nil {
return "", err return "", err
} }
defer req.Body.Close()
resp, err := h.client.MakeRequest(req) resp, err := h.client.MakeRequest(req)
if err != nil { if err != nil {
return "", err return "", err
} }
defer resp.Body.Close()
defer resp.Body.Close() defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body) body, err := ioutil.ReadAll(resp.Body)

11
plugins/jolokia/jolokia.go
View File

@ -62,17 +62,6 @@ func (j *Jolokia) SampleConfig() string {
[[plugins.jolokia.metrics]] [[plugins.jolokia.metrics]]
name = "heap_memory_usage" name = "heap_memory_usage"
jmx = "/java.lang:type=Memory/HeapMemoryUsage" jmx = "/java.lang:type=Memory/HeapMemoryUsage"
# This drops the 'committed' value from Eden space measurement
[[plugins.jolokia.metrics]]
name = "memory_eden"
jmx = "/java.lang:type=MemoryPool,name=PS Eden Space/Usage"
# This passes only DaemonThreadCount and ThreadCount
[[plugins.jolokia.metrics]]
name = "heap_threads"
jmx = "/java.lang:type=Threading"
` `
} }