Parse statsd lines with multiple metric bits

closes #354

CHANGELOG.md

@@ -1,5 +1,40 @@
 ## v0.2.3 [unreleased]
 
+### Release Notes
+- **breaking change** The `kafka` plugin has been renamed to `kafka_consumer`.
+and most of the config option names have changed.
+This only affects the kafka consumer _plugin_ (not the
+output). There were a number of problems with the kafka plugin that led to it
+only collecting data once at startup, so the kafka plugin was basically non-
+functional.
+- Plugins can now be specified as a list, and multiple plugin instances of the
+same type can be specified, like this:
+
+```
+[[plugins.cpu]]
+  percpu = false
+  totalcpu = true
+
+[[plugins.cpu]]
+  percpu = true
+  totalcpu = false
+  drop = ["cpu_time"]
+```
+
+- Riemann output added
+- Aerospike plugin: tag changed from `host` -> `aerospike_host`
+
+### Features
+- [#379](https://github.com/influxdb/telegraf/pull/379): Riemann output, thanks @allenj!
+- [#375](https://github.com/influxdb/telegraf/pull/375): kafka_consumer service plugin.
+- [#392](https://github.com/influxdb/telegraf/pull/392): Procstat plugin can now accept pgrep -f pattern, thanks @ecarreras!
+- [#383](https://github.com/influxdb/telegraf/pull/383): Specify plugins as a list.
+- [#354](https://github.com/influxdb/telegraf/pull/354): Add ability to specify multiple metrics in one statsd line. Thanks @MerlinDMC!
+
+### Bugfixes
+- [#371](https://github.com/influxdb/telegraf/issues/371): Kafka consumer plugin not functioning.
+- [#389](https://github.com/influxdb/telegraf/issues/389): NaN value panic
+
 ## v0.2.2 [2015-11-18]
 
 ### Release Notes

Godeps/Godeps.json

@@ -22,6 +22,10 @@
 			"Comment": "v0.8.6-7-g9c060de",
 			"Rev": "9c060de643590dae45da9d7c26276463bfc46fa0"
 		},
+		{
+			"ImportPath": "github.com/amir/raidman",
+			"Rev": "6a8e089bbe32e6b907feae5ba688841974b3c339"
+		},
 		{
 			"ImportPath": "github.com/armon/go-metrics",
 			"Rev": "b2d95e5291cdbc26997d1301a5e467ecbb240e25"
@@ -164,7 +168,47 @@
 			"Rev": "5bb5cfc093ad18a28148c578f8632cfdb4d802e4"
 		},
 		{
-			"ImportPath": "github.com/shirou/gopsutil",
+			"ImportPath": "github.com/shirou/gopsutil/common",
+			"Comment": "1.0.0-173-g1e9aabb",
+			"Rev": "1e9aabb3c8132314662698c9d1c0aef68d9da617"
+		},
+		{
+			"ImportPath": "github.com/shirou/gopsutil/cpu",
+			"Comment": "1.0.0-173-g1e9aabb",
+			"Rev": "1e9aabb3c8132314662698c9d1c0aef68d9da617"
+		},
+		{
+			"ImportPath": "github.com/shirou/gopsutil/disk",
+			"Comment": "1.0.0-173-g1e9aabb",
+			"Rev": "1e9aabb3c8132314662698c9d1c0aef68d9da617"
+		},
+		{
+			"ImportPath": "github.com/shirou/gopsutil/docker",
+			"Comment": "1.0.0-173-g1e9aabb",
+			"Rev": "1e9aabb3c8132314662698c9d1c0aef68d9da617"
+		},
+		{
+			"ImportPath": "github.com/shirou/gopsutil/host",
+			"Comment": "1.0.0-173-g1e9aabb",
+			"Rev": "1e9aabb3c8132314662698c9d1c0aef68d9da617"
+		},
+		{
+			"ImportPath": "github.com/shirou/gopsutil/load",
+			"Comment": "1.0.0-173-g1e9aabb",
+			"Rev": "1e9aabb3c8132314662698c9d1c0aef68d9da617"
+		},
+		{
+			"ImportPath": "github.com/shirou/gopsutil/mem",
+			"Comment": "1.0.0-173-g1e9aabb",
+			"Rev": "1e9aabb3c8132314662698c9d1c0aef68d9da617"
+		},
+		{
+			"ImportPath": "github.com/shirou/gopsutil/net",
+			"Comment": "1.0.0-173-g1e9aabb",
+			"Rev": "1e9aabb3c8132314662698c9d1c0aef68d9da617"
+		},
+		{
+			"ImportPath": "github.com/shirou/gopsutil/process",
 			"Comment": "1.0.0-173-g1e9aabb",
 			"Rev": "1e9aabb3c8132314662698c9d1c0aef68d9da617"
 		},

Godeps/_workspace/src/github.com/amir/raidman/README.md

@@ -0,0 +1,73 @@
+Raidman
+=======
+
+Go Riemann client
+
+```go
+package main
+
+import (
+        "github.com/amir/raidman"
+)
+
+func main() {
+        c, err := raidman.Dial("tcp", "localhost:5555")
+        if err != nil {
+                panic(err)
+        }
+
+        var event = &raidman.Event{
+                State:   "success",
+                Host:    "raidman",
+                Service: "raidman-sample",
+                Metric:  100,
+                Ttl:     10,
+        }
+
+        // send one event
+        err = c.Send(event)
+        if err != nil {
+                panic(err)
+        }
+        
+        // send multiple events at once
+        err = c.SendMulti([]*raidman.Event{
+                &raidman.Event{
+                        State:   "success",
+                        Host:    "raidman",
+                        Service: "raidman-sample",
+                        Metric:  100,
+                        Ttl:     10,
+                },
+                &raidman.Event{
+                        State:   "failure",
+                        Host:    "raidman",
+                        Service: "raidman-sample",
+                        Metric:  100,
+                        Ttl:     10,
+                },
+                &raidman.Event{
+                        State:   "success",
+                        Host:    "raidman",
+                        Service: "raidman-sample",
+                        Metric:  100,
+                        Ttl:     10,
+                },
+        })
+        if err != nil {
+                panic(err)
+        }
+
+        events, err := c.Query("host = \"raidman\"")
+        if err != nil {
+                panic(err)
+        }
+
+        if len(events) < 1 {
+                panic("Submitted event not found")
+        }
+
+        c.Close()
+}
+
+```

Godeps/_workspace/src/github.com/amir/raidman/UNLICENSE

@@ -0,0 +1,24 @@
+This is free and unencumbered software released into the public domain.
+
+Anyone is free to copy, modify, publish, use, compile, sell, or
+distribute this software, either in source code form or as a compiled
+binary, for any purpose, commercial or non-commercial, and by any
+means.
+
+In jurisdictions that recognize copyright laws, the author or authors
+of this software dedicate any and all copyright interest in the
+software to the public domain. We make this dedication for the benefit
+of the public at large and to the detriment of our heirs and
+successors. We intend this dedication to be an overt act of
+relinquishment in perpetuity of all present and future rights to this
+software under copyright law.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+OTHER DEALINGS IN THE SOFTWARE.
+
+For more information, please refer to <http://unlicense.org/>

Godeps/_workspace/src/github.com/amir/raidman/proto/Makefile

@@ -0,0 +1,6 @@
+proto.pb.go: proto.proto
+	mkdir -p _pb
+	protoc --go_out=_pb $<
+	cat _pb/$@\
+	|gofmt >$@
+	rm -rf _pb

Godeps/_workspace/src/github.com/amir/raidman/proto/proto.pb.go

@@ -0,0 +1,273 @@
+// Code generated by protoc-gen-go.
+// source: proto.proto
+// DO NOT EDIT!
+
+package proto
+
+import proto1 "github.com/golang/protobuf/proto"
+import json "encoding/json"
+import math "math"
+
+// Reference proto, json, and math imports to suppress error if they are not otherwise used.
+var _ = proto1.Marshal
+var _ = &json.SyntaxError{}
+var _ = math.Inf
+
+type State struct {
+	Time             *int64   `protobuf:"varint,1,opt,name=time" json:"time,omitempty"`
+	State            *string  `protobuf:"bytes,2,opt,name=state" json:"state,omitempty"`
+	Service          *string  `protobuf:"bytes,3,opt,name=service" json:"service,omitempty"`
+	Host             *string  `protobuf:"bytes,4,opt,name=host" json:"host,omitempty"`
+	Description      *string  `protobuf:"bytes,5,opt,name=description" json:"description,omitempty"`
+	Once             *bool    `protobuf:"varint,6,opt,name=once" json:"once,omitempty"`
+	Tags             []string `protobuf:"bytes,7,rep,name=tags" json:"tags,omitempty"`
+	Ttl              *float32 `protobuf:"fixed32,8,opt,name=ttl" json:"ttl,omitempty"`
+	XXX_unrecognized []byte   `json:"-"`
+}
+
+func (this *State) Reset()         { *this = State{} }
+func (this *State) String() string { return proto1.CompactTextString(this) }
+func (*State) ProtoMessage()       {}
+
+func (this *State) GetTime() int64 {
+	if this != nil && this.Time != nil {
+		return *this.Time
+	}
+	return 0
+}
+
+func (this *State) GetState() string {
+	if this != nil && this.State != nil {
+		return *this.State
+	}
+	return ""
+}
+
+func (this *State) GetService() string {
+	if this != nil && this.Service != nil {
+		return *this.Service
+	}
+	return ""
+}
+
+func (this *State) GetHost() string {
+	if this != nil && this.Host != nil {
+		return *this.Host
+	}
+	return ""
+}
+
+func (this *State) GetDescription() string {
+	if this != nil && this.Description != nil {
+		return *this.Description
+	}
+	return ""
+}
+
+func (this *State) GetOnce() bool {
+	if this != nil && this.Once != nil {
+		return *this.Once
+	}
+	return false
+}
+
+func (this *State) GetTags() []string {
+	if this != nil {
+		return this.Tags
+	}
+	return nil
+}
+
+func (this *State) GetTtl() float32 {
+	if this != nil && this.Ttl != nil {
+		return *this.Ttl
+	}
+	return 0
+}
+
+type Event struct {
+	Time             *int64       `protobuf:"varint,1,opt,name=time" json:"time,omitempty"`
+	State            *string      `protobuf:"bytes,2,opt,name=state" json:"state,omitempty"`
+	Service          *string      `protobuf:"bytes,3,opt,name=service" json:"service,omitempty"`
+	Host             *string      `protobuf:"bytes,4,opt,name=host" json:"host,omitempty"`
+	Description      *string      `protobuf:"bytes,5,opt,name=description" json:"description,omitempty"`
+	Tags             []string     `protobuf:"bytes,7,rep,name=tags" json:"tags,omitempty"`
+	Ttl              *float32     `protobuf:"fixed32,8,opt,name=ttl" json:"ttl,omitempty"`
+	Attributes       []*Attribute `protobuf:"bytes,9,rep,name=attributes" json:"attributes,omitempty"`
+	MetricSint64     *int64       `protobuf:"zigzag64,13,opt,name=metric_sint64" json:"metric_sint64,omitempty"`
+	MetricD          *float64     `protobuf:"fixed64,14,opt,name=metric_d" json:"metric_d,omitempty"`
+	MetricF          *float32     `protobuf:"fixed32,15,opt,name=metric_f" json:"metric_f,omitempty"`
+	XXX_unrecognized []byte       `json:"-"`
+}
+
+func (this *Event) Reset()         { *this = Event{} }
+func (this *Event) String() string { return proto1.CompactTextString(this) }
+func (*Event) ProtoMessage()       {}
+
+func (this *Event) GetTime() int64 {
+	if this != nil && this.Time != nil {
+		return *this.Time
+	}
+	return 0
+}
+
+func (this *Event) GetState() string {
+	if this != nil && this.State != nil {
+		return *this.State
+	}
+	return ""
+}
+
+func (this *Event) GetService() string {
+	if this != nil && this.Service != nil {
+		return *this.Service
+	}
+	return ""
+}
+
+func (this *Event) GetHost() string {
+	if this != nil && this.Host != nil {
+		return *this.Host
+	}
+	return ""
+}
+
+func (this *Event) GetDescription() string {
+	if this != nil && this.Description != nil {
+		return *this.Description
+	}
+	return ""
+}
+
+func (this *Event) GetTags() []string {
+	if this != nil {
+		return this.Tags
+	}
+	return nil
+}
+
+func (this *Event) GetTtl() float32 {
+	if this != nil && this.Ttl != nil {
+		return *this.Ttl
+	}
+	return 0
+}
+
+func (this *Event) GetAttributes() []*Attribute {
+	if this != nil {
+		return this.Attributes
+	}
+	return nil
+}
+
+func (this *Event) GetMetricSint64() int64 {
+	if this != nil && this.MetricSint64 != nil {
+		return *this.MetricSint64
+	}
+	return 0
+}
+
+func (this *Event) GetMetricD() float64 {
+	if this != nil && this.MetricD != nil {
+		return *this.MetricD
+	}
+	return 0
+}
+
+func (this *Event) GetMetricF() float32 {
+	if this != nil && this.MetricF != nil {
+		return *this.MetricF
+	}
+	return 0
+}
+
+type Query struct {
+	String_          *string `protobuf:"bytes,1,opt,name=string" json:"string,omitempty"`
+	XXX_unrecognized []byte  `json:"-"`
+}
+
+func (this *Query) Reset()         { *this = Query{} }
+func (this *Query) String() string { return proto1.CompactTextString(this) }
+func (*Query) ProtoMessage()       {}
+
+func (this *Query) GetString_() string {
+	if this != nil && this.String_ != nil {
+		return *this.String_
+	}
+	return ""
+}
+
+type Msg struct {
+	Ok               *bool    `protobuf:"varint,2,opt,name=ok" json:"ok,omitempty"`
+	Error            *string  `protobuf:"bytes,3,opt,name=error" json:"error,omitempty"`
+	States           []*State `protobuf:"bytes,4,rep,name=states" json:"states,omitempty"`
+	Query            *Query   `protobuf:"bytes,5,opt,name=query" json:"query,omitempty"`
+	Events           []*Event `protobuf:"bytes,6,rep,name=events" json:"events,omitempty"`
+	XXX_unrecognized []byte   `json:"-"`
+}
+
+func (this *Msg) Reset()         { *this = Msg{} }
+func (this *Msg) String() string { return proto1.CompactTextString(this) }
+func (*Msg) ProtoMessage()       {}
+
+func (this *Msg) GetOk() bool {
+	if this != nil && this.Ok != nil {
+		return *this.Ok
+	}
+	return false
+}
+
+func (this *Msg) GetError() string {
+	if this != nil && this.Error != nil {
+		return *this.Error
+	}
+	return ""
+}
+
+func (this *Msg) GetStates() []*State {
+	if this != nil {
+		return this.States
+	}
+	return nil
+}
+
+func (this *Msg) GetQuery() *Query {
+	if this != nil {
+		return this.Query
+	}
+	return nil
+}
+
+func (this *Msg) GetEvents() []*Event {
+	if this != nil {
+		return this.Events
+	}
+	return nil
+}
+
+type Attribute struct {
+	Key              *string `protobuf:"bytes,1,req,name=key" json:"key,omitempty"`
+	Value            *string `protobuf:"bytes,2,opt,name=value" json:"value,omitempty"`
+	XXX_unrecognized []byte  `json:"-"`
+}
+
+func (this *Attribute) Reset()         { *this = Attribute{} }
+func (this *Attribute) String() string { return proto1.CompactTextString(this) }
+func (*Attribute) ProtoMessage()       {}
+
+func (this *Attribute) GetKey() string {
+	if this != nil && this.Key != nil {
+		return *this.Key
+	}
+	return ""
+}
+
+func (this *Attribute) GetValue() string {
+	if this != nil && this.Value != nil {
+		return *this.Value
+	}
+	return ""
+}
+
+func init() {
+}

Godeps/_workspace/src/github.com/amir/raidman/proto/proto.proto

@@ -0,0 +1,45 @@
+option java_package = "com.aphyr.riemann";
+option java_outer_classname = "Proto";
+
+message State {
+  optional int64 time = 1;
+  optional string state = 2;
+  optional string service = 3;
+  optional string host = 4;
+  optional string description = 5;
+  optional bool once = 6;
+  repeated string tags = 7;
+  optional float ttl = 8;
+}
+
+message Event {
+  optional int64 time = 1;
+  optional string state = 2;
+  optional string service = 3;
+  optional string host = 4;
+  optional string description = 5;
+  repeated string tags = 7;
+  optional float ttl = 8;
+  repeated Attribute attributes = 9;
+
+  optional sint64 metric_sint64 = 13;
+  optional double metric_d = 14;
+  optional float metric_f = 15;
+}
+
+message Query {
+  optional string string = 1;
+}
+
+message Msg {
+  optional bool ok = 2;
+  optional string error = 3;
+  repeated State states = 4;
+  optional Query query = 5;
+  repeated Event events = 6;
+}
+
+message Attribute {
+  required string key = 1;
+  optional string value = 2;
+}

Godeps/_workspace/src/github.com/amir/raidman/raidman.go

@@ -0,0 +1,313 @@
+// Go Riemann client
+package raidman
+
+import (
+	"bytes"
+	"encoding/binary"
+	"errors"
+	"fmt"
+	"io"
+	"net"
+	"os"
+	"reflect"
+	"sync"
+	"time"
+
+	"github.com/amir/raidman/proto"
+	pb "github.com/golang/protobuf/proto"
+)
+
+type network interface {
+	Send(message *proto.Msg, conn net.Conn) (*proto.Msg, error)
+}
+
+type tcp struct{}
+
+type udp struct{}
+
+// Client represents a connection to a Riemann server
+type Client struct {
+	sync.Mutex
+	net        network
+	connection net.Conn
+	timeout    time.Duration
+}
+
+// An Event represents a single Riemann event
+type Event struct {
+	Ttl         float32
+	Time        int64
+	Tags        []string
+	Host        string // Defaults to os.Hostname()
+	State       string
+	Service     string
+	Metric      interface{} // Could be Int, Float32, Float64
+	Description string
+	Attributes  map[string]string
+}
+
+// Dial establishes a connection to a Riemann server at addr, on the network
+// netwrk, with a timeout of timeout
+//
+// Known networks are "tcp", "tcp4", "tcp6", "udp", "udp4", and "udp6".
+func DialWithTimeout(netwrk, addr string, timeout time.Duration) (c *Client, err error) {
+	c = new(Client)
+
+	var cnet network
+	switch netwrk {
+	case "tcp", "tcp4", "tcp6":
+		cnet = new(tcp)
+	case "udp", "udp4", "udp6":
+		cnet = new(udp)
+	default:
+		return nil, fmt.Errorf("dial %q: unsupported network %q", netwrk, netwrk)
+	}
+
+	c.net = cnet
+	c.timeout = timeout
+	c.connection, err = net.Dial(netwrk, addr)
+	if err != nil {
+		return nil, err
+	}
+
+	return c, nil
+}
+
+// Dial establishes a connection to a Riemann server at addr, on the network
+// netwrk.
+//
+// Known networks are "tcp", "tcp4", "tcp6", "udp", "udp4", and "udp6".
+func Dial(netwrk, addr string) (c *Client, err error) {
+	return DialWithTimeout(netwrk, addr, 0)
+}
+
+func (network *tcp) Send(message *proto.Msg, conn net.Conn) (*proto.Msg, error) {
+	msg := &proto.Msg{}
+	data, err := pb.Marshal(message)
+	if err != nil {
+		return msg, err
+	}
+	b := new(bytes.Buffer)
+	if err = binary.Write(b, binary.BigEndian, uint32(len(data))); err != nil {
+		return msg, err
+	}
+	if _, err = conn.Write(b.Bytes()); err != nil {
+		return msg, err
+	}
+	if _, err = conn.Write(data); err != nil {
+		return msg, err
+	}
+	var header uint32
+	if err = binary.Read(conn, binary.BigEndian, &header); err != nil {
+		return msg, err
+	}
+	response := make([]byte, header)
+	if err = readFully(conn, response); err != nil {
+		return msg, err
+	}
+	if err = pb.Unmarshal(response, msg); err != nil {
+		return msg, err
+	}
+	if msg.GetOk() != true {
+		return msg, errors.New(msg.GetError())
+	}
+	return msg, nil
+}
+
+func readFully(r io.Reader, p []byte) error {
+	for len(p) > 0 {
+		n, err := r.Read(p)
+		p = p[n:]
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (network *udp) Send(message *proto.Msg, conn net.Conn) (*proto.Msg, error) {
+	data, err := pb.Marshal(message)
+	if err != nil {
+		return nil, err
+	}
+	if _, err = conn.Write(data); err != nil {
+		return nil, err
+	}
+
+	return nil, nil
+}
+
+func isZero(v reflect.Value) bool {
+	switch v.Kind() {
+	case reflect.Map:
+		return v.IsNil()
+	case reflect.Slice:
+		zero := true
+		for i := 0; i < v.Len(); i++ {
+			zero = zero && isZero(v.Index(i))
+		}
+		return zero
+	}
+	zero := reflect.Zero(v.Type())
+	return v.Interface() == zero.Interface()
+}
+
+func eventToPbEvent(event *Event) (*proto.Event, error) {
+	var e proto.Event
+
+	if event.Host == "" {
+		event.Host, _ = os.Hostname()
+	}
+	t := reflect.ValueOf(&e).Elem()
+	s := reflect.ValueOf(event).Elem()
+	typeOfEvent := s.Type()
+	for i := 0; i < s.NumField(); i++ {
+		f := s.Field(i)
+		value := reflect.ValueOf(f.Interface())
+		if !isZero(f) {
+			name := typeOfEvent.Field(i).Name
+			switch name {
+			case "State", "Service", "Host", "Description":
+				tmp := reflect.ValueOf(pb.String(value.String()))
+				t.FieldByName(name).Set(tmp)
+			case "Ttl":
+				tmp := reflect.ValueOf(pb.Float32(float32(value.Float())))
+				t.FieldByName(name).Set(tmp)
+			case "Time":
+				tmp := reflect.ValueOf(pb.Int64(value.Int()))
+				t.FieldByName(name).Set(tmp)
+			case "Tags":
+				tmp := reflect.ValueOf(value.Interface().([]string))
+				t.FieldByName(name).Set(tmp)
+			case "Metric":
+				switch reflect.TypeOf(f.Interface()).Kind() {
+				case reflect.Int:
+					tmp := reflect.ValueOf(pb.Int64(int64(value.Int())))
+					t.FieldByName("MetricSint64").Set(tmp)
+				case reflect.Int64:
+					tmp := reflect.ValueOf(pb.Int64(int64(value.Int())))
+					t.FieldByName("MetricSint64").Set(tmp)
+				case reflect.Float32:
+					tmp := reflect.ValueOf(pb.Float32(float32(value.Float())))
+					t.FieldByName("MetricF").Set(tmp)
+				case reflect.Float64:
+					tmp := reflect.ValueOf(pb.Float64(value.Float()))
+					t.FieldByName("MetricD").Set(tmp)
+				default:
+					return nil, fmt.Errorf("Metric of invalid type (type %v)",
+						reflect.TypeOf(f.Interface()).Kind())
+				}
+			case "Attributes":
+				var attrs []*proto.Attribute
+				for k, v := range value.Interface().(map[string]string) {
+					// Copy k,v so we can take
+					// pointers to the new
+					// temporaries
+					k_, v_ := k, v
+					attrs = append(attrs, &proto.Attribute{
+						Key:   &k_,
+						Value: &v_,
+					})
+				}
+				t.FieldByName(name).Set(reflect.ValueOf(attrs))
+			}
+		}
+	}
+
+	return &e, nil
+}
+
+func pbEventsToEvents(pbEvents []*proto.Event) []Event {
+	var events []Event
+
+	for _, event := range pbEvents {
+		e := Event{
+			State:       event.GetState(),
+			Service:     event.GetService(),
+			Host:        event.GetHost(),
+			Description: event.GetDescription(),
+			Ttl:         event.GetTtl(),
+			Time:        event.GetTime(),
+			Tags:        event.GetTags(),
+		}
+		if event.MetricF != nil {
+			e.Metric = event.GetMetricF()
+		} else if event.MetricD != nil {
+			e.Metric = event.GetMetricD()
+		} else {
+			e.Metric = event.GetMetricSint64()
+		}
+		if event.Attributes != nil {
+			e.Attributes = make(map[string]string, len(event.GetAttributes()))
+			for _, attr := range event.GetAttributes() {
+				e.Attributes[attr.GetKey()] = attr.GetValue()
+			}
+		}
+
+		events = append(events, e)
+	}
+
+	return events
+}
+
+// Send sends an event to Riemann
+func (c *Client) Send(event *Event) error {
+	return c.SendMulti([]*Event{event})
+}
+
+// SendMulti sends multiple events to Riemann
+func (c *Client) SendMulti(events []*Event) error {
+	message := &proto.Msg{}
+
+	for _, event := range events {
+		e, err := eventToPbEvent(event)
+		if err != nil {
+			return err
+		}
+
+		message.Events = append(message.Events, e)
+	}
+
+	c.Lock()
+	defer c.Unlock()
+
+	if c.timeout > 0 {
+		err := c.connection.SetDeadline(time.Now().Add(c.timeout))
+		if err != nil {
+			return err
+		}
+	}
+
+	_, err := c.net.Send(message, c.connection)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// Query returns a list of events matched by query
+func (c *Client) Query(q string) ([]Event, error) {
+	switch c.net.(type) {
+	case *udp:
+		return nil, errors.New("Querying over UDP is not supported")
+	}
+	query := &proto.Query{}
+	query.String_ = pb.String(q)
+	message := &proto.Msg{}
+	message.Query = query
+	c.Lock()
+	defer c.Unlock()
+	response, err := c.net.Send(message, c.connection)
+	if err != nil {
+		return nil, err
+	}
+	return pbEventsToEvents(response.GetEvents()), nil
+}
+
+// Close closes the connection to Riemann
+func (c *Client) Close() {
+	c.Lock()
+	c.connection.Close()
+	c.Unlock()
+}

Godeps/_workspace/src/github.com/amir/raidman/raidman_test.go

@@ -0,0 +1,268 @@
+package raidman
+
+import (
+	"reflect"
+	"testing"
+)
+
+func TestTCP(t *testing.T) {
+	c, err := Dial("tcp", "localhost:5555")
+	if err != nil {
+		t.Fatal(err.Error())
+	}
+	var event = &Event{
+		State:      "success",
+		Host:       "raidman",
+		Service:    "tcp",
+		Metric:     42,
+		Ttl:        1,
+		Tags:       []string{"tcp", "test", "raidman"},
+		Attributes: map[string]string{"type": "test"},
+	}
+
+	err = c.Send(event)
+	if err != nil {
+		t.Error(err.Error())
+	}
+
+	events, err := c.Query("tagged \"test\"")
+	if err != nil {
+		t.Error(err.Error())
+	}
+
+	if len(events) < 1 {
+		t.Error("Submitted event not found")
+	}
+
+	testAttributeExists := false
+	for _, event := range events {
+		if val, ok := event.Attributes["type"]; ok && val == "test" {
+			testAttributeExists = true
+		}
+	}
+
+	if !testAttributeExists {
+		t.Error("Attribute \"type\" is missing")
+	}
+
+	c.Close()
+}
+
+func TestMultiTCP(t *testing.T) {
+	c, err := Dial("tcp", "localhost:5555")
+	if err != nil {
+		t.Fatal(err.Error())
+	}
+
+	err = c.SendMulti([]*Event{
+		&Event{
+			State:      "success",
+			Host:       "raidman",
+			Service:    "tcp-multi-1",
+			Metric:     42,
+			Ttl:        1,
+			Tags:       []string{"tcp", "test", "raidman", "multi"},
+			Attributes: map[string]string{"type": "test"},
+		},
+		&Event{
+			State:      "success",
+			Host:       "raidman",
+			Service:    "tcp-multi-2",
+			Metric:     42,
+			Ttl:        1,
+			Tags:       []string{"tcp", "test", "raidman", "multi"},
+			Attributes: map[string]string{"type": "test"},
+		},
+	})
+	if err != nil {
+		t.Error(err.Error())
+	}
+
+	events, err := c.Query("tagged \"test\" and tagged \"multi\"")
+	if err != nil {
+		t.Error(err.Error())
+	}
+
+	if len(events) != 2 {
+		t.Error("Submitted event not found")
+	}
+
+	c.Close()
+}
+
+func TestMetricIsInt64(t *testing.T) {
+	c, err := Dial("tcp", "localhost:5555")
+	if err != nil {
+		t.Fatal(err.Error())
+	}
+
+	var int64metric int64 = 9223372036854775807
+
+	var event = &Event{
+		State:      "success",
+		Host:       "raidman",
+		Service:    "tcp",
+		Metric:     int64metric,
+		Ttl:        1,
+		Tags:       []string{"tcp", "test", "raidman"},
+		Attributes: map[string]string{"type": "test"},
+	}
+
+	err = c.Send(event)
+	if err != nil {
+		t.Error(err.Error())
+	}
+}
+
+func TestUDP(t *testing.T) {
+	c, err := Dial("udp", "localhost:5555")
+	if err != nil {
+		t.Fatal(err.Error())
+	}
+	var event = &Event{
+		State:   "warning",
+		Host:    "raidman",
+		Service: "udp",
+		Metric:  3.4,
+		Ttl:     10.7,
+	}
+
+	err = c.Send(event)
+	if err != nil {
+		t.Error(err.Error())
+	}
+	c.Close()
+}
+
+func TestTCPWithoutHost(t *testing.T) {
+	c, err := Dial("tcp", "localhost:5555")
+	if err != nil {
+		t.Fatal(err.Error())
+	}
+	defer c.Close()
+
+	var event = &Event{
+		State:   "success",
+		Service: "tcp-host-not-set",
+		Ttl:     5,
+	}
+
+	err = c.Send(event)
+	if err != nil {
+		t.Error(err.Error())
+	}
+
+	events, err := c.Query("service = \"tcp-host-not-set\"")
+	if err != nil {
+		t.Error(err.Error())
+	}
+
+	if len(events) < 1 {
+		t.Error("Submitted event not found")
+	}
+
+	for _, e := range events {
+		if e.Host == "" {
+			t.Error("Default host name is not set")
+		}
+	}
+}
+
+func TestIsZero(t *testing.T) {
+	event := &Event{
+		Time: 1,
+	}
+	elem := reflect.ValueOf(event).Elem()
+	eventType := elem.Type()
+	for i := 0; i < elem.NumField(); i++ {
+		field := elem.Field(i)
+		name := eventType.Field(i).Name
+		if name == "Time" {
+			if isZero(field) {
+				t.Error("Time should not be zero")
+			}
+		} else {
+			if !isZero(field) {
+				t.Errorf("%s should be zero", name)
+			}
+		}
+	}
+}
+
+func BenchmarkTCP(b *testing.B) {
+	c, err := Dial("tcp", "localhost:5555")
+
+	var event = &Event{
+		State:   "good",
+		Host:    "raidman",
+		Service: "benchmark",
+	}
+
+	if err == nil {
+		for i := 0; i < b.N; i++ {
+			c.Send(event)
+		}
+	}
+	c.Close()
+}
+
+func BenchmarkUDP(b *testing.B) {
+	c, err := Dial("udp", "localhost:5555")
+
+	var event = &Event{
+		State:   "good",
+		Host:    "raidman",
+		Service: "benchmark",
+	}
+
+	if err == nil {
+		for i := 0; i < b.N; i++ {
+			c.Send(event)
+		}
+	}
+	c.Close()
+}
+
+func BenchmarkConcurrentTCP(b *testing.B) {
+	c, err := Dial("tcp", "localhost:5555")
+
+	var event = &Event{
+		Host:    "raidman",
+		Service: "tcp_concurrent",
+		Tags:    []string{"concurrent", "tcp", "benchmark"},
+	}
+
+	ch := make(chan int, b.N)
+	for i := 0; i < b.N; i++ {
+		go func(metric int) {
+			event.Metric = metric
+			err = c.Send(event)
+			ch <- i
+		}(i)
+	}
+	<-ch
+
+	c.Close()
+}
+
+func BenchmarkConcurrentUDP(b *testing.B) {
+	c, err := Dial("udp", "localhost:5555")
+
+	var event = &Event{
+		Host:    "raidman",
+		Service: "udp_concurrent",
+		Tags:    []string{"concurrent", "udp", "benchmark"},
+	}
+
+	ch := make(chan int, b.N)
+	for i := 0; i < b.N; i++ {
+		go func(metric int) {
+			event.Metric = metric
+			err = c.Send(event)
+			ch <- i
+		}(i)
+	}
+	<-ch
+
+	c.Close()
+}

Godeps/_workspace/src/github.com/shirou/gopsutil/.gitignore

@@ -1,4 +0,0 @@
-*~
-#*
-_obj
-*.tmp

Godeps/_workspace/src/github.com/shirou/gopsutil/LICENSE

@@ -1,27 +0,0 @@
-gopsutil is distributed under BSD license reproduced below.
-
-Copyright (c) 2014, WAKAYAMA Shirou
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without modification,
-are permitted provided that the following conditions are met:
-
- * Redistributions of source code must retain the above copyright notice, this
-   list of conditions and the following disclaimer.
- * Redistributions in binary form must reproduce the above copyright notice,
-   this list of conditions and the following disclaimer in the documentation
-   and/or other materials provided with the distribution.
- * Neither the name of the gopsutil authors nor the names of its contributors
-   may be used to endorse or promote products derived from this software without
-   specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
-ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
-ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
-(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
-LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
-ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

Godeps/_workspace/src/github.com/shirou/gopsutil/README.rst

@@ -1,270 +0,0 @@
-gopsutil: psutil for golang
-==============================
-
-.. image:: https://drone.io/github.com/shirou/gopsutil/status.png
-        :target: https://drone.io/github.com/shirou/gopsutil
-
-.. image:: https://coveralls.io/repos/shirou/gopsutil/badge.png?branch=master
-        :target: https://coveralls.io/r/shirou/gopsutil?branch=master
-
-
-This is a port of psutil (http://pythonhosted.org/psutil/). The challenge is porting all
-psutil functions on some architectures...
-
-.. highlights:: Package Structure Changed!
-
-   Package (a.k.a. directory) structure has been changed!! see `issue 24 <https://github.com/shirou/gopsutil/issues/24>`_
-
-.. highlights:: golang 1.4 will become REQUIRED!
-
-   Since syscall package becomes frozen, we should use golang/x/sys of golang 1.4 as soon as possible.
-
-
-Available Architectures
-------------------------------------
-
-- FreeBSD i386/amd64
-- Linux i386/amd64/arm(raspberry pi)
-- Windows/amd64
-- Darwin/amd64
-
-All works are implemented without cgo by porting c struct to golang struct.
-
-
-Usage
----------
-
-.. code:: go
-
-   import (
-   	"fmt"
-
-   	"github.com/shirou/gopsutil/mem"
-   )
-
-   func main() {
-   	v, _ := mem.VirtualMemory()
-
-   	// almost every return value is a struct
-   	fmt.Printf("Total: %v, Free:%v, UsedPercent:%f%%\n", v.Total, v.Free, v.UsedPercent)
-
-   	// convert to JSON. String() is also implemented
-   	fmt.Println(v)
-   }
-
-The output is below.
-
-::
-
-  Total: 3179569152, Free:284233728, UsedPercent:84.508194%
-  {"total":3179569152,"available":492572672,"used":2895335424,"usedPercent":84.50819439828305, (snip)}
-
-You can set an alternative location to /proc by setting the HOST_PROC environment variable.
-
-Documentation
-------------------------
-
-see http://godoc.org/github.com/shirou/gopsutil
-
-
-More Info
---------------------
-
-Several methods have been added which are not present in psutil, but will provide useful information.
-
-- host/HostInfo()  (linux)
-
-  - Hostname
-  - Uptime
-  - Procs
-  - OS                    (ex: "linux")
-  - Platform              (ex: "ubuntu", "arch")
-  - PlatformFamily        (ex: "debian")
-  - PlatformVersion       (ex: "Ubuntu 13.10")
-  - VirtualizationSystem  (ex: "LXC")
-  - VirtualizationRole    (ex: "guest"/"host")
-
-- cpu/CPUInfo()  (linux, freebsd)
-
-  - CPU          (ex: 0, 1, ...)
-  - VendorID     (ex: "GenuineIntel")
-  - Family
-  - Model
-  - Stepping
-  - PhysicalID
-  - CoreID
-  - Cores        (ex: 2)
-  - ModelName    (ex: "Intel(R) Core(TM) i7-2640M CPU @ 2.80GHz")
-  - Mhz
-  - CacheSize
-  - Flags        (ex: "fpu vme de pse tsc msr pae mce cx8 ...")
-
-- load/LoadAvg()  (linux, freebsd)
-
-  - Load1
-  - Load5
-  - Load15
-
-- docker/GetDockerIDList() (linux only)
-
-  - container id list ([]string)
-
-- docker/CgroupCPU() (linux only)
-
-  - user
-  - system
-
-- docker/CgroupMem() (linux only)
-
-  - various status
-
-Some codes are ported from Ohai. many thanks.
-
-
-Current Status
-------------------
-
-- x: work
-- b: almost work but something broken
-
-================= ====== ======= ====== =======
-name              Linux  FreeBSD MacOSX Windows
-cpu_times            x      x      x       x
-cpu_count            x      x      x       x
-cpu_percent          x      x      x       x
-cpu_times_percent    x      x      x       x
-virtual_memory       x      x      x       x
-swap_memory          x      x      x
-disk_partitions      x      x      x       x
-disk_io_counters     x      x
-disk_usage           x      x      x       x
-net_io_counters      x      x      b       x
-boot_time            x      x      x       x
-users                x      x      x       x
-pids                 x      x      x       x
-pid_exists           x      x      x       x
-net_connections      x             x
-net_if_addrs
-net_if_stats
-================= ====== ======= ====== =======
-
-Process class
-^^^^^^^^^^^^^^^
-
-================ ===== ======= ====== =======
-name             Linux FreeBSD MacOSX Windows
-pid                 x     x      x       x
-ppid                x     x      x       x
-name                x     x      x       x
-cmdline             x            x       x
-create_time         x
-status              x     x      x
-cwd                 x
-exe                 x     x              x
-uids                x     x      x
-gids                x     x      x
-terminal            x     x      x
-io_counters         x
-nice                x            x       x
-num_fds             x
-num_ctx_switches    x
-num_threads         x     x      x       x
-cpu_times           x
-memory_info         x     x      x
-memory_info_ex      x
-memory_maps         x
-open_files          x
-send_signal         x     x      x
-suspend             x     x      x
-resume              x     x      x
-terminate           x     x      x
-kill                x     x      x
-username            x
-ionice
-rlimit
-num_handlres
-threads
-cpu_percent         x            x
-cpu_affinity
-memory_percent
-parent              x            x
-children
-connections         x            x
-is_running
-================ ===== ======= ====== =======
-
-Original Metrics
-^^^^^^^^^^^^^^^^^^^
-================== ===== ======= ====== =======
-item               Linux FreeBSD MacOSX Windows
-**HostInfo**
-hostname              x     x      x       x
-  uptime              x     x      x
-  proces              x     x
-  os                  x     x      x       x
-  platform            x     x      x
-  platformfamiliy     x     x      x
-  virtualization      x
-**CPU**
-  VendorID            x     x      x       x
-  Family              x     x      x       x
-  Model               x     x      x       x
-  Stepping            x     x      x       x
-  PhysicalID          x
-  CoreID              x
-  Cores               x                    x
-  ModelName           x     x      x       x
-**LoadAvg**
-  Load1               x     x      x
-  Load5               x     x      x
-  Load15              x     x      x
-**GetDockerID**
-  container id        x     no    no      no
-**CgroupsCPU**
-  user                x     no    no      no
-  system              x     no    no      no
-**CgroupsMem**
-  various             x     no    no      no
-================== ===== ======= ====== =======
-
-- future work
-
-  - process_iter
-  - wait_procs
-  - Process class
-
-    - as_dict
-    - wait
-
-
-License
-------------
-
-New BSD License (same as psutil)
-
-
-Related Works
------------------------
-
-I have been influenced by the following great works:
-
-- psutil: http://pythonhosted.org/psutil/
-- dstat: https://github.com/dagwieers/dstat
-- gosigar: https://github.com/cloudfoundry/gosigar/
-- goprocinfo: https://github.com/c9s/goprocinfo
-- go-ps: https://github.com/mitchellh/go-ps
-- ohai: https://github.com/opscode/ohai/
-- bosun: https://github.com/bosun-monitor/bosun/tree/master/cmd/scollector/collectors
-- mackerel: https://github.com/mackerelio/mackerel-agent/tree/master/metrics
-
-How to Contribute
----------------------------
-
-1. Fork it
-2. Create your feature branch (git checkout -b my-new-feature)
-3. Commit your changes (git commit -am 'Add some feature')
-4. Push to the branch (git push origin my-new-feature)
-5. Create new Pull Request
-
-My English is terrible, so documentation or correcting comments are also
-welcome.

Godeps/_workspace/src/github.com/shirou/gopsutil/coverall.sh

@@ -1,26 +0,0 @@
-#/bin/sh
-
-# see http://www.songmu.jp/riji/entry/2015-01-15-goveralls-multi-package.html
-
-set -e
-# cleanup
-cleanup() {
-  if [ $tmpprof != "" ] && [ -f $tmpprof ]; then
-    rm -f $tmpprof
-  fi
-  exit
-}
-trap cleanup INT QUIT TERM EXIT
-
-# メインの処理
-prof=${1:-".profile.cov"}
-echo "mode: count" > $prof
-gopath1=$(echo $GOPATH | cut -d: -f1)
-for pkg in $(go list ./...); do
-  tmpprof=$gopath1/src/$pkg/profile.tmp
-  go test -covermode=count -coverprofile=$tmpprof $pkg
-  if [ -f $tmpprof ]; then
-    cat $tmpprof | tail -n +2 >> $prof
-    rm $tmpprof
-  fi
-done

Godeps/_workspace/src/github.com/shirou/gopsutil/doc.go

@@ -1 +0,0 @@
-package gopsutil

Godeps/_workspace/src/github.com/shirou/gopsutil/mktypes.sh

@@ -1,37 +0,0 @@
-
-DIRS="cpu disk docker host load mem net process"
-
-GOOS=`uname | tr '[:upper:]' '[:lower:]'`
-ARCH=`uname -m`
-
-case $ARCH in
-	amd64)
-		GOARCH="amd64"
-		;;
-	x86_64)
-		GOARCH="amd64"
-		;;
-	i386)
-		GOARCH="386"
-		;;
-	i686)
-		GOARCH="386"
-		;;
-	arm)
-		GOARCH="arm"
-		;;
-	*)
-		echo "unknown arch: $ARCH"
-		exit 1
-esac
-
-for DIR in $DIRS
-do
-	if [ -e ${DIR}/types_${GOOS}.go ]; then
-		echo "// +build $GOOS" > ${DIR}/${DIR}_${GOOS}_${GOARCH}.go
-		echo "// +build $GOARCH" >> ${DIR}/${DIR}_${GOOS}_${GOARCH}.go
-		go tool cgo -godefs ${DIR}/types_${GOOS}.go >> ${DIR}/${DIR}_${GOOS}_${GOARCH}.go
-	fi
-done
-
-

Godeps/_workspace/src/github.com/shirou/gopsutil/windows_memo.rst

@@ -1,36 +0,0 @@
-Windows memo
-=====================
-
-Size
-----------
-
-DWORD
-  32-bit unsigned integer
-DWORDLONG
-  64-bit unsigned integer
-DWORD_PTR
-  unsigned long type for pointer precision
-DWORD32
-  32-bit unsigned integer
-DWORD64
-  64-bit unsigned integer
-HALF_PTR
-  _WIN64 = int, else short
-INT
-  32-bit signed integer
-INT_PTR
-  _WIN64 = __int64 else int
-LONG
-  32-bit signed integer
-LONGLONG
-  64-bit signed integer
-LONG_PTR
-  _WIN64 = __int64 else long
-SHORT
-  16-bit integer
-SIZE_T
-  maximum number of bytes to which a pointer can point. typedef ULONG_PTR SIZE_T;
-SSIZE_T
-  signed version of SIZE_T. typedef LONG_PTR SSIZE_T;
-WORD
-  16-bit unsigned integer

Makefile

@@ -1,30 +1,32 @@
 UNAME := $(shell sh -c 'uname')
 VERSION := $(shell sh -c 'git describe --always --tags')
-ifndef GOBIN
-	GOBIN = $(GOPATH)/bin
+ifdef GOBIN
+PATH := $(GOBIN):$(PATH)
+else
+PATH := $(subst :,/bin:,$(GOPATH))/bin:$(PATH)
 endif
 
 # Standard Telegraf build
 build: prepare
-	$(GOBIN)/godep go build -o telegraf -ldflags \
+	godep go build -o telegraf -ldflags \
 		"-X main.Version=$(VERSION)" \
 		./cmd/telegraf/telegraf.go
 
 # Build with race detector
 dev: prepare
-	$(GOBIN)/godep go build -race -o telegraf -ldflags \
+	godep go build -race -o telegraf -ldflags \
 		"-X main.Version=$(VERSION)" \
 		./cmd/telegraf/telegraf.go
 
 # Build linux 64-bit, 32-bit and arm architectures
 build-linux-bins: prepare
-	GOARCH=amd64 GOOS=linux $(GOBIN)/godep go build -o telegraf_linux_amd64 \
+	GOARCH=amd64 GOOS=linux godep go build -o telegraf_linux_amd64 \
 								-ldflags "-X main.Version=$(VERSION)" \
 								./cmd/telegraf/telegraf.go
-	GOARCH=386 GOOS=linux $(GOBIN)/godep go build -o telegraf_linux_386 \
+	GOARCH=386 GOOS=linux godep go build -o telegraf_linux_386 \
 								-ldflags "-X main.Version=$(VERSION)" \
 								./cmd/telegraf/telegraf.go
-	GOARCH=arm GOOS=linux $(GOBIN)/godep go build -o telegraf_linux_arm \
+	GOARCH=arm GOOS=linux godep go build -o telegraf_linux_arm \
 								-ldflags "-X main.Version=$(VERSION)" \
 								./cmd/telegraf/telegraf.go
 
@@ -57,6 +59,7 @@ endif
 	docker run --name aerospike -p "3000:3000" -d aerospike
 	docker run --name nsq -p "4150:4150" -d nsqio/nsq /nsqd
 	docker run --name mqtt -p "1883:1883" -d ncarlier/mqtt
+	docker run --name riemann -p "5555:5555" -d blalor/riemann
 
 # Run docker containers necessary for CircleCI unit tests
 docker-run-circle:
@@ -69,11 +72,12 @@ docker-run-circle:
 	docker run --name aerospike -p "3000:3000" -d aerospike
 	docker run --name nsq -p "4150:4150" -d nsqio/nsq /nsqd
 	docker run --name mqtt -p "1883:1883" -d ncarlier/mqtt
+	docker run --name riemann -p "5555:5555" -d blalor/riemann
 
 # Kill all docker containers, ignore errors
 docker-kill:
-	-docker kill nsq aerospike redis opentsdb rabbitmq postgres memcached mysql kafka mqtt
-	-docker rm nsq aerospike redis opentsdb rabbitmq postgres memcached mysql kafka mqtt
+	-docker kill nsq aerospike redis opentsdb rabbitmq postgres memcached mysql kafka mqtt riemann
+	-docker rm nsq aerospike redis opentsdb rabbitmq postgres memcached mysql kafka mqtt riemann
 
 # Run full unit tests using docker containers (includes setup and teardown)
 test: docker-kill prepare docker-run
@@ -84,6 +88,6 @@ test: docker-kill prepare docker-run
 
 # Run "short" unit tests
 test-short: prepare
-	$(GOBIN)/godep go test -short ./...
+	godep go test -short ./...
 
 .PHONY: test

README.md

@@ -18,8 +18,8 @@ writing new plugins.
 ### Linux deb and rpm packages:
 
 Latest:
-* http://get.influxdb.org/telegraf/telegraf_0.2.1_amd64.deb
-* http://get.influxdb.org/telegraf/telegraf-0.2.1-1.x86_64.rpm
+* http://get.influxdb.org/telegraf/telegraf_0.2.2_amd64.deb
+* http://get.influxdb.org/telegraf/telegraf-0.2.2-1.x86_64.rpm
 
 ##### Package instructions:
 
@@ -33,9 +33,9 @@ controlled via `systemctl [action] telegraf`
 ### Linux binaries:
 
 Latest:
-* http://get.influxdb.org/telegraf/telegraf_linux_amd64_0.2.1.tar.gz
-* http://get.influxdb.org/telegraf/telegraf_linux_386_0.2.1.tar.gz
-* http://get.influxdb.org/telegraf/telegraf_linux_arm_0.2.1.tar.gz
+* http://get.influxdb.org/telegraf/telegraf_linux_amd64_0.2.2.tar.gz
+* http://get.influxdb.org/telegraf/telegraf_linux_386_0.2.2.tar.gz
+* http://get.influxdb.org/telegraf/telegraf_linux_arm_0.2.2.tar.gz
 
 ##### Binary instructions:
 
@@ -110,37 +110,45 @@ you can configure that here.
 
 This is a full working config that will output CPU data to an InfluxDB instance
 at 192.168.59.103:8086, tagging measurements with dc="denver-1". It will output
-measurements at a 10s interval and will collect totalcpu & percpu data.
+measurements at a 10s interval and will collect per-cpu data, dropping any
+measurements which begin with `cpu_time`.
 
 ```
 [tags]
-    dc = "denver-1"
+  dc = "denver-1"
 
 [agent]
-    interval = "10s"
+  interval = "10s"
 
 # OUTPUTS
 [outputs]
 [[outputs.influxdb]]
-    url = "http://192.168.59.103:8086" # required.
-    database = "telegraf" # required.
-    precision = "s"
+  url = "http://192.168.59.103:8086" # required.
+  database = "telegraf" # required.
+  precision = "s"
 
 # PLUGINS
-[cpu]
-    percpu = true
-    totalcpu = true
+[plugins]
+[[plugins.cpu]]
+  percpu = true
+  totalcpu = false
+  drop = ["cpu_time"]
 ```
 
 Below is how to configure `tagpass` and `tagdrop` parameters (added in 0.1.5)
 
 ```
-# Don't collect CPU data for cpu6 & cpu7
-[cpu.tagdrop]
+[plugins]
+[[plugins.cpu]]
+  percpu = true
+  totalcpu = false
+  drop = ["cpu_time"]
+  # Don't collect CPU data for cpu6 & cpu7
+  [plugins.cpu.tagdrop]
     cpu = [ "cpu6", "cpu7" ]
 
-[disk]
-[disk.tagpass]
+[[plugins.disk]]
+  [plugins.disk.tagpass]
     # tagpass conditions are OR, not AND.
     # If the (filesystem is ext4 or xfs) OR (the path is /opt or /home)
     # then the metric passes
@@ -148,6 +156,15 @@ Below is how to configure `tagpass` and `tagdrop` parameters (added in 0.1.5)
     path = [ "/opt", "/home" ]
 ```
 
+Additional plugins (or outputs) of the same type can be specified,
+just define another instance in the config file:
+
+```
+[[plugins.cpu]]
+  percpu = false
+  totalcpu = true
+```
+
 ## Supported Plugins
 
 **You can view usage instructions for each plugin by running**
@@ -164,7 +181,6 @@ Telegraf currently has support for collecting metrics from:
 * haproxy
 * httpjson (generic JSON-emitting http service plugin)
 * jolokia (remote JMX with JSON over HTTP)
-* kafka_consumer
 * leofs
 * lustre2
 * memcached
@@ -197,6 +213,7 @@ Telegraf currently has support for collecting metrics from:
 Telegraf can collect metrics via the following services:
 
 * statsd
+* kafka_consumer
 
 We'll be adding support for many more over the coming months. Read on if you
 want to add support for another service or third-party API.
@@ -219,6 +236,7 @@ found by running `telegraf -sample-config`.
 * librato
 * prometheus
 * amon
+* riemann
 
 ## Contributing
 

accumulator.go

@@ -3,9 +3,12 @@ package telegraf
 import (
 	"fmt"
 	"log"
+	"math"
 	"sync"
 	"time"
 
+	"github.com/influxdb/telegraf/internal/config"
+
 	"github.com/influxdb/influxdb/client/v2"
 )
 
@@ -26,12 +29,12 @@ type Accumulator interface {
 }
 
 func NewAccumulator(
-	plugin *ConfiguredPlugin,
+	pluginConfig *config.PluginConfig,
 	points chan *client.Point,
 ) Accumulator {
 	acc := accumulator{}
 	acc.points = points
-	acc.plugin = plugin
+	acc.pluginConfig = pluginConfig
 	return &acc
 }
 
@@ -44,7 +47,7 @@ type accumulator struct {
 
 	debug bool
 
-	plugin *ConfiguredPlugin
+	pluginConfig *config.PluginConfig
 
 	prefix string
 }
@@ -66,25 +69,32 @@ func (ac *accumulator) AddFields(
 	tags map[string]string,
 	t ...time.Time,
 ) {
-
-	if tags == nil {
-		tags = make(map[string]string)
-	}
-
-	// InfluxDB client/points does not support writing uint64
-	// TODO fix when it does
-	// https://github.com/influxdb/influxdb/pull/4508
+	// Validate uint64 and float64 fields
 	for k, v := range fields {
 		switch val := v.(type) {
 		case uint64:
+			// InfluxDB does not support writing uint64
 			if val < uint64(9223372036854775808) {
 				fields[k] = int64(val)
 			} else {
 				fields[k] = int64(9223372036854775807)
 			}
+		case float64:
+			// NaNs are invalid values in influxdb, skip measurement
+			if math.IsNaN(val) || math.IsInf(val, 0) {
+				if ac.debug {
+					log.Printf("Measurement [%s] has a NaN or Inf field, skipping",
+						measurement)
+				}
+				return
+			}
 		}
 	}
 
+	if tags == nil {
+		tags = make(map[string]string)
+	}
+
 	var timestamp time.Time
 	if len(t) > 0 {
 		timestamp = t[0]
@@ -96,8 +106,8 @@ func (ac *accumulator) AddFields(
 		measurement = ac.prefix + measurement
 	}
 
-	if ac.plugin != nil {
-		if !ac.plugin.ShouldPass(measurement, tags) {
+	if ac.pluginConfig != nil {
+		if !ac.pluginConfig.ShouldPass(measurement) || !ac.pluginConfig.ShouldTagsPass(tags) {
 			return
 		}
 	}
@@ -111,6 +121,7 @@ func (ac *accumulator) AddFields(
 	pt, err := client.NewPoint(measurement, tags, fields, timestamp)
 	if err != nil {
 		log.Printf("Error adding point [%s]: %s\n", measurement, err.Error())
+		return
 	}
 	if ac.debug {
 		fmt.Println("> " + pt.String())

agent.go

@@ -6,126 +6,67 @@ import (
 	"log"
 	"math/big"
 	"os"
-	"sort"
-	"strings"
 	"sync"
 	"time"
 
-	"github.com/influxdb/telegraf/internal"
+	"github.com/influxdb/telegraf/internal/config"
 	"github.com/influxdb/telegraf/outputs"
 	"github.com/influxdb/telegraf/plugins"
 
 	"github.com/influxdb/influxdb/client/v2"
 )
 
-type runningOutput struct {
-	name   string
-	output outputs.Output
-}
-
-type runningPlugin struct {
-	name   string
-	plugin plugins.Plugin
-	config *ConfiguredPlugin
-}
-
 // Agent runs telegraf and collects data based on the given config
 type Agent struct {
-
-	// Interval at which to gather information
-	Interval internal.Duration
-
-	// RoundInterval rounds collection interval to 'interval'.
-	//     ie, if Interval=10s then always collect on :00, :10, :20, etc.
-	RoundInterval bool
-
-	// Interval at which to flush data
-	FlushInterval internal.Duration
-
-	// FlushRetries is the number of times to retry each data flush
-	FlushRetries int
-
-	// FlushJitter tells
-	FlushJitter internal.Duration
-
-	// TODO(cam): Remove UTC and Precision parameters, they are no longer
-	// valid for the agent config. Leaving them here for now for backwards-
-	// compatability
-
-	// Option for outputting data in UTC
-	UTC bool `toml:"utc"`
-
-	// Precision to write data at
-	// Valid values for Precision are n, u, ms, s, m, and h
-	Precision string
-
-	// Option for running in debug mode
-	Debug    bool
-	Hostname string
-
-	Tags map[string]string
-
-	outputs []*runningOutput
-	plugins []*runningPlugin
+	Config *config.Config
 }
 
 // NewAgent returns an Agent struct based off the given Config
-func NewAgent(config *Config) (*Agent, error) {
-	agent := &Agent{
-		Tags:          make(map[string]string),
-		Interval:      internal.Duration{10 * time.Second},
-		RoundInterval: true,
-		FlushInterval: internal.Duration{10 * time.Second},
-		FlushRetries:  2,
-		FlushJitter:   internal.Duration{5 * time.Second},
+func NewAgent(config *config.Config) (*Agent, error) {
+	a := &Agent{
+		Config: config,
 	}
 
-	// Apply the toml table to the agent config, overriding defaults
-	err := config.ApplyAgent(agent)
-	if err != nil {
-		return nil, err
-	}
-
-	if agent.Hostname == "" {
+	if a.Config.Agent.Hostname == "" {
 		hostname, err := os.Hostname()
 		if err != nil {
 			return nil, err
 		}
 
-		agent.Hostname = hostname
+		a.Config.Agent.Hostname = hostname
 	}
 
-	agent.Tags["host"] = agent.Hostname
+	config.Tags["host"] = a.Config.Agent.Hostname
 
-	return agent, nil
+	return a, nil
 }
 
 // Connect connects to all configured outputs
 func (a *Agent) Connect() error {
-	for _, o := range a.outputs {
-		switch ot := o.output.(type) {
+	for _, o := range a.Config.Outputs {
+		switch ot := o.Output.(type) {
 		case outputs.ServiceOutput:
 			if err := ot.Start(); err != nil {
 				log.Printf("Service for output %s failed to start, exiting\n%s\n",
-					o.name, err.Error())
+					o.Name, err.Error())
 				return err
 			}
 		}
 
-		if a.Debug {
-			log.Printf("Attempting connection to output: %s\n", o.name)
+		if a.Config.Agent.Debug {
+			log.Printf("Attempting connection to output: %s\n", o.Name)
 		}
-		err := o.output.Connect()
+		err := o.Output.Connect()
 		if err != nil {
-			log.Printf("Failed to connect to output %s, retrying in 15s\n", o.name)
+			log.Printf("Failed to connect to output %s, retrying in 15s\n", o.Name)
 			time.Sleep(15 * time.Second)
-			err = o.output.Connect()
+			err = o.Output.Connect()
 			if err != nil {
 				return err
 			}
 		}
-		if a.Debug {
-			log.Printf("Successfully connected to output: %s\n", o.name)
+		if a.Config.Agent.Debug {
+			log.Printf("Successfully connected to output: %s\n", o.Name)
 		}
 	}
 	return nil
@@ -134,9 +75,9 @@ func (a *Agent) Connect() error {
 // Close closes the connection to all configured outputs
 func (a *Agent) Close() error {
 	var err error
-	for _, o := range a.outputs {
-		err = o.output.Close()
-		switch ot := o.output.(type) {
+	for _, o := range a.Config.Outputs {
+		err = o.Output.Close()
+		switch ot := o.Output.(type) {
 		case outputs.ServiceOutput:
 			ot.Stop()
 		}
@@ -144,50 +85,6 @@ func (a *Agent) Close() error {
 	return err
 }
 
-// LoadOutputs loads the agent's outputs
-func (a *Agent) LoadOutputs(filters []string, config *Config) ([]string, error) {
-	var names []string
-
-	for name, output := range config.OutputsDeclared() {
-		// Trim the ID off the output name for filtering
-		filtername := strings.TrimRight(name, "-0123456789")
-		if sliceContains(filtername, filters) || len(filters) == 0 {
-			if a.Debug {
-				log.Println("Output Enabled: ", name)
-			}
-
-			err := config.ApplyOutput(name, output)
-			if err != nil {
-				return nil, err
-			}
-
-			a.outputs = append(a.outputs, &runningOutput{name, output})
-			names = append(names, name)
-		}
-	}
-
-	sort.Strings(names)
-
-	return names, nil
-}
-
-// LoadPlugins loads the agent's plugins
-func (a *Agent) LoadPlugins(filters []string, config *Config) ([]string, error) {
-	var names []string
-
-	for name, plugin := range config.PluginsDeclared() {
-		if sliceContains(name, filters) || len(filters) == 0 {
-			config := config.GetPluginConfig(name)
-			a.plugins = append(a.plugins, &runningPlugin{name, plugin, config})
-			names = append(names, name)
-		}
-	}
-
-	sort.Strings(names)
-
-	return names, nil
-}
-
 // gatherParallel runs the plugins that are using the same reporting interval
 // as the telegraf agent.
 func (a *Agent) gatherParallel(pointChan chan *client.Point) error {
@@ -195,23 +92,23 @@ func (a *Agent) gatherParallel(pointChan chan *client.Point) error {
 
 	start := time.Now()
 	counter := 0
-	for _, plugin := range a.plugins {
-		if plugin.config.Interval != 0 {
+	for _, plugin := range a.Config.Plugins {
+		if plugin.Config.Interval != 0 {
 			continue
 		}
 
 		wg.Add(1)
 		counter++
-		go func(plugin *runningPlugin) {
+		go func(plugin *config.RunningPlugin) {
 			defer wg.Done()
 
-			acc := NewAccumulator(plugin.config, pointChan)
-			acc.SetDebug(a.Debug)
-			acc.SetPrefix(plugin.name + "_")
-			acc.SetDefaultTags(a.Tags)
+			acc := NewAccumulator(plugin.Config, pointChan)
+			acc.SetDebug(a.Config.Agent.Debug)
+			acc.SetPrefix(plugin.Name + "_")
+			acc.SetDefaultTags(a.Config.Tags)
 
-			if err := plugin.plugin.Gather(acc); err != nil {
-				log.Printf("Error in plugin [%s]: %s", plugin.name, err)
+			if err := plugin.Plugin.Gather(acc); err != nil {
+				log.Printf("Error in plugin [%s]: %s", plugin.Name, err)
 			}
 
 		}(plugin)
@@ -221,7 +118,7 @@ func (a *Agent) gatherParallel(pointChan chan *client.Point) error {
 
 	elapsed := time.Since(start)
 	log.Printf("Gathered metrics, (%s interval), from %d plugins in %s\n",
-		a.Interval, counter, elapsed)
+		a.Config.Agent.Interval, counter, elapsed)
 	return nil
 }
 
@@ -229,27 +126,27 @@ func (a *Agent) gatherParallel(pointChan chan *client.Point) error {
 // reporting interval.
 func (a *Agent) gatherSeparate(
 	shutdown chan struct{},
-	plugin *runningPlugin,
+	plugin *config.RunningPlugin,
 	pointChan chan *client.Point,
 ) error {
-	ticker := time.NewTicker(plugin.config.Interval)
+	ticker := time.NewTicker(plugin.Config.Interval)
 
 	for {
 		var outerr error
 		start := time.Now()
 
-		acc := NewAccumulator(plugin.config, pointChan)
-		acc.SetDebug(a.Debug)
-		acc.SetPrefix(plugin.name + "_")
-		acc.SetDefaultTags(a.Tags)
+		acc := NewAccumulator(plugin.Config, pointChan)
+		acc.SetDebug(a.Config.Agent.Debug)
+		acc.SetPrefix(plugin.Name + "_")
+		acc.SetDefaultTags(a.Config.Tags)
 
-		if err := plugin.plugin.Gather(acc); err != nil {
-			log.Printf("Error in plugin [%s]: %s", plugin.name, err)
+		if err := plugin.Plugin.Gather(acc); err != nil {
+			log.Printf("Error in plugin [%s]: %s", plugin.Name, err)
 		}
 
 		elapsed := time.Since(start)
 		log.Printf("Gathered metrics, (separate %s interval), from %s in %s\n",
-			plugin.config.Interval, plugin.name, elapsed)
+			plugin.Config.Interval, plugin.Name, elapsed)
 
 		if outerr != nil {
 			return outerr
@@ -283,27 +180,27 @@ func (a *Agent) Test() error {
 		}
 	}()
 
-	for _, plugin := range a.plugins {
-		acc := NewAccumulator(plugin.config, pointChan)
+	for _, plugin := range a.Config.Plugins {
+		acc := NewAccumulator(plugin.Config, pointChan)
 		acc.SetDebug(true)
-		acc.SetPrefix(plugin.name + "_")
+		acc.SetPrefix(plugin.Name + "_")
 
-		fmt.Printf("* Plugin: %s, Collection 1\n", plugin.name)
-		if plugin.config.Interval != 0 {
-			fmt.Printf("* Internal: %s\n", plugin.config.Interval)
+		fmt.Printf("* Plugin: %s, Collection 1\n", plugin.Name)
+		if plugin.Config.Interval != 0 {
+			fmt.Printf("* Internal: %s\n", plugin.Config.Interval)
 		}
 
-		if err := plugin.plugin.Gather(acc); err != nil {
+		if err := plugin.Plugin.Gather(acc); err != nil {
 			return err
 		}
 
 		// Special instructions for some plugins. cpu, for example, needs to be
 		// run twice in order to return cpu usage percentages.
-		switch plugin.name {
+		switch plugin.Name {
 		case "cpu", "mongodb":
 			time.Sleep(500 * time.Millisecond)
-			fmt.Printf("* Plugin: %s, Collection 2\n", plugin.name)
-			if err := plugin.plugin.Gather(acc); err != nil {
+			fmt.Printf("* Plugin: %s, Collection 2\n", plugin.Name)
+			if err := plugin.Plugin.Gather(acc); err != nil {
 				return err
 			}
 		}
@@ -316,7 +213,7 @@ func (a *Agent) Test() error {
 // Optionally takes a `done` channel to indicate that it is done writing.
 func (a *Agent) writeOutput(
 	points []*client.Point,
-	ro *runningOutput,
+	ro *config.RunningOutput,
 	shutdown chan struct{},
 	wg *sync.WaitGroup,
 ) {
@@ -325,16 +222,16 @@ func (a *Agent) writeOutput(
 		return
 	}
 	retry := 0
-	retries := a.FlushRetries
+	retries := a.Config.Agent.FlushRetries
 	start := time.Now()
 
 	for {
-		err := ro.output.Write(points)
+		err := ro.Output.Write(points)
 		if err == nil {
 			// Write successful
 			elapsed := time.Since(start)
 			log.Printf("Flushed %d metrics to output %s in %s\n",
-				len(points), ro.name, elapsed)
+				len(points), ro.Name, elapsed)
 			return
 		}
 
@@ -346,13 +243,13 @@ func (a *Agent) writeOutput(
 				// No more retries
 				msg := "FATAL: Write to output [%s] failed %d times, dropping" +
 					" %d metrics\n"
-				log.Printf(msg, ro.name, retries+1, len(points))
+				log.Printf(msg, ro.Name, retries+1, len(points))
 				return
 			} else if err != nil {
 				// Sleep for a retry
 				log.Printf("Error in output [%s]: %s, retrying in %s",
-					ro.name, err.Error(), a.FlushInterval.Duration)
-				time.Sleep(a.FlushInterval.Duration)
+					ro.Name, err.Error(), a.Config.Agent.FlushInterval.Duration)
+				time.Sleep(a.Config.Agent.FlushInterval.Duration)
 			}
 		}
 
@@ -367,7 +264,7 @@ func (a *Agent) flush(
 	wait bool,
 ) {
 	var wg sync.WaitGroup
-	for _, o := range a.outputs {
+	for _, o := range a.Config.Outputs {
 		wg.Add(1)
 		go a.writeOutput(points, o, shutdown, &wg)
 	}
@@ -382,7 +279,7 @@ func (a *Agent) flusher(shutdown chan struct{}, pointChan chan *client.Point) er
 	// the flusher will flush after metrics are collected.
 	time.Sleep(time.Millisecond * 100)
 
-	ticker := time.NewTicker(a.FlushInterval.Duration)
+	ticker := time.NewTicker(a.Config.Agent.FlushInterval.Duration)
 	points := make([]*client.Point, 0)
 
 	for {
@@ -425,22 +322,23 @@ func jitterInterval(ininterval, injitter time.Duration) time.Duration {
 func (a *Agent) Run(shutdown chan struct{}) error {
 	var wg sync.WaitGroup
 
-	a.FlushInterval.Duration = jitterInterval(a.FlushInterval.Duration,
-		a.FlushJitter.Duration)
+	a.Config.Agent.FlushInterval.Duration = jitterInterval(a.Config.Agent.FlushInterval.Duration,
+		a.Config.Agent.FlushJitter.Duration)
 
 	log.Printf("Agent Config: Interval:%s, Debug:%#v, Hostname:%#v, "+
 		"Flush Interval:%s\n",
-		a.Interval, a.Debug, a.Hostname, a.FlushInterval)
+		a.Config.Agent.Interval, a.Config.Agent.Debug,
+		a.Config.Agent.Hostname, a.Config.Agent.FlushInterval)
 
 	// channel shared between all plugin threads for accumulating points
 	pointChan := make(chan *client.Point, 1000)
 
 	// Round collection to nearest interval by sleeping
-	if a.RoundInterval {
-		i := int64(a.Interval.Duration)
+	if a.Config.Agent.RoundInterval {
+		i := int64(a.Config.Agent.Interval.Duration)
 		time.Sleep(time.Duration(i - (time.Now().UnixNano() % i)))
 	}
-	ticker := time.NewTicker(a.Interval.Duration)
+	ticker := time.NewTicker(a.Config.Agent.Interval.Duration)
 
 	wg.Add(1)
 	go func() {
@@ -451,14 +349,14 @@ func (a *Agent) Run(shutdown chan struct{}) error {
 		}
 	}()
 
-	for _, plugin := range a.plugins {
+	for _, plugin := range a.Config.Plugins {
 
 		// Start service of any ServicePlugins
-		switch p := plugin.plugin.(type) {
+		switch p := plugin.Plugin.(type) {
 		case plugins.ServicePlugin:
 			if err := p.Start(); err != nil {
 				log.Printf("Service for plugin %s failed to start, exiting\n%s\n",
-					plugin.name, err.Error())
+					plugin.Name, err.Error())
 				return err
 			}
 			defer p.Stop()
@@ -466,9 +364,9 @@ func (a *Agent) Run(shutdown chan struct{}) error {
 
 		// Special handling for plugins that have their own collection interval
 		// configured. Default intervals are handled below with gatherParallel
-		if plugin.config.Interval != 0 {
+		if plugin.Config.Interval != 0 {
 			wg.Add(1)
-			go func(plugin *runningPlugin) {
+			go func(plugin *config.RunningPlugin) {
 				defer wg.Done()
 				if err := a.gatherSeparate(shutdown, plugin, pointChan); err != nil {
 					log.Printf(err.Error())

agent_test.go

@@ -5,7 +5,7 @@ import (
 	"testing"
 	"time"
 
-	"github.com/influxdb/telegraf/internal"
+	"github.com/influxdb/telegraf/internal/config"
 
 	// needing to load the plugins
 	_ "github.com/influxdb/telegraf/plugins/all"
@@ -14,58 +14,78 @@ import (
 )
 
 func TestAgent_LoadPlugin(t *testing.T) {
+	c := config.NewConfig()
+	c.PluginFilters = []string{"mysql"}
+	c.LoadConfig("./internal/config/testdata/telegraf-agent.toml")
+	a, _ := NewAgent(c)
+	assert.Equal(t, 1, len(a.Config.Plugins))
 
-	// load a dedicated configuration file
-	config, _ := LoadConfig("./testdata/telegraf-agent.toml")
-	a, _ := NewAgent(config)
+	c = config.NewConfig()
+	c.PluginFilters = []string{"foo"}
+	c.LoadConfig("./internal/config/testdata/telegraf-agent.toml")
+	a, _ = NewAgent(c)
+	assert.Equal(t, 0, len(a.Config.Plugins))
 
-	pluginsEnabled, _ := a.LoadPlugins([]string{"mysql"}, config)
-	assert.Equal(t, 1, len(pluginsEnabled))
+	c = config.NewConfig()
+	c.PluginFilters = []string{"mysql", "foo"}
+	c.LoadConfig("./internal/config/testdata/telegraf-agent.toml")
+	a, _ = NewAgent(c)
+	assert.Equal(t, 1, len(a.Config.Plugins))
 
-	pluginsEnabled, _ = a.LoadPlugins([]string{"foo"}, config)
-	assert.Equal(t, 0, len(pluginsEnabled))
+	c = config.NewConfig()
+	c.PluginFilters = []string{"mysql", "redis"}
+	c.LoadConfig("./internal/config/testdata/telegraf-agent.toml")
+	a, _ = NewAgent(c)
+	assert.Equal(t, 2, len(a.Config.Plugins))
 
-	pluginsEnabled, _ = a.LoadPlugins([]string{"mysql", "foo"}, config)
-	assert.Equal(t, 1, len(pluginsEnabled))
-
-	pluginsEnabled, _ = a.LoadPlugins([]string{"mysql", "redis"}, config)
-	assert.Equal(t, 2, len(pluginsEnabled))
-
-	pluginsEnabled, _ = a.LoadPlugins([]string{"mysql", "foo", "redis", "bar"}, config)
-	assert.Equal(t, 2, len(pluginsEnabled))
+	c = config.NewConfig()
+	c.PluginFilters = []string{"mysql", "foo", "redis", "bar"}
+	c.LoadConfig("./internal/config/testdata/telegraf-agent.toml")
+	a, _ = NewAgent(c)
+	assert.Equal(t, 2, len(a.Config.Plugins))
 }
 
 func TestAgent_LoadOutput(t *testing.T) {
-	// load a dedicated configuration file
-	config, _ := LoadConfig("./testdata/telegraf-agent.toml")
-	a, _ := NewAgent(config)
+	c := config.NewConfig()
+	c.OutputFilters = []string{"influxdb"}
+	c.LoadConfig("./internal/config/testdata/telegraf-agent.toml")
+	a, _ := NewAgent(c)
+	assert.Equal(t, 2, len(a.Config.Outputs))
 
-	outputsEnabled, _ := a.LoadOutputs([]string{"influxdb"}, config)
-	assert.Equal(t, 2, len(outputsEnabled))
+	c = config.NewConfig()
+	c.OutputFilters = []string{}
+	c.LoadConfig("./internal/config/testdata/telegraf-agent.toml")
+	a, _ = NewAgent(c)
+	assert.Equal(t, 3, len(a.Config.Outputs))
 
-	outputsEnabled, _ = a.LoadOutputs([]string{}, config)
-	assert.Equal(t, 3, len(outputsEnabled))
+	c = config.NewConfig()
+	c.OutputFilters = []string{"foo"}
+	c.LoadConfig("./internal/config/testdata/telegraf-agent.toml")
+	a, _ = NewAgent(c)
+	assert.Equal(t, 0, len(a.Config.Outputs))
 
-	outputsEnabled, _ = a.LoadOutputs([]string{"foo"}, config)
-	assert.Equal(t, 0, len(outputsEnabled))
+	c = config.NewConfig()
+	c.OutputFilters = []string{"influxdb", "foo"}
+	c.LoadConfig("./internal/config/testdata/telegraf-agent.toml")
+	a, _ = NewAgent(c)
+	assert.Equal(t, 2, len(a.Config.Outputs))
 
-	outputsEnabled, _ = a.LoadOutputs([]string{"influxdb", "foo"}, config)
-	assert.Equal(t, 2, len(outputsEnabled))
+	c = config.NewConfig()
+	c.OutputFilters = []string{"influxdb", "kafka"}
+	c.LoadConfig("./internal/config/testdata/telegraf-agent.toml")
+	a, _ = NewAgent(c)
+	assert.Equal(t, 3, len(a.Config.Outputs))
 
-	outputsEnabled, _ = a.LoadOutputs([]string{"influxdb", "kafka"}, config)
-	assert.Equal(t, 3, len(outputsEnabled))
-
-	outputsEnabled, _ = a.LoadOutputs([]string{"influxdb", "foo", "kafka", "bar"}, config)
-	assert.Equal(t, 3, len(outputsEnabled))
+	c = config.NewConfig()
+	c.OutputFilters = []string{"influxdb", "foo", "kafka", "bar"}
+	c.LoadConfig("./internal/config/testdata/telegraf-agent.toml")
+	a, _ = NewAgent(c)
+	assert.Equal(t, 3, len(a.Config.Outputs))
 }
 
 func TestAgent_ZeroJitter(t *testing.T) {
-	a := &Agent{
-		FlushInterval: internal.Duration{10 * time.Second},
-		FlushJitter:   internal.Duration{0 * time.Second},
-	}
-	flushinterval := jitterInterval(a.FlushInterval.Duration,
-		a.FlushJitter.Duration)
+	flushinterval := jitterInterval(time.Duration(10*time.Second),
+		time.Duration(0*time.Second))
 
 	actual := flushinterval.Nanoseconds()
 	exp := time.Duration(10 * time.Second).Nanoseconds()
@@ -80,13 +100,8 @@ func TestAgent_ZeroInterval(t *testing.T) {
 	max := time.Duration(5 * time.Second).Nanoseconds()
 
 	for i := 0; i < 1000; i++ {
-		a := &Agent{
-			FlushInterval: internal.Duration{0 * time.Second},
-			FlushJitter:   internal.Duration{5 * time.Second},
-		}
-
-		flushinterval := jitterInterval(a.FlushInterval.Duration,
-			a.FlushJitter.Duration)
+		flushinterval := jitterInterval(time.Duration(0*time.Second),
+			time.Duration(5*time.Second))
 		actual := flushinterval.Nanoseconds()
 
 		if actual > max {
@@ -101,13 +116,8 @@ func TestAgent_ZeroInterval(t *testing.T) {
 }
 
 func TestAgent_ZeroBoth(t *testing.T) {
-	a := &Agent{
-		FlushInterval: internal.Duration{0 * time.Second},
-		FlushJitter:   internal.Duration{0 * time.Second},
-	}
-
-	flushinterval := jitterInterval(a.FlushInterval.Duration,
-		a.FlushJitter.Duration)
+	flushinterval := jitterInterval(time.Duration(0*time.Second),
+		time.Duration(0*time.Second))
 
 	actual := flushinterval
 	exp := time.Duration(500 * time.Millisecond)
@@ -121,12 +131,8 @@ func TestAgent_JitterMax(t *testing.T) {
 	max := time.Duration(32 * time.Second).Nanoseconds()
 
 	for i := 0; i < 1000; i++ {
-		a := &Agent{
-			FlushInterval: internal.Duration{30 * time.Second},
-			FlushJitter:   internal.Duration{2 * time.Second},
-		}
-		flushinterval := jitterInterval(a.FlushInterval.Duration,
-			a.FlushJitter.Duration)
+		flushinterval := jitterInterval(time.Duration(30*time.Second),
+			time.Duration(2*time.Second))
 		actual := flushinterval.Nanoseconds()
 		if actual > max {
 			t.Errorf("Didn't expect interval %d to be > %d", actual, max)
@@ -139,12 +145,8 @@ func TestAgent_JitterMin(t *testing.T) {
 	min := time.Duration(30 * time.Second).Nanoseconds()
 
 	for i := 0; i < 1000; i++ {
-		a := &Agent{
-			FlushInterval: internal.Duration{30 * time.Second},
-			FlushJitter:   internal.Duration{2 * time.Second},
-		}
-		flushinterval := jitterInterval(a.FlushInterval.Duration,
-			a.FlushJitter.Duration)
+		flushinterval := jitterInterval(time.Duration(30*time.Second),
+			time.Duration(2*time.Second))
 		actual := flushinterval.Nanoseconds()
 		if actual < min {
 			t.Errorf("Didn't expect interval %d to be < %d", actual, min)

cmd/telegraf/telegraf.go

@@ -9,6 +9,7 @@ import (
 	"strings"
 
 	"github.com/influxdb/telegraf"
+	"github.com/influxdb/telegraf/internal/config"
 	_ "github.com/influxdb/telegraf/outputs/all"
 	_ "github.com/influxdb/telegraf/plugins/all"
 )
@@ -18,7 +19,7 @@ var fDebug = flag.Bool("debug", false,
 var fTest = flag.Bool("test", false, "gather metrics, print them out, and exit")
 var fConfig = flag.String("config", "", "configuration file to load")
 var fConfigDirectory = flag.String("configdirectory", "",
-	"directory containing additional configuration files")
+	"directory containing additional *.conf files")
 var fVersion = flag.Bool("version", false, "display the version")
 var fSampleConfig = flag.Bool("sample-config", false,
 	"print out full sample configuration")
@@ -56,13 +57,13 @@ func main() {
 	}
 
 	if *fSampleConfig {
-		telegraf.PrintSampleConfig(pluginFilters, outputFilters)
+		config.PrintSampleConfig(pluginFilters, outputFilters)
 		return
 	}
 
 	if *fUsage != "" {
-		if err := telegraf.PrintPluginConfig(*fUsage); err != nil {
-			if err2 := telegraf.PrintOutputConfig(*fUsage); err2 != nil {
+		if err := config.PrintPluginConfig(*fUsage); err != nil {
+			if err2 := config.PrintOutputConfig(*fUsage); err2 != nil {
 				log.Fatalf("%s and %s", err, err2)
 			}
 		}
@@ -70,12 +71,15 @@ func main() {
 	}
 
 	var (
-		config *telegraf.Config
-		err    error
+		c   *config.Config
+		err error
 	)
 
 	if *fConfig != "" {
-		config, err = telegraf.LoadConfig(*fConfig)
+		c = config.NewConfig()
+		c.OutputFilters = outputFilters
+		c.PluginFilters = pluginFilters
+		err = c.LoadConfig(*fConfig)
 		if err != nil {
 			log.Fatal(err)
 		}
@@ -86,37 +90,25 @@ func main() {
 	}
 
 	if *fConfigDirectory != "" {
-		err = config.LoadDirectory(*fConfigDirectory)
+		err = c.LoadDirectory(*fConfigDirectory)
 		if err != nil {
 			log.Fatal(err)
 		}
 	}
+	if len(c.Outputs) == 0 {
+		log.Fatalf("Error: no outputs found, did you provide a valid config file?")
+	}
+	if len(c.Plugins) == 0 {
+		log.Fatalf("Error: no plugins found, did you provide a valid config file?")
+	}
 
-	ag, err := telegraf.NewAgent(config)
+	ag, err := telegraf.NewAgent(c)
 	if err != nil {
 		log.Fatal(err)
 	}
 
 	if *fDebug {
-		ag.Debug = true
-	}
-
-	outputs, err := ag.LoadOutputs(outputFilters, config)
-	if err != nil {
-		log.Fatal(err)
-	}
-	if len(outputs) == 0 {
-		log.Printf("Error: no outputs found, did you provide a valid config file?")
-		os.Exit(1)
-	}
-
-	plugins, err := ag.LoadPlugins(pluginFilters, config)
-	if err != nil {
-		log.Fatal(err)
-	}
-	if len(plugins) == 0 {
-		log.Printf("Error: no plugins found, did you provide a valid config file?")
-		os.Exit(1)
+		ag.Config.Agent.Debug = true
 	}
 
 	if *fTest {
@@ -141,9 +133,9 @@ func main() {
 	}()
 
 	log.Printf("Starting Telegraf (version %s)\n", Version)
-	log.Printf("Loaded outputs: %s", strings.Join(outputs, " "))
-	log.Printf("Loaded plugins: %s", strings.Join(plugins, " "))
-	log.Printf("Tags enabled: %s", config.ListTags())
+	log.Printf("Loaded outputs: %s", strings.Join(c.OutputNames(), " "))
+	log.Printf("Loaded plugins: %s", strings.Join(c.PluginNames(), " "))
+	log.Printf("Tags enabled: %s", c.ListTags())
 
 	if *fPidfile != "" {
 		f, err := os.Create(*fPidfile)

config.go

@@ -1,694 +0,0 @@
-package telegraf
-
-import (
-	"errors"
-	"fmt"
-	"io/ioutil"
-	"log"
-	"path/filepath"
-	"reflect"
-	"sort"
-	"strings"
-	"time"
-
-	"github.com/influxdb/telegraf/outputs"
-	"github.com/influxdb/telegraf/plugins"
-	"github.com/naoina/toml"
-	"github.com/naoina/toml/ast"
-)
-
-// Config specifies the URL/user/password for the database that telegraf
-// will be logging to, as well as all the plugins that the user has
-// specified
-type Config struct {
-	// This lives outside the agent because mergeStruct doesn't need to handle
-	// maps normally. We just copy the elements manually in ApplyAgent.
-	Tags map[string]string
-
-	agent                *Agent
-	plugins              map[string]plugins.Plugin
-	pluginConfigurations map[string]*ConfiguredPlugin
-	outputs              map[string]outputs.Output
-
-	agentFieldsSet               []string
-	pluginFieldsSet              map[string][]string
-	pluginConfigurationFieldsSet map[string][]string
-	outputFieldsSet              map[string][]string
-}
-
-// Plugins returns the configured plugins as a map of name -> plugins.Plugin
-func (c *Config) Plugins() map[string]plugins.Plugin {
-	return c.plugins
-}
-
-// Outputs returns the configured outputs as a map of name -> outputs.Output
-func (c *Config) Outputs() map[string]outputs.Output {
-	return c.outputs
-}
-
-// TagFilter is the name of a tag, and the values on which to filter
-type TagFilter struct {
-	Name   string
-	Filter []string
-}
-
-// ConfiguredPlugin containing a name, interval, and drop/pass prefix lists
-// Also lists the tags to filter
-type ConfiguredPlugin struct {
-	Name string
-
-	Drop []string
-	Pass []string
-
-	TagDrop []TagFilter
-	TagPass []TagFilter
-
-	Interval time.Duration
-}
-
-// ShouldPass returns true if the metric should pass, false if should drop
-func (cp *ConfiguredPlugin) ShouldPass(measurement string, tags map[string]string) bool {
-	if cp.Pass != nil {
-		for _, pat := range cp.Pass {
-			if strings.HasPrefix(measurement, pat) {
-				return true
-			}
-		}
-
-		return false
-	}
-
-	if cp.Drop != nil {
-		for _, pat := range cp.Drop {
-			if strings.HasPrefix(measurement, pat) {
-				return false
-			}
-		}
-
-		return true
-	}
-
-	if cp.TagPass != nil {
-		for _, pat := range cp.TagPass {
-			if tagval, ok := tags[pat.Name]; ok {
-				for _, filter := range pat.Filter {
-					if filter == tagval {
-						return true
-					}
-				}
-			}
-		}
-		return false
-	}
-
-	if cp.TagDrop != nil {
-		for _, pat := range cp.TagDrop {
-			if tagval, ok := tags[pat.Name]; ok {
-				for _, filter := range pat.Filter {
-					if filter == tagval {
-						return false
-					}
-				}
-			}
-		}
-		return true
-	}
-
-	return true
-}
-
-// ApplyOutput loads the Output struct built from the config into the given Output struct.
-// Overrides only values in the given struct that were set in the config.
-func (c *Config) ApplyOutput(name string, v interface{}) error {
-	if c.outputs[name] != nil {
-		return mergeStruct(v, c.outputs[name], c.outputFieldsSet[name])
-	}
-	return nil
-}
-
-// ApplyAgent loads the Agent struct built from the config into the given Agent struct.
-// Overrides only values in the given struct that were set in the config.
-func (c *Config) ApplyAgent(a *Agent) error {
-	if c.agent != nil {
-		for key, value := range c.Tags {
-			a.Tags[key] = value
-		}
-		return mergeStruct(a, c.agent, c.agentFieldsSet)
-	}
-
-	return nil
-}
-
-func (c *Config) GetPluginConfig(name string) *ConfiguredPlugin {
-	return c.pluginConfigurations[name]
-}
-
-// Couldn't figure out how to get this to work with the declared function.
-
-// PluginsDeclared returns the name of all plugins declared in the config.
-func (c *Config) PluginsDeclared() map[string]plugins.Plugin {
-	return c.plugins
-}
-
-// OutputsDeclared returns the name of all outputs declared in the config.
-func (c *Config) OutputsDeclared() map[string]outputs.Output {
-	return c.outputs
-}
-
-// ListTags returns a string of tags specified in the config,
-// line-protocol style
-func (c *Config) ListTags() string {
-	var tags []string
-
-	for k, v := range c.Tags {
-		tags = append(tags, fmt.Sprintf("%s=%s", k, v))
-	}
-
-	sort.Strings(tags)
-
-	return strings.Join(tags, " ")
-}
-
-type hasConfig interface {
-	BasicConfig() string
-}
-
-type hasDescr interface {
-	Description() string
-}
-
-var header = `# Telegraf configuration
-
-# Telegraf is entirely plugin driven. All metrics are gathered from the
-# declared plugins.
-
-# Even if a plugin has no configuration, it must be declared in here
-# to be active. Declaring a plugin means just specifying the name
-# as a section with no variables. To deactivate a plugin, comment
-# out the name and any variables.
-
-# Use 'telegraf -config telegraf.toml -test' to see what metrics a config
-# file would generate.
-
-# One rule that plugins conform to is wherever a connection string
-# can be passed, the values '' and 'localhost' are treated specially.
-# They indicate to the plugin to use their own builtin configuration to
-# connect to the local system.
-
-# NOTE: The configuration has a few required parameters. They are marked
-# with 'required'. Be sure to edit those to make this configuration work.
-
-# Tags can also be specified via a normal map, but only one form at a time:
-[tags]
-  # dc = "us-east-1"
-
-# Configuration for telegraf agent
-[agent]
-  # Default data collection interval for all plugins
-  interval = "10s"
-  # Rounds collection interval to 'interval'
-  # ie, if interval="10s" then always collect on :00, :10, :20, etc.
-  round_interval = true
-
-  # Default data flushing interval for all outputs. You should not set this below
-  # interval. Maximum flush_interval will be flush_interval + flush_jitter
-  flush_interval = "10s"
-  # Jitter the flush interval by a random amount. This is primarily to avoid
-  # large write spikes for users running a large number of telegraf instances.
-  # ie, a jitter of 5s and interval 10s means flushes will happen every 10-15s
-  flush_jitter = "0s"
-
-  # Run telegraf in debug mode
-  debug = false
-  # Override default hostname, if empty use os.Hostname()
-  hostname = ""
-
-
-###############################################################################
-#                                  OUTPUTS                                    #
-###############################################################################
-
-[outputs]
-`
-
-var pluginHeader = `
-
-###############################################################################
-#                                  PLUGINS                                    #
-###############################################################################
-`
-
-var servicePluginHeader = `
-
-###############################################################################
-#                              SERVICE PLUGINS                                #
-###############################################################################
-`
-
-// PrintSampleConfig prints the sample config
-func PrintSampleConfig(pluginFilters []string, outputFilters []string) {
-	fmt.Printf(header)
-
-	// Filter outputs
-	var onames []string
-	for oname := range outputs.Outputs {
-		if len(outputFilters) == 0 || sliceContains(oname, outputFilters) {
-			onames = append(onames, oname)
-		}
-	}
-	sort.Strings(onames)
-
-	// Print Outputs
-	for _, oname := range onames {
-		creator := outputs.Outputs[oname]
-		output := creator()
-
-		fmt.Printf("\n# %s\n[[outputs.%s]]", output.Description(), oname)
-
-		config := output.SampleConfig()
-		if config == "" {
-			fmt.Printf("\n  # no configuration\n")
-		} else {
-			fmt.Printf(config)
-		}
-	}
-
-	// Filter plugins
-	var pnames []string
-	for pname := range plugins.Plugins {
-		if len(pluginFilters) == 0 || sliceContains(pname, pluginFilters) {
-			pnames = append(pnames, pname)
-		}
-	}
-	sort.Strings(pnames)
-
-	// Print Plugins
-	fmt.Printf(pluginHeader)
-	servPlugins := make(map[string]plugins.ServicePlugin)
-	for _, pname := range pnames {
-		creator := plugins.Plugins[pname]
-		plugin := creator()
-
-		switch p := plugin.(type) {
-		case plugins.ServicePlugin:
-			servPlugins[pname] = p
-			continue
-		}
-
-		printConfig(pname, plugin)
-	}
-
-	// Print Service Plugins
-	fmt.Printf(servicePluginHeader)
-	for name, plugin := range servPlugins {
-		printConfig(name, plugin)
-	}
-}
-
-type printer interface {
-	Description() string
-	SampleConfig() string
-}
-
-func printConfig(name string, p printer) {
-	fmt.Printf("\n# %s\n[%s]", p.Description(), name)
-	config := p.SampleConfig()
-	if config == "" {
-		fmt.Printf("\n  # no configuration\n")
-	} else {
-		fmt.Printf(config)
-	}
-}
-
-func sliceContains(name string, list []string) bool {
-	for _, b := range list {
-		if b == name {
-			return true
-		}
-	}
-	return false
-}
-
-// PrintPluginConfig prints the config usage of a single plugin.
-func PrintPluginConfig(name string) error {
-	if creator, ok := plugins.Plugins[name]; ok {
-		printConfig(name, creator())
-	} else {
-		return errors.New(fmt.Sprintf("Plugin %s not found", name))
-	}
-	return nil
-}
-
-// PrintOutputConfig prints the config usage of a single output.
-func PrintOutputConfig(name string) error {
-	if creator, ok := outputs.Outputs[name]; ok {
-		printConfig(name, creator())
-	} else {
-		return errors.New(fmt.Sprintf("Output %s not found", name))
-	}
-	return nil
-}
-
-// Find the field with a name matching fieldName, respecting the struct tag and ignoring case and underscores.
-// If no field is found, return the zero reflect.Value, which should be checked for with .IsValid().
-func findField(fieldName string, value reflect.Value) reflect.Value {
-	r := strings.NewReplacer("_", "")
-	vType := value.Type()
-	for i := 0; i < vType.NumField(); i++ {
-		fieldType := vType.Field(i)
-
-		// if we have toml tag, use it
-		if tag := fieldType.Tag.Get("toml"); tag != "" {
-			if tag == "-" { // omit
-				continue
-			}
-			if tag == fieldName {
-				return value.Field(i)
-			}
-		} else {
-			if strings.ToLower(fieldType.Name) == strings.ToLower(r.Replace(fieldName)) {
-				return value.Field(i)
-			}
-		}
-	}
-	return reflect.Value{}
-}
-
-// A very limited merge. Merges the fields named in the fields parameter,
-// replacing most values, but appending to arrays.
-func mergeStruct(base, overlay interface{}, fields []string) error {
-	baseValue := reflect.ValueOf(base).Elem()
-	overlayValue := reflect.ValueOf(overlay).Elem()
-	if baseValue.Kind() != reflect.Struct {
-		return fmt.Errorf("Tried to merge something that wasn't a struct: type %v was %v",
-			baseValue.Type(), baseValue.Kind())
-	}
-	if baseValue.Type() != overlayValue.Type() {
-		return fmt.Errorf("Tried to merge two different types: %v and %v",
-			baseValue.Type(), overlayValue.Type())
-	}
-	for _, field := range fields {
-		overlayFieldValue := findField(field, overlayValue)
-		if !overlayFieldValue.IsValid() {
-			return fmt.Errorf("could not find field in %v matching %v",
-				overlayValue.Type(), field)
-		}
-		baseFieldValue := findField(field, baseValue)
-		baseFieldValue.Set(overlayFieldValue)
-	}
-	return nil
-}
-
-func (c *Config) LoadDirectory(path string) error {
-	directoryEntries, err := ioutil.ReadDir(path)
-	if err != nil {
-		return err
-	}
-	for _, entry := range directoryEntries {
-		if entry.IsDir() {
-			continue
-		}
-		name := entry.Name()
-		if name[len(name)-5:] != ".conf" {
-			continue
-		}
-		subConfig, err := LoadConfig(filepath.Join(path, name))
-		if err != nil {
-			return err
-		}
-		if subConfig.agent != nil {
-			err = mergeStruct(c.agent, subConfig.agent, subConfig.agentFieldsSet)
-			if err != nil {
-				return err
-			}
-			for _, field := range subConfig.agentFieldsSet {
-				if !sliceContains(field, c.agentFieldsSet) {
-					c.agentFieldsSet = append(c.agentFieldsSet, field)
-				}
-			}
-		}
-		for pluginName, plugin := range subConfig.plugins {
-			if _, ok := c.plugins[pluginName]; !ok {
-				c.plugins[pluginName] = plugin
-				c.pluginFieldsSet[pluginName] = subConfig.pluginFieldsSet[pluginName]
-				c.pluginConfigurations[pluginName] = subConfig.pluginConfigurations[pluginName]
-				c.pluginConfigurationFieldsSet[pluginName] = subConfig.pluginConfigurationFieldsSet[pluginName]
-				continue
-			}
-			err = mergeStruct(c.plugins[pluginName], plugin, subConfig.pluginFieldsSet[pluginName])
-			if err != nil {
-				return err
-			}
-			for _, field := range subConfig.pluginFieldsSet[pluginName] {
-				if !sliceContains(field, c.pluginFieldsSet[pluginName]) {
-					c.pluginFieldsSet[pluginName] = append(c.pluginFieldsSet[pluginName], field)
-				}
-			}
-			err = mergeStruct(c.pluginConfigurations[pluginName], subConfig.pluginConfigurations[pluginName], subConfig.pluginConfigurationFieldsSet[pluginName])
-			if err != nil {
-				return err
-			}
-			for _, field := range subConfig.pluginConfigurationFieldsSet[pluginName] {
-				if !sliceContains(field, c.pluginConfigurationFieldsSet[pluginName]) {
-					c.pluginConfigurationFieldsSet[pluginName] = append(c.pluginConfigurationFieldsSet[pluginName], field)
-				}
-			}
-		}
-		for outputName, output := range subConfig.outputs {
-			if _, ok := c.outputs[outputName]; !ok {
-				c.outputs[outputName] = output
-				c.outputFieldsSet[outputName] = subConfig.outputFieldsSet[outputName]
-				continue
-			}
-			err = mergeStruct(c.outputs[outputName], output, subConfig.outputFieldsSet[outputName])
-			if err != nil {
-				return err
-			}
-			for _, field := range subConfig.outputFieldsSet[outputName] {
-				if !sliceContains(field, c.outputFieldsSet[outputName]) {
-					c.outputFieldsSet[outputName] = append(c.outputFieldsSet[outputName], field)
-				}
-			}
-		}
-	}
-	return nil
-}
-
-// hazmat area. Keeping the ast parsing here.
-
-// LoadConfig loads the given config file and returns a *Config pointer
-func LoadConfig(path string) (*Config, error) {
-	data, err := ioutil.ReadFile(path)
-	if err != nil {
-		return nil, err
-	}
-
-	tbl, err := toml.Parse(data)
-	if err != nil {
-		return nil, err
-	}
-
-	c := &Config{
-		Tags:                         make(map[string]string),
-		plugins:                      make(map[string]plugins.Plugin),
-		pluginConfigurations:         make(map[string]*ConfiguredPlugin),
-		outputs:                      make(map[string]outputs.Output),
-		pluginFieldsSet:              make(map[string][]string),
-		pluginConfigurationFieldsSet: make(map[string][]string),
-		outputFieldsSet:              make(map[string][]string),
-	}
-
-	for name, val := range tbl.Fields {
-		subTable, ok := val.(*ast.Table)
-		if !ok {
-			return nil, errors.New("invalid configuration")
-		}
-
-		switch name {
-		case "agent":
-			err := c.parseAgent(subTable)
-			if err != nil {
-				log.Printf("Could not parse [agent] config\n")
-				return nil, err
-			}
-		case "tags":
-			if err = toml.UnmarshalTable(subTable, c.Tags); err != nil {
-				log.Printf("Could not parse [tags] config\n")
-				return nil, err
-			}
-		case "outputs":
-			for outputName, outputVal := range subTable.Fields {
-				switch outputSubTable := outputVal.(type) {
-				case *ast.Table:
-					err = c.parseOutput(outputName, outputSubTable, 0)
-					if err != nil {
-						log.Printf("Could not parse config for output: %s\n",
-							outputName)
-						return nil, err
-					}
-				case []*ast.Table:
-					for id, t := range outputSubTable {
-						err = c.parseOutput(outputName, t, id)
-						if err != nil {
-							log.Printf("Could not parse config for output: %s\n",
-								outputName)
-							return nil, err
-						}
-					}
-				default:
-					return nil, fmt.Errorf("Unsupported config format: %s",
-						outputName)
-				}
-			}
-		default:
-			err = c.parsePlugin(name, subTable)
-			if err != nil {
-				return nil, err
-			}
-		}
-	}
-
-	return c, nil
-}
-
-// Needs to have the field names, for merging later.
-func extractFieldNames(ast *ast.Table) []string {
-	// A reasonable capacity?
-	var names []string
-	for name := range ast.Fields {
-		names = append(names, name)
-	}
-	return names
-}
-
-// Parse the agent config out of the given *ast.Table.
-func (c *Config) parseAgent(agentAst *ast.Table) error {
-	c.agentFieldsSet = extractFieldNames(agentAst)
-	agent := &Agent{}
-	err := toml.UnmarshalTable(agentAst, agent)
-	if err != nil {
-		return err
-	}
-	c.agent = agent
-	return nil
-}
-
-// Parse an output config out of the given *ast.Table.
-func (c *Config) parseOutput(name string, outputAst *ast.Table, id int) error {
-	c.outputFieldsSet[name] = extractFieldNames(outputAst)
-	creator, ok := outputs.Outputs[name]
-	if !ok {
-		return fmt.Errorf("Undefined but requested output: %s", name)
-	}
-	output := creator()
-	err := toml.UnmarshalTable(outputAst, output)
-	if err != nil {
-		return err
-	}
-	c.outputs[fmt.Sprintf("%s-%d", name, id)] = output
-	return nil
-}
-
-// Parse a plugin config, plus plugin meta-config, out of the given *ast.Table.
-func (c *Config) parsePlugin(name string, pluginAst *ast.Table) error {
-	creator, ok := plugins.Plugins[name]
-	if !ok {
-		return fmt.Errorf("Undefined but requested plugin: %s", name)
-	}
-	plugin := creator()
-	cp := &ConfiguredPlugin{Name: name}
-	cpFields := make([]string, 0, 5)
-
-	if node, ok := pluginAst.Fields["pass"]; ok {
-		if kv, ok := node.(*ast.KeyValue); ok {
-			if ary, ok := kv.Value.(*ast.Array); ok {
-				for _, elem := range ary.Value {
-					if str, ok := elem.(*ast.String); ok {
-						cp.Pass = append(cp.Pass, str.Value)
-					}
-				}
-				cpFields = append(cpFields, "pass")
-			}
-		}
-	}
-
-	if node, ok := pluginAst.Fields["drop"]; ok {
-		if kv, ok := node.(*ast.KeyValue); ok {
-			if ary, ok := kv.Value.(*ast.Array); ok {
-				for _, elem := range ary.Value {
-					if str, ok := elem.(*ast.String); ok {
-						cp.Drop = append(cp.Drop, str.Value)
-					}
-				}
-				cpFields = append(cpFields, "drop")
-			}
-		}
-	}
-
-	if node, ok := pluginAst.Fields["interval"]; ok {
-		if kv, ok := node.(*ast.KeyValue); ok {
-			if str, ok := kv.Value.(*ast.String); ok {
-				dur, err := time.ParseDuration(str.Value)
-				if err != nil {
-					return err
-				}
-
-				cp.Interval = dur
-				cpFields = append(cpFields, "interval")
-			}
-		}
-	}
-
-	if node, ok := pluginAst.Fields["tagpass"]; ok {
-		if subtbl, ok := node.(*ast.Table); ok {
-			for name, val := range subtbl.Fields {
-				if kv, ok := val.(*ast.KeyValue); ok {
-					tagfilter := &TagFilter{Name: name}
-					if ary, ok := kv.Value.(*ast.Array); ok {
-						for _, elem := range ary.Value {
-							if str, ok := elem.(*ast.String); ok {
-								tagfilter.Filter = append(tagfilter.Filter, str.Value)
-							}
-						}
-					}
-					cp.TagPass = append(cp.TagPass, *tagfilter)
-					cpFields = append(cpFields, "tagpass")
-				}
-			}
-		}
-	}
-
-	if node, ok := pluginAst.Fields["tagdrop"]; ok {
-		if subtbl, ok := node.(*ast.Table); ok {
-			for name, val := range subtbl.Fields {
-				if kv, ok := val.(*ast.KeyValue); ok {
-					tagfilter := &TagFilter{Name: name}
-					if ary, ok := kv.Value.(*ast.Array); ok {
-						for _, elem := range ary.Value {
-							if str, ok := elem.(*ast.String); ok {
-								tagfilter.Filter = append(tagfilter.Filter, str.Value)
-							}
-						}
-					}
-					cp.TagDrop = append(cp.TagDrop, *tagfilter)
-					cpFields = append(cpFields, "tagdrop")
-				}
-			}
-		}
-	}
-
-	delete(pluginAst.Fields, "drop")
-	delete(pluginAst.Fields, "pass")
-	delete(pluginAst.Fields, "interval")
-	delete(pluginAst.Fields, "tagdrop")
-	delete(pluginAst.Fields, "tagpass")
-	c.pluginFieldsSet[name] = extractFieldNames(pluginAst)
-	c.pluginConfigurationFieldsSet[name] = cpFields
-	err := toml.UnmarshalTable(pluginAst, plugin)
-	if err != nil {
-		return err
-	}
-	c.plugins[name] = plugin
-	c.pluginConfigurations[name] = cp
-	return nil
-}

config_test.go

@@ -1,313 +0,0 @@
-package telegraf
-
-import (
-	"fmt"
-	"io/ioutil"
-	"testing"
-	"time"
-
-	"github.com/influxdb/telegraf/plugins"
-	"github.com/influxdb/telegraf/plugins/exec"
-	"github.com/influxdb/telegraf/plugins/kafka_consumer"
-	"github.com/influxdb/telegraf/plugins/procstat"
-	"github.com/naoina/toml"
-	"github.com/naoina/toml/ast"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/suite"
-)
-
-type subTest struct {
-	AField       string
-	AnotherField int
-}
-type test struct {
-	StringField     string
-	IntegerField    int
-	FloatField      float32
-	BooleansField   bool `toml:"boolean_field"`
-	DatetimeField   time.Time
-	ArrayField      []string
-	TableArrayField []subTest
-}
-
-type MergeStructSuite struct {
-	suite.Suite
-	EmptyStruct       *test
-	FullStruct        *test
-	AnotherFullStruct *test
-	AllFields         []string
-}
-
-func (s *MergeStructSuite) SetupSuite() {
-	s.AllFields = []string{"string_field", "integer_field", "float_field",
-		"boolean_field", "date_time_field", "array_field", "table_array_field"}
-}
-
-func (s *MergeStructSuite) SetupTest() {
-	s.EmptyStruct = &test{
-		ArrayField:      []string{},
-		TableArrayField: []subTest{},
-	}
-	s.FullStruct = &test{
-		StringField:   "one",
-		IntegerField:  1,
-		FloatField:    1.1,
-		BooleansField: false,
-		DatetimeField: time.Date(1963, time.August, 28, 17, 0, 0, 0, time.UTC),
-		ArrayField:    []string{"one", "two", "three"},
-		TableArrayField: []subTest{
-			subTest{
-				AField:       "one",
-				AnotherField: 1,
-			},
-			subTest{
-				AField:       "two",
-				AnotherField: 2,
-			},
-		},
-	}
-	s.AnotherFullStruct = &test{
-		StringField:   "two",
-		IntegerField:  2,
-		FloatField:    2.2,
-		BooleansField: true,
-		DatetimeField: time.Date(1965, time.March, 25, 17, 0, 0, 0, time.UTC),
-		ArrayField:    []string{"four", "five", "six"},
-		TableArrayField: []subTest{
-			subTest{
-				AField:       "three",
-				AnotherField: 3,
-			},
-			subTest{
-				AField:       "four",
-				AnotherField: 4,
-			},
-		},
-	}
-}
-
-func (s *MergeStructSuite) TestEmptyMerge() {
-	err := mergeStruct(s.EmptyStruct, s.FullStruct, s.AllFields)
-	if err != nil {
-		s.T().Error(err)
-	}
-	s.Equal(s.FullStruct, s.EmptyStruct,
-		fmt.Sprintf("Full merge of %v onto an empty struct failed.", s.FullStruct))
-}
-
-func (s *MergeStructSuite) TestFullMerge() {
-	result := &test{
-		StringField:   "two",
-		IntegerField:  2,
-		FloatField:    2.2,
-		BooleansField: true,
-		DatetimeField: time.Date(1965, time.March, 25, 17, 0, 0, 0, time.UTC),
-		ArrayField:    []string{"four", "five", "six"},
-		TableArrayField: []subTest{
-			subTest{
-				AField:       "three",
-				AnotherField: 3,
-			},
-			subTest{
-				AField:       "four",
-				AnotherField: 4,
-			},
-		},
-	}
-
-	err := mergeStruct(s.FullStruct, s.AnotherFullStruct, s.AllFields)
-	if err != nil {
-		s.T().Error(err)
-	}
-	s.Equal(result, s.FullStruct,
-		fmt.Sprintf("Full merge of %v onto FullStruct failed.", s.AnotherFullStruct))
-}
-
-func (s *MergeStructSuite) TestPartialMergeWithoutSlices() {
-	result := &test{
-		StringField:   "two",
-		IntegerField:  1,
-		FloatField:    2.2,
-		BooleansField: false,
-		DatetimeField: time.Date(1965, time.March, 25, 17, 0, 0, 0, time.UTC),
-		ArrayField:    []string{"one", "two", "three"},
-		TableArrayField: []subTest{
-			subTest{
-				AField:       "one",
-				AnotherField: 1,
-			},
-			subTest{
-				AField:       "two",
-				AnotherField: 2,
-			},
-		},
-	}
-
-	err := mergeStruct(s.FullStruct, s.AnotherFullStruct,
-		[]string{"string_field", "float_field", "date_time_field"})
-	if err != nil {
-		s.T().Error(err)
-	}
-	s.Equal(result, s.FullStruct,
-		fmt.Sprintf("Partial merge without slices of %v onto FullStruct failed.",
-			s.AnotherFullStruct))
-}
-
-func (s *MergeStructSuite) TestPartialMergeWithSlices() {
-	result := &test{
-		StringField:   "two",
-		IntegerField:  1,
-		FloatField:    2.2,
-		BooleansField: false,
-		DatetimeField: time.Date(1965, time.March, 25, 17, 0, 0, 0, time.UTC),
-		ArrayField:    []string{"one", "two", "three"},
-		TableArrayField: []subTest{
-			subTest{
-				AField:       "three",
-				AnotherField: 3,
-			},
-			subTest{
-				AField:       "four",
-				AnotherField: 4,
-			},
-		},
-	}
-
-	err := mergeStruct(s.FullStruct, s.AnotherFullStruct,
-		[]string{"string_field", "float_field", "date_time_field", "table_array_field"})
-	if err != nil {
-		s.T().Error(err)
-	}
-	s.Equal(result, s.FullStruct,
-		fmt.Sprintf("Partial merge with slices of %v onto FullStruct failed.",
-			s.AnotherFullStruct))
-}
-
-func TestConfig_mergeStruct(t *testing.T) {
-	suite.Run(t, new(MergeStructSuite))
-}
-
-func TestConfig_parsePlugin(t *testing.T) {
-	data, err := ioutil.ReadFile("./testdata/single_plugin.toml")
-	if err != nil {
-		t.Error(err)
-	}
-
-	tbl, err := toml.Parse(data)
-	if err != nil {
-		t.Error(err)
-	}
-
-	c := &Config{
-		plugins:                      make(map[string]plugins.Plugin),
-		pluginConfigurations:         make(map[string]*ConfiguredPlugin),
-		pluginFieldsSet:              make(map[string][]string),
-		pluginConfigurationFieldsSet: make(map[string][]string),
-	}
-
-	subtbl := tbl.Fields["kafka"].(*ast.Table)
-	err = c.parsePlugin("kafka", subtbl)
-
-	kafka := plugins.Plugins["kafka"]().(*kafka_consumer.Kafka)
-	kafka.ConsumerGroupName = "telegraf_metrics_consumers"
-	kafka.Topic = "topic_with_metrics"
-	kafka.ZookeeperPeers = []string{"test.example.com:2181"}
-	kafka.BatchSize = 1000
-
-	kConfig := &ConfiguredPlugin{
-		Name: "kafka",
-		Drop: []string{"other", "stuff"},
-		Pass: []string{"some", "strings"},
-		TagDrop: []TagFilter{
-			TagFilter{
-				Name:   "badtag",
-				Filter: []string{"othertag"},
-			},
-		},
-		TagPass: []TagFilter{
-			TagFilter{
-				Name:   "goodtag",
-				Filter: []string{"mytag"},
-			},
-		},
-		Interval: 5 * time.Second,
-	}
-
-	assert.Equal(t, kafka, c.plugins["kafka"],
-		"Testdata did not produce a correct kafka struct.")
-	assert.Equal(t, kConfig, c.pluginConfigurations["kafka"],
-		"Testdata did not produce correct kafka metadata.")
-}
-
-func TestConfig_LoadDirectory(t *testing.T) {
-	c, err := LoadConfig("./testdata/telegraf-agent.toml")
-	if err != nil {
-		t.Error(err)
-	}
-	err = c.LoadDirectory("./testdata/subconfig")
-	if err != nil {
-		t.Error(err)
-	}
-
-	kafka := plugins.Plugins["kafka"]().(*kafka_consumer.Kafka)
-	kafka.ConsumerGroupName = "telegraf_metrics_consumers"
-	kafka.Topic = "topic_with_metrics"
-	kafka.ZookeeperPeers = []string{"test.example.com:2181"}
-	kafka.BatchSize = 10000
-
-	kConfig := &ConfiguredPlugin{
-		Name: "kafka",
-		Drop: []string{"other", "stuff"},
-		Pass: []string{"some", "strings"},
-		TagDrop: []TagFilter{
-			TagFilter{
-				Name:   "badtag",
-				Filter: []string{"othertag"},
-			},
-		},
-		TagPass: []TagFilter{
-			TagFilter{
-				Name:   "goodtag",
-				Filter: []string{"mytag"},
-			},
-		},
-		Interval: 5 * time.Second,
-	}
-
-	ex := plugins.Plugins["exec"]().(*exec.Exec)
-	ex.Commands = []*exec.Command{
-		&exec.Command{
-			Command: "/usr/bin/myothercollector --foo=bar",
-			Name:    "myothercollector",
-		},
-	}
-
-	eConfig := &ConfiguredPlugin{Name: "exec"}
-
-	pstat := plugins.Plugins["procstat"]().(*procstat.Procstat)
-	pstat.Specifications = []*procstat.Specification{
-		&procstat.Specification{
-			PidFile: "/var/run/grafana-server.pid",
-		},
-		&procstat.Specification{
-			PidFile: "/var/run/influxdb/influxd.pid",
-		},
-	}
-
-	pConfig := &ConfiguredPlugin{Name: "procstat"}
-
-	assert.Equal(t, kafka, c.plugins["kafka"],
-		"Merged Testdata did not produce a correct kafka struct.")
-	assert.Equal(t, kConfig, c.pluginConfigurations["kafka"],
-		"Merged Testdata did not produce correct kafka metadata.")
-
-	assert.Equal(t, ex, c.plugins["exec"],
-		"Merged Testdata did not produce a correct exec struct.")
-	assert.Equal(t, eConfig, c.pluginConfigurations["exec"],
-		"Merged Testdata did not produce correct exec metadata.")
-
-	assert.Equal(t, pstat, c.plugins["procstat"],
-		"Merged Testdata did not produce a correct procstat struct.")
-	assert.Equal(t, pConfig, c.pluginConfigurations["procstat"],
-		"Merged Testdata did not produce correct procstat metadata.")
-}

etc/telegraf.conf

@@ -31,13 +31,13 @@
   # ie, if interval="10s" then always collect on :00, :10, :20, etc.
   round_interval = true
 
-  # Default data flushing interval for all outputs
+  # Default data flushing interval for all outputs. You should not set this below
+  # interval. Maximum flush_interval will be flush_interval + flush_jitter
   flush_interval = "10s"
-  # Jitter the flush interval by a random range
-  # ie, a jitter of 5s and interval 10s means flush will happen every 10-15s
-  flush_jitter = "5s"
-  # Number of times to retry each data flush
-  flush_retries = 2
+  # Jitter the flush interval by a random amount. This is primarily to avoid
+  # large write spikes for users running a large number of telegraf instances.
+  # ie, a jitter of 5s and interval 10s means flushes will happen every 10-15s
+  flush_jitter = "0s"
 
   # Run telegraf in debug mode
   debug = false
@@ -53,33 +53,36 @@
 
 # Configuration for influxdb server to send metrics to
 [[outputs.influxdb]]
-  # The full HTTP endpoint URL for your InfluxDB instance
-  # Multiple urls can be specified for InfluxDB cluster support. Server to
-  # write to will be randomly chosen each interval.
-  urls = ["http://localhost:8086"] # required.
-  # The target database for metrics. This database must already exist
-  database = "telegraf" # required.
+  # The full HTTP or UDP endpoint URL for your InfluxDB instance.
+  # Multiple urls can be specified but it is assumed that they are part of the same
+  # cluster, this means that only ONE of the urls will be written to each interval.
+  # urls = ["udp://localhost:8089"] # UDP endpoint example
+  urls = ["http://localhost:8086"] # required
+  # The target database for metrics (telegraf will create it if not exists)
+  database = "telegraf" # required
   # Precision of writes, valid values are n, u, ms, s, m, and h
   # note: using second precision greatly helps InfluxDB compression
   precision = "s"
 
   # Connection timeout (for the connection with InfluxDB), formatted as a string.
-  # Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h".
   # If not provided, will default to 0 (no timeout)
   # timeout = "5s"
   # username = "telegraf"
   # password = "metricsmetricsmetricsmetrics"
-
-  # Set the user agent for the POSTs (can be useful for log differentiation)
+  # Set the user agent for HTTP POSTs (can be useful for log differentiation)
   # user_agent = "telegraf"
+  # Set UDP payload size, defaults to InfluxDB UDP Client default (512 bytes)
+  # udp_payload = 512
 
 
 ###############################################################################
 #                                  PLUGINS                                    #
 ###############################################################################
 
+[plugins]
+
 # Read metrics about cpu usage
-[cpu]
+[[plugins.cpu]]
   # Whether to report per-cpu stats or not
   percpu = true
   # Whether to report total system cpu stats or not
@@ -88,21 +91,33 @@
   drop = ["cpu_time"]
 
 # Read metrics about disk usage by mount point
-[disk]
-  # no configuration
+[[plugins.disk]]
+  # By default, telegraf gather stats for all mountpoints.
+  # Setting mountpoints will restrict the stats to the specified mountpoints.
+  # Mountpoints=["/"]
 
 # Read metrics about disk IO by device
-[io]
-  # no configuration
+[[plugins.io]]
+  # By default, telegraf will gather stats for all devices including 
+  # disk partitions.
+  # Setting devices will restrict the stats to the specified devcies.
+  # Devices=["sda","sdb"]
+  # Uncomment the following line if you do not need disk serial numbers.
+  # SkipSerialNumber = true
 
 # Read metrics about memory usage
-[mem]
+[[plugins.mem]]
   # no configuration
 
 # Read metrics about swap memory usage
-[swap]
+[[plugins.swap]]
   # no configuration
 
 # Read metrics about system load & uptime
-[system]
+[[plugins.system]]
   # no configuration
+
+
+###############################################################################
+#                              SERVICE PLUGINS                                #
+###############################################################################

internal/config/config.go

@@ -0,0 +1,594 @@
+package config
+
+import (
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"path/filepath"
+	"sort"
+	"strings"
+	"time"
+
+	"github.com/influxdb/telegraf/internal"
+	"github.com/influxdb/telegraf/outputs"
+	"github.com/influxdb/telegraf/plugins"
+
+	"github.com/naoina/toml"
+	"github.com/naoina/toml/ast"
+)
+
+// Config specifies the URL/user/password for the database that telegraf
+// will be logging to, as well as all the plugins that the user has
+// specified
+type Config struct {
+	Tags          map[string]string
+	PluginFilters []string
+	OutputFilters []string
+
+	Agent   *AgentConfig
+	Plugins []*RunningPlugin
+	Outputs []*RunningOutput
+}
+
+func NewConfig() *Config {
+	c := &Config{
+		// Agent defaults:
+		Agent: &AgentConfig{
+			Interval:      internal.Duration{10 * time.Second},
+			RoundInterval: true,
+			FlushInterval: internal.Duration{10 * time.Second},
+			FlushRetries:  2,
+			FlushJitter:   internal.Duration{5 * time.Second},
+		},
+
+		Tags:          make(map[string]string),
+		Plugins:       make([]*RunningPlugin, 0),
+		Outputs:       make([]*RunningOutput, 0),
+		PluginFilters: make([]string, 0),
+		OutputFilters: make([]string, 0),
+	}
+	return c
+}
+
+type AgentConfig struct {
+	// Interval at which to gather information
+	Interval internal.Duration
+
+	// RoundInterval rounds collection interval to 'interval'.
+	//     ie, if Interval=10s then always collect on :00, :10, :20, etc.
+	RoundInterval bool
+
+	// Interval at which to flush data
+	FlushInterval internal.Duration
+
+	// FlushRetries is the number of times to retry each data flush
+	FlushRetries int
+
+	// FlushJitter tells
+	FlushJitter internal.Duration
+
+	// TODO(cam): Remove UTC and Precision parameters, they are no longer
+	// valid for the agent config. Leaving them here for now for backwards-
+	// compatability
+	UTC       bool `toml:"utc"`
+	Precision string
+
+	// Option for running in debug mode
+	Debug    bool
+	Hostname string
+}
+
+// TagFilter is the name of a tag, and the values on which to filter
+type TagFilter struct {
+	Name   string
+	Filter []string
+}
+
+type RunningOutput struct {
+	Name   string
+	Output outputs.Output
+}
+
+type RunningPlugin struct {
+	Name   string
+	Plugin plugins.Plugin
+	Config *PluginConfig
+}
+
+// PluginConfig containing a name, interval, and drop/pass prefix lists
+// Also lists the tags to filter
+type PluginConfig struct {
+	Name string
+
+	Drop []string
+	Pass []string
+
+	TagDrop []TagFilter
+	TagPass []TagFilter
+
+	Interval time.Duration
+}
+
+// ShouldPass returns true if the metric should pass, false if should drop
+// based on the drop/pass plugin parameters
+func (cp *PluginConfig) ShouldPass(measurement string) bool {
+	if cp.Pass != nil {
+		for _, pat := range cp.Pass {
+			if strings.HasPrefix(measurement, pat) {
+				return true
+			}
+		}
+		return false
+	}
+
+	if cp.Drop != nil {
+		for _, pat := range cp.Drop {
+			if strings.HasPrefix(measurement, pat) {
+				return false
+			}
+		}
+
+		return true
+	}
+	return true
+}
+
+// ShouldTagsPass returns true if the metric should pass, false if should drop
+// based on the tagdrop/tagpass plugin parameters
+func (cp *PluginConfig) ShouldTagsPass(tags map[string]string) bool {
+	if cp.TagPass != nil {
+		for _, pat := range cp.TagPass {
+			if tagval, ok := tags[pat.Name]; ok {
+				for _, filter := range pat.Filter {
+					if filter == tagval {
+						return true
+					}
+				}
+			}
+		}
+		return false
+	}
+
+	if cp.TagDrop != nil {
+		for _, pat := range cp.TagDrop {
+			if tagval, ok := tags[pat.Name]; ok {
+				for _, filter := range pat.Filter {
+					if filter == tagval {
+						return false
+					}
+				}
+			}
+		}
+		return true
+	}
+
+	return true
+}
+
+// Plugins returns a list of strings of the configured plugins.
+func (c *Config) PluginNames() []string {
+	var name []string
+	for _, plugin := range c.Plugins {
+		name = append(name, plugin.Name)
+	}
+	return name
+}
+
+// Outputs returns a list of strings of the configured plugins.
+func (c *Config) OutputNames() []string {
+	var name []string
+	for _, output := range c.Outputs {
+		name = append(name, output.Name)
+	}
+	return name
+}
+
+// ListTags returns a string of tags specified in the config,
+// line-protocol style
+func (c *Config) ListTags() string {
+	var tags []string
+
+	for k, v := range c.Tags {
+		tags = append(tags, fmt.Sprintf("%s=%s", k, v))
+	}
+
+	sort.Strings(tags)
+
+	return strings.Join(tags, " ")
+}
+
+var header = `# Telegraf configuration
+
+# Telegraf is entirely plugin driven. All metrics are gathered from the
+# declared plugins.
+
+# Even if a plugin has no configuration, it must be declared in here
+# to be active. Declaring a plugin means just specifying the name
+# as a section with no variables. To deactivate a plugin, comment
+# out the name and any variables.
+
+# Use 'telegraf -config telegraf.toml -test' to see what metrics a config
+# file would generate.
+
+# One rule that plugins conform to is wherever a connection string
+# can be passed, the values '' and 'localhost' are treated specially.
+# They indicate to the plugin to use their own builtin configuration to
+# connect to the local system.
+
+# NOTE: The configuration has a few required parameters. They are marked
+# with 'required'. Be sure to edit those to make this configuration work.
+
+# Tags can also be specified via a normal map, but only one form at a time:
+[tags]
+  # dc = "us-east-1"
+
+# Configuration for telegraf agent
+[agent]
+  # Default data collection interval for all plugins
+  interval = "10s"
+  # Rounds collection interval to 'interval'
+  # ie, if interval="10s" then always collect on :00, :10, :20, etc.
+  round_interval = true
+
+  # Default data flushing interval for all outputs. You should not set this below
+  # interval. Maximum flush_interval will be flush_interval + flush_jitter
+  flush_interval = "10s"
+  # Jitter the flush interval by a random amount. This is primarily to avoid
+  # large write spikes for users running a large number of telegraf instances.
+  # ie, a jitter of 5s and interval 10s means flushes will happen every 10-15s
+  flush_jitter = "0s"
+
+  # Run telegraf in debug mode
+  debug = false
+  # Override default hostname, if empty use os.Hostname()
+  hostname = ""
+
+
+###############################################################################
+#                                  OUTPUTS                                    #
+###############################################################################
+
+[outputs]
+`
+
+var pluginHeader = `
+
+###############################################################################
+#                                  PLUGINS                                    #
+###############################################################################
+
+[plugins]
+`
+
+var servicePluginHeader = `
+
+###############################################################################
+#                              SERVICE PLUGINS                                #
+###############################################################################
+`
+
+// PrintSampleConfig prints the sample config
+func PrintSampleConfig(pluginFilters []string, outputFilters []string) {
+	fmt.Printf(header)
+
+	// Filter outputs
+	var onames []string
+	for oname := range outputs.Outputs {
+		if len(outputFilters) == 0 || sliceContains(oname, outputFilters) {
+			onames = append(onames, oname)
+		}
+	}
+	sort.Strings(onames)
+
+	// Print Outputs
+	for _, oname := range onames {
+		creator := outputs.Outputs[oname]
+		output := creator()
+		printConfig(oname, output, "outputs")
+	}
+
+	// Filter plugins
+	var pnames []string
+	for pname := range plugins.Plugins {
+		if len(pluginFilters) == 0 || sliceContains(pname, pluginFilters) {
+			pnames = append(pnames, pname)
+		}
+	}
+	sort.Strings(pnames)
+
+	// Print Plugins
+	fmt.Printf(pluginHeader)
+	servPlugins := make(map[string]plugins.ServicePlugin)
+	for _, pname := range pnames {
+		creator := plugins.Plugins[pname]
+		plugin := creator()
+
+		switch p := plugin.(type) {
+		case plugins.ServicePlugin:
+			servPlugins[pname] = p
+			continue
+		}
+
+		printConfig(pname, plugin, "plugins")
+	}
+
+	// Print Service Plugins
+	fmt.Printf(servicePluginHeader)
+	for name, plugin := range servPlugins {
+		printConfig(name, plugin, "plugins")
+	}
+}
+
+type printer interface {
+	Description() string
+	SampleConfig() string
+}
+
+func printConfig(name string, p printer, op string) {
+	fmt.Printf("\n# %s\n[[%s.%s]]", p.Description(), op, name)
+	config := p.SampleConfig()
+	if config == "" {
+		fmt.Printf("\n  # no configuration\n")
+	} else {
+		fmt.Printf(config)
+	}
+}
+
+func sliceContains(name string, list []string) bool {
+	for _, b := range list {
+		if b == name {
+			return true
+		}
+	}
+	return false
+}
+
+// PrintPluginConfig prints the config usage of a single plugin.
+func PrintPluginConfig(name string) error {
+	if creator, ok := plugins.Plugins[name]; ok {
+		printConfig(name, creator(), "plugins")
+	} else {
+		return errors.New(fmt.Sprintf("Plugin %s not found", name))
+	}
+	return nil
+}
+
+// PrintOutputConfig prints the config usage of a single output.
+func PrintOutputConfig(name string) error {
+	if creator, ok := outputs.Outputs[name]; ok {
+		printConfig(name, creator(), "outputs")
+	} else {
+		return errors.New(fmt.Sprintf("Output %s not found", name))
+	}
+	return nil
+}
+
+func (c *Config) LoadDirectory(path string) error {
+	directoryEntries, err := ioutil.ReadDir(path)
+	if err != nil {
+		return err
+	}
+	for _, entry := range directoryEntries {
+		if entry.IsDir() {
+			continue
+		}
+		name := entry.Name()
+		if len(name) < 6 || name[len(name)-5:] != ".conf" {
+			continue
+		}
+		err := c.LoadConfig(filepath.Join(path, name))
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// LoadConfig loads the given config file and applies it to c
+func (c *Config) LoadConfig(path string) error {
+	data, err := ioutil.ReadFile(path)
+	if err != nil {
+		return err
+	}
+
+	tbl, err := toml.Parse(data)
+	if err != nil {
+		return err
+	}
+
+	for name, val := range tbl.Fields {
+		subTable, ok := val.(*ast.Table)
+		if !ok {
+			return errors.New("invalid configuration")
+		}
+
+		switch name {
+		case "agent":
+			if err = toml.UnmarshalTable(subTable, c.Agent); err != nil {
+				log.Printf("Could not parse [agent] config\n")
+				return err
+			}
+		case "tags":
+			if err = toml.UnmarshalTable(subTable, c.Tags); err != nil {
+				log.Printf("Could not parse [tags] config\n")
+				return err
+			}
+		case "outputs":
+			for outputName, outputVal := range subTable.Fields {
+				switch outputSubTable := outputVal.(type) {
+				case *ast.Table:
+					if err = c.addOutput(outputName, outputSubTable); err != nil {
+						return err
+					}
+				case []*ast.Table:
+					for _, t := range outputSubTable {
+						if err = c.addOutput(outputName, t); err != nil {
+							return err
+						}
+					}
+				default:
+					return fmt.Errorf("Unsupported config format: %s",
+						outputName)
+				}
+			}
+		case "plugins":
+			for pluginName, pluginVal := range subTable.Fields {
+				switch pluginSubTable := pluginVal.(type) {
+				case *ast.Table:
+					if err = c.addPlugin(pluginName, pluginSubTable); err != nil {
+						return err
+					}
+				case []*ast.Table:
+					for _, t := range pluginSubTable {
+						if err = c.addPlugin(pluginName, t); err != nil {
+							return err
+						}
+					}
+				default:
+					return fmt.Errorf("Unsupported config format: %s",
+						pluginName)
+				}
+			}
+		// Assume it's a plugin for legacy config file support if no other
+		// identifiers are present
+		default:
+			if err = c.addPlugin(name, subTable); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+func (c *Config) addOutput(name string, table *ast.Table) error {
+	if len(c.OutputFilters) > 0 && !sliceContains(name, c.OutputFilters) {
+		return nil
+	}
+	creator, ok := outputs.Outputs[name]
+	if !ok {
+		return fmt.Errorf("Undefined but requested output: %s", name)
+	}
+	o := creator()
+
+	if err := toml.UnmarshalTable(table, o); err != nil {
+		return err
+	}
+
+	ro := &RunningOutput{
+		Name:   name,
+		Output: o,
+	}
+	c.Outputs = append(c.Outputs, ro)
+	return nil
+}
+
+func (c *Config) addPlugin(name string, table *ast.Table) error {
+	if len(c.PluginFilters) > 0 && !sliceContains(name, c.PluginFilters) {
+		return nil
+	}
+	creator, ok := plugins.Plugins[name]
+	if !ok {
+		return fmt.Errorf("Undefined but requested plugin: %s", name)
+	}
+	plugin := creator()
+
+	pluginConfig, err := applyPlugin(name, table, plugin)
+	if err != nil {
+		return err
+	}
+	rp := &RunningPlugin{
+		Name:   name,
+		Plugin: plugin,
+		Config: pluginConfig,
+	}
+	c.Plugins = append(c.Plugins, rp)
+	return nil
+}
+
+// applyPlugin takes defined plugin names and applies them to the given
+// interface, returning a PluginConfig object in the end that can
+// be inserted into a runningPlugin by the agent.
+func applyPlugin(name string, tbl *ast.Table, p plugins.Plugin) (*PluginConfig, error) {
+	cp := &PluginConfig{Name: name}
+
+	if node, ok := tbl.Fields["pass"]; ok {
+		if kv, ok := node.(*ast.KeyValue); ok {
+			if ary, ok := kv.Value.(*ast.Array); ok {
+				for _, elem := range ary.Value {
+					if str, ok := elem.(*ast.String); ok {
+						cp.Pass = append(cp.Pass, str.Value)
+					}
+				}
+			}
+		}
+	}
+
+	if node, ok := tbl.Fields["drop"]; ok {
+		if kv, ok := node.(*ast.KeyValue); ok {
+			if ary, ok := kv.Value.(*ast.Array); ok {
+				for _, elem := range ary.Value {
+					if str, ok := elem.(*ast.String); ok {
+						cp.Drop = append(cp.Drop, str.Value)
+					}
+				}
+			}
+		}
+	}
+
+	if node, ok := tbl.Fields["interval"]; ok {
+		if kv, ok := node.(*ast.KeyValue); ok {
+			if str, ok := kv.Value.(*ast.String); ok {
+				dur, err := time.ParseDuration(str.Value)
+				if err != nil {
+					return nil, err
+				}
+
+				cp.Interval = dur
+			}
+		}
+	}
+
+	if node, ok := tbl.Fields["tagpass"]; ok {
+		if subtbl, ok := node.(*ast.Table); ok {
+			for name, val := range subtbl.Fields {
+				if kv, ok := val.(*ast.KeyValue); ok {
+					tagfilter := &TagFilter{Name: name}
+					if ary, ok := kv.Value.(*ast.Array); ok {
+						for _, elem := range ary.Value {
+							if str, ok := elem.(*ast.String); ok {
+								tagfilter.Filter = append(tagfilter.Filter, str.Value)
+							}
+						}
+					}
+					cp.TagPass = append(cp.TagPass, *tagfilter)
+				}
+			}
+		}
+	}
+
+	if node, ok := tbl.Fields["tagdrop"]; ok {
+		if subtbl, ok := node.(*ast.Table); ok {
+			for name, val := range subtbl.Fields {
+				if kv, ok := val.(*ast.KeyValue); ok {
+					tagfilter := &TagFilter{Name: name}
+					if ary, ok := kv.Value.(*ast.Array); ok {
+						for _, elem := range ary.Value {
+							if str, ok := elem.(*ast.String); ok {
+								tagfilter.Filter = append(tagfilter.Filter, str.Value)
+							}
+						}
+					}
+					cp.TagDrop = append(cp.TagDrop, *tagfilter)
+				}
+			}
+		}
+	}
+
+	delete(tbl.Fields, "drop")
+	delete(tbl.Fields, "pass")
+	delete(tbl.Fields, "interval")
+	delete(tbl.Fields, "tagdrop")
+	delete(tbl.Fields, "tagpass")
+	return cp, toml.UnmarshalTable(tbl, p)
+}

internal/config/config_test.go

@@ -0,0 +1,118 @@
+package config
+
+import (
+	"testing"
+	"time"
+
+	"github.com/influxdb/telegraf/plugins"
+	"github.com/influxdb/telegraf/plugins/exec"
+	"github.com/influxdb/telegraf/plugins/memcached"
+	"github.com/influxdb/telegraf/plugins/procstat"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestConfig_LoadSinglePlugin(t *testing.T) {
+	c := NewConfig()
+	c.LoadConfig("./testdata/single_plugin.toml")
+
+	memcached := plugins.Plugins["memcached"]().(*memcached.Memcached)
+	memcached.Servers = []string{"localhost"}
+
+	mConfig := &PluginConfig{
+		Name: "memcached",
+		Drop: []string{"other", "stuff"},
+		Pass: []string{"some", "strings"},
+		TagDrop: []TagFilter{
+			TagFilter{
+				Name:   "badtag",
+				Filter: []string{"othertag"},
+			},
+		},
+		TagPass: []TagFilter{
+			TagFilter{
+				Name:   "goodtag",
+				Filter: []string{"mytag"},
+			},
+		},
+		Interval: 5 * time.Second,
+	}
+
+	assert.Equal(t, memcached, c.Plugins[0].Plugin,
+		"Testdata did not produce a correct memcached struct.")
+	assert.Equal(t, mConfig, c.Plugins[0].Config,
+		"Testdata did not produce correct memcached metadata.")
+}
+
+func TestConfig_LoadDirectory(t *testing.T) {
+	c := NewConfig()
+	err := c.LoadConfig("./testdata/single_plugin.toml")
+	if err != nil {
+		t.Error(err)
+	}
+	err = c.LoadDirectory("./testdata/subconfig")
+	if err != nil {
+		t.Error(err)
+	}
+
+	memcached := plugins.Plugins["memcached"]().(*memcached.Memcached)
+	memcached.Servers = []string{"localhost"}
+
+	mConfig := &PluginConfig{
+		Name: "memcached",
+		Drop: []string{"other", "stuff"},
+		Pass: []string{"some", "strings"},
+		TagDrop: []TagFilter{
+			TagFilter{
+				Name:   "badtag",
+				Filter: []string{"othertag"},
+			},
+		},
+		TagPass: []TagFilter{
+			TagFilter{
+				Name:   "goodtag",
+				Filter: []string{"mytag"},
+			},
+		},
+		Interval: 5 * time.Second,
+	}
+	assert.Equal(t, memcached, c.Plugins[0].Plugin,
+		"Testdata did not produce a correct memcached struct.")
+	assert.Equal(t, mConfig, c.Plugins[0].Config,
+		"Testdata did not produce correct memcached metadata.")
+
+	ex := plugins.Plugins["exec"]().(*exec.Exec)
+	ex.Commands = []*exec.Command{
+		&exec.Command{
+			Command: "/usr/bin/myothercollector --foo=bar",
+			Name:    "myothercollector",
+		},
+	}
+	eConfig := &PluginConfig{Name: "exec"}
+	assert.Equal(t, ex, c.Plugins[1].Plugin,
+		"Merged Testdata did not produce a correct exec struct.")
+	assert.Equal(t, eConfig, c.Plugins[1].Config,
+		"Merged Testdata did not produce correct exec metadata.")
+
+	memcached.Servers = []string{"192.168.1.1"}
+	assert.Equal(t, memcached, c.Plugins[2].Plugin,
+		"Testdata did not produce a correct memcached struct.")
+	assert.Equal(t, mConfig, c.Plugins[2].Config,
+		"Testdata did not produce correct memcached metadata.")
+
+	pstat := plugins.Plugins["procstat"]().(*procstat.Procstat)
+	pstat.Specifications = []*procstat.Specification{
+		&procstat.Specification{
+			PidFile: "/var/run/grafana-server.pid",
+		},
+		&procstat.Specification{
+			PidFile: "/var/run/influxdb/influxd.pid",
+		},
+	}
+
+	pConfig := &PluginConfig{Name: "procstat"}
+
+	assert.Equal(t, pstat, c.Plugins[3].Plugin,
+		"Merged Testdata did not produce a correct procstat struct.")
+	assert.Equal(t, pConfig, c.Plugins[3].Config,
+		"Merged Testdata did not produce correct procstat metadata.")
+}

internal/config/testdata/single_plugin.toml

@@ -1,10 +1,9 @@
-[kafka]
-  zookeeperPeers = ["test.example.com:2181"]
-  batchSize = 10000
+[[plugins.memcached]]
+  servers = ["localhost"]
   pass = ["some", "strings"]
   drop = ["other", "stuff"]
   interval = "5s"
-  [kafka.tagpass]
+  [plugins.memcached.tagpass]
     goodtag = ["mytag"]
-  [kafka.tagdrop]
+  [plugins.memcached.tagdrop]
     badtag = ["othertag"]

internal/config/testdata/subconfig/exec.conf

@@ -1,6 +1,6 @@
-[exec]
+[[plugins.exec]]
   # specify commands via an array of tables
-  [[exec.commands]]
+  [[plugins.exec.commands]]
   # the command to run
   command = "/usr/bin/myothercollector --foo=bar"
 

internal/config/testdata/subconfig/memcached.conf

@@ -0,0 +1,9 @@
+[[plugins.memcached]]
+  servers = ["192.168.1.1"]
+  pass = ["some", "strings"]
+  drop = ["other", "stuff"]
+  interval = "5s"
+  [plugins.memcached.tagpass]
+    goodtag = ["mytag"]
+  [plugins.memcached.tagdrop]
+    badtag = ["othertag"]

internal/config/testdata/subconfig/procstat.conf

@@ -0,0 +1,5 @@
+[[plugins.procstat]]
+  [[plugins.procstat.specifications]]
+  pid_file = "/var/run/grafana-server.pid"
+  [[plugins.procstat.specifications]]
+  pid_file = "/var/run/influxdb/influxd.pid"

internal/config/testdata/telegraf-agent.toml

@@ -0,0 +1,334 @@
+# Telegraf configuration
+
+# Telegraf is entirely plugin driven. All metrics are gathered from the
+# declared plugins.
+
+# Even if a plugin has no configuration, it must be declared in here
+# to be active. Declaring a plugin means just specifying the name
+# as a section with no variables. To deactivate a plugin, comment
+# out the name and any variables.
+
+# Use 'telegraf -config telegraf.toml -test' to see what metrics a config
+# file would generate.
+
+# One rule that plugins conform to is wherever a connection string
+# can be passed, the values '' and 'localhost' are treated specially.
+# They indicate to the plugin to use their own builtin configuration to
+# connect to the local system.
+
+# NOTE: The configuration has a few required parameters. They are marked
+# with 'required'. Be sure to edit those to make this configuration work.
+
+# Tags can also be specified via a normal map, but only one form at a time:
+[tags]
+  # dc = "us-east-1"
+
+# Configuration for telegraf agent
+[agent]
+  # Default data collection interval for all plugins
+  interval = "10s"
+
+  # If utc = false, uses local time (utc is highly recommended)
+  utc = true
+
+  # Precision of writes, valid values are n, u, ms, s, m, and h
+  # note: using second precision greatly helps InfluxDB compression
+  precision = "s"
+
+  # run telegraf in debug mode
+  debug = false
+
+  # Override default hostname, if empty use os.Hostname()
+  hostname = ""
+
+
+###############################################################################
+#                                  OUTPUTS                                    #
+###############################################################################
+
+[outputs]
+
+# Configuration for influxdb server to send metrics to
+[[outputs.influxdb]]
+  # The full HTTP endpoint URL for your InfluxDB instance
+  # Multiple urls can be specified for InfluxDB cluster support. Server to
+  # write to will be randomly chosen each interval.
+  urls = ["http://localhost:8086"] # required.
+
+  # The target database for metrics. This database must already exist
+  database = "telegraf" # required.
+
+  # Connection timeout (for the connection with InfluxDB), formatted as a string.
+  # Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h".
+  # If not provided, will default to 0 (no timeout)
+  # timeout = "5s"
+
+  # username = "telegraf"
+  # password = "metricsmetricsmetricsmetrics"
+
+  # Set the user agent for the POSTs (can be useful for log differentiation)
+  # user_agent = "telegraf"
+
+[[outputs.influxdb]]
+  urls = ["udp://localhost:8089"]
+  database = "udp-telegraf"
+
+# Configuration for the Kafka server to send metrics to
+[[outputs.kafka]]
+  # URLs of kafka brokers
+  brokers = ["localhost:9092"]
+  # Kafka topic for producer messages
+  topic = "telegraf"
+  # Telegraf tag to use as a routing key
+  #  ie, if this tag exists, it's value will be used as the routing key
+  routing_tag = "host"
+
+
+###############################################################################
+#                                  PLUGINS                                    #
+###############################################################################
+
+[plugins]
+
+# Read Apache status information (mod_status)
+[[plugins.apache]]
+# An array of Apache status URI to gather stats.
+urls = ["http://localhost/server-status?auto"]
+
+# Read metrics about cpu usage
+[[plugins.cpu]]
+  # Whether to report per-cpu stats or not
+  percpu = true
+  # Whether to report total system cpu stats or not
+  totalcpu = true
+  # Comment this line if you want the raw CPU time metrics
+  drop = ["cpu_time"]
+
+# Read metrics about disk usage by mount point
+[[plugins.disk]]
+  # no configuration
+
+# Read metrics from one or many disque servers
+[[plugins.disque]]
+  # An array of URI to gather stats about. Specify an ip or hostname
+  # with optional port and password. ie disque://localhost, disque://10.10.3.33:18832,
+  # 10.0.0.1:10000, etc.
+  #
+  # If no servers are specified, then localhost is used as the host.
+  servers = ["localhost"]
+
+# Read stats from one or more Elasticsearch servers or clusters
+[[plugins.elasticsearch]]
+  # specify a list of one or more Elasticsearch servers
+  servers = ["http://localhost:9200"]
+
+  # set local to false when you want to read the indices stats from all nodes
+  # within the cluster
+  local = true
+
+# Read flattened metrics from one or more commands that output JSON to stdout
+[[plugins.exec]]
+  # specify commands via an array of tables
+  [[exec.commands]]
+  # the command to run
+  command = "/usr/bin/mycollector --foo=bar"
+
+  # name of the command (used as a prefix for measurements)
+  name = "mycollector"
+
+# Read metrics of haproxy, via socket or csv stats page
+[[plugins.haproxy]]
+  # An array of address to gather stats about. Specify an ip on hostname
+  # with optional port. ie localhost, 10.10.3.33:1936, etc.
+  #
+  # If no servers are specified, then default to 127.0.0.1:1936
+  servers = ["http://myhaproxy.com:1936", "http://anotherhaproxy.com:1936"]
+  # Or you can also use local socket(not work yet)
+  # servers = ["socket:/run/haproxy/admin.sock"]
+
+# Read flattened metrics from one or more JSON HTTP endpoints
+[[plugins.httpjson]]
+  # Specify services via an array of tables
+  [[httpjson.services]]
+
+    # a name for the service being polled
+    name = "webserver_stats"
+
+    # URL of each server in the service's cluster
+    servers = [
+      "http://localhost:9999/stats/",
+      "http://localhost:9998/stats/",
+    ]
+
+    # HTTP method to use (case-sensitive)
+    method = "GET"
+
+    # HTTP parameters (all values must be strings)
+    [httpjson.services.parameters]
+      event_type = "cpu_spike"
+      threshold = "0.75"
+
+# Read metrics about disk IO by device
+[[plugins.io]]
+  # no configuration
+
+# read metrics from a Kafka topic
+[[plugins.kafka_consumer]]
+  # topic(s) to consume
+  topics = ["telegraf"]
+  # an array of Zookeeper connection strings
+  zookeeper_peers = ["localhost:2181"]
+  # the name of the consumer group
+  consumer_group = "telegraf_metrics_consumers"
+  # Maximum number of points to buffer between collection intervals
+  point_buffer = 100000
+  # Offset (must be either "oldest" or "newest")
+  offset = "oldest"
+
+# Read metrics from a LeoFS Server via SNMP
+[[plugins.leofs]]
+  # An array of URI to gather stats about LeoFS.
+  # Specify an ip or hostname with port. ie 127.0.0.1:4020
+  #
+  # If no servers are specified, then 127.0.0.1 is used as the host and 4020 as the port.
+  servers = ["127.0.0.1:4021"]
+
+# Read metrics from local Lustre service on OST, MDS
+[[plugins.lustre2]]
+  # An array of /proc globs to search for Lustre stats
+  # If not specified, the default will work on Lustre 2.5.x
+  #
+  # ost_procfiles = ["/proc/fs/lustre/obdfilter/*/stats", "/proc/fs/lustre/osd-ldiskfs/*/stats"]
+  # mds_procfiles = ["/proc/fs/lustre/mdt/*/md_stats"]
+
+# Read metrics about memory usage
+[[plugins.mem]]
+  # no configuration
+
+# Read metrics from one or many memcached servers
+[[plugins.memcached]]
+  # An array of address to gather stats about. Specify an ip on hostname
+  # with optional port. ie localhost, 10.0.0.1:11211, etc.
+  #
+  # If no servers are specified, then localhost is used as the host.
+  servers = ["localhost"]
+
+# Read metrics from one or many MongoDB servers
+[[plugins.mongodb]]
+  # An array of URI to gather stats about. Specify an ip or hostname
+  # with optional port add password. ie mongodb://user:auth_key@10.10.3.30:27017,
+  # mongodb://10.10.3.33:18832, 10.0.0.1:10000, etc.
+  #
+  # If no servers are specified, then 127.0.0.1 is used as the host and 27107 as the port.
+  servers = ["127.0.0.1:27017"]
+
+# Read metrics from one or many mysql servers
+[[plugins.mysql]]
+  # specify servers via a url matching:
+  #  [username[:password]@][protocol[(address)]]/[?tls=[true|false|skip-verify]]
+  #  e.g.
+  #    servers = ["root:root@http://10.0.0.18/?tls=false"]
+  #    servers = ["root:passwd@tcp(127.0.0.1:3306)/"]
+  #
+  # If no servers are specified, then localhost is used as the host.
+  servers = ["localhost"]
+
+# Read metrics about network interface usage
+[[plugins.net]]
+  # By default, telegraf gathers stats from any up interface (excluding loopback)
+  # Setting interfaces will tell it to gather these explicit interfaces,
+  # regardless of status.
+  #
+  # interfaces = ["eth0", ... ]
+
+# Read Nginx's basic status information (ngx_http_stub_status_module)
+[[plugins.nginx]]
+  # An array of Nginx stub_status URI to gather stats.
+  urls = ["http://localhost/status"]
+
+# Ping given url(s) and return statistics
+[[plugins.ping]]
+  # urls to ping
+  urls = ["www.google.com"] # required
+  # number of pings to send (ping -c <COUNT>)
+  count = 1 # required
+  # interval, in s, at which to ping. 0 == default (ping -i <PING_INTERVAL>)
+  ping_interval = 0.0
+  # ping timeout, in s. 0 == no timeout (ping -t <TIMEOUT>)
+  timeout = 0.0
+  # interface to send ping from (ping -I <INTERFACE>)
+  interface = ""
+
+# Read metrics from one or many postgresql servers
+[[plugins.postgresql]]
+  # specify servers via an array of tables
+  [[postgresql.servers]]
+
+  # specify address via a url matching:
+  #   postgres://[pqgotest[:password]]@localhost[/dbname]?sslmode=[disable|verify-ca|verify-full]
+  # or a simple string:
+  #   host=localhost user=pqotest password=... sslmode=... dbname=app_production
+  #
+  # All connection parameters are optional. By default, the host is localhost
+  # and the user is the currently running user. For localhost, we default
+  # to sslmode=disable as well.
+  #
+  # Without the dbname parameter, the driver will default to a database
+  # with the same name as the user. This dbname is just for instantiating a
+  # connection with the server and doesn't restrict the databases we are trying
+  # to grab metrics for.
+  #
+
+  address = "sslmode=disable"
+
+  # A list of databases to pull metrics about. If not specified, metrics for all
+  # databases are gathered.
+
+  # databases = ["app_production", "blah_testing"]
+
+  # [[postgresql.servers]]
+  # address = "influx@remoteserver"
+
+# Read metrics from one or many prometheus clients
+[[plugins.prometheus]]
+  # An array of urls to scrape metrics from.
+  urls = ["http://localhost:9100/metrics"]
+
+# Read metrics from one or many RabbitMQ servers via the management API
+[[plugins.rabbitmq]]
+  # Specify servers via an array of tables
+  [[rabbitmq.servers]]
+  # name = "rmq-server-1" # optional tag
+  # url = "http://localhost:15672"
+  # username = "guest"
+  # password = "guest"
+
+  # A list of nodes to pull metrics about. If not specified, metrics for
+  # all nodes are gathered.
+  # nodes = ["rabbit@node1", "rabbit@node2"]
+
+# Read metrics from one or many redis servers
+[[plugins.redis]]
+  # An array of URI to gather stats about. Specify an ip or hostname
+  # with optional port add password. ie redis://localhost, redis://10.10.3.33:18832,
+  # 10.0.0.1:10000, etc.
+  #
+  # If no servers are specified, then localhost is used as the host.
+  servers = ["localhost"]
+
+# Read metrics from one or many RethinkDB servers
+[[plugins.rethinkdb]]
+  # An array of URI to gather stats about. Specify an ip or hostname
+  # with optional port add password. ie rethinkdb://user:auth_key@10.10.3.30:28105,
+  # rethinkdb://10.10.3.33:18832, 10.0.0.1:10000, etc.
+  #
+  # If no servers are specified, then 127.0.0.1 is used as the host and 28015 as the port.
+  servers = ["127.0.0.1:28015"]
+
+# Read metrics about swap memory usage
+[[plugins.swap]]
+  # no configuration
+
+# Read metrics about system load & uptime
+[[plugins.system]]
+  # no configuration

outputs/all/all.go

@@ -11,4 +11,5 @@ import (
 	_ "github.com/influxdb/telegraf/outputs/nsq"
 	_ "github.com/influxdb/telegraf/outputs/opentsdb"
 	_ "github.com/influxdb/telegraf/outputs/prometheus_client"
+	_ "github.com/influxdb/telegraf/outputs/riemann"
 )

outputs/influxdb/influxdb.go

@@ -29,8 +29,9 @@ type InfluxDB struct {
 }
 
 var sampleConfig = `
-  # The full HTTP or UDP endpoint URL for your InfluxDB instance
-  # Multiple urls can be specified for InfluxDB cluster support.
+  # The full HTTP or UDP endpoint URL for your InfluxDB instance.
+  # Multiple urls can be specified but it is assumed that they are part of the same
+  # cluster, this means that only ONE of the urls will be written to each interval.
   # urls = ["udp://localhost:8089"] # UDP endpoint example
   urls = ["http://localhost:8086"] # required
   # The target database for metrics (telegraf will create it if not exists)

outputs/riemann/riemann.go

@@ -0,0 +1,95 @@
+package riemann
+
+import (
+	"errors"
+	"fmt"
+	"os"
+
+	"github.com/amir/raidman"
+	"github.com/influxdb/influxdb/client/v2"
+	"github.com/influxdb/telegraf/outputs"
+)
+
+type Riemann struct {
+	URL       string
+	Transport string
+
+	client *raidman.Client
+}
+
+var sampleConfig = `
+  # URL of server
+  url = "localhost:5555"
+  # transport protocol to use either tcp or udp
+  transport = "tcp"
+`
+
+func (r *Riemann) Connect() error {
+	c, err := raidman.Dial(r.Transport, r.URL)
+
+	if err != nil {
+		return err
+	}
+
+	r.client = c
+	return nil
+}
+
+func (r *Riemann) Close() error {
+	r.client.Close()
+	return nil
+}
+
+func (r *Riemann) SampleConfig() string {
+	return sampleConfig
+}
+
+func (r *Riemann) Description() string {
+	return "Configuration for the Riemann server to send metrics to"
+}
+
+func (r *Riemann) Write(points []*client.Point) error {
+	if len(points) == 0 {
+		return nil
+	}
+
+	var events []*raidman.Event
+	for _, p := range points {
+		ev := buildEvent(p)
+		events = append(events, ev)
+	}
+
+	var senderr = r.client.SendMulti(events)
+	if senderr != nil {
+		return errors.New(fmt.Sprintf("FAILED to send riemann message: %s\n",
+			senderr))
+	}
+
+	return nil
+}
+
+func buildEvent(p *client.Point) *raidman.Event {
+	host, ok := p.Tags()["host"]
+	if !ok {
+		hostname, err := os.Hostname()
+		if err != nil {
+			host = "unknown"
+		} else {
+			host = hostname
+		}
+	}
+
+	var event = &raidman.Event{
+		Host:    host,
+		Service: p.Name(),
+		Metric:  p.Fields()["value"],
+	}
+
+	return event
+}
+
+func init() {
+	outputs.Add("riemann", func() outputs.Output {
+		return &Riemann{}
+	})
+}

outputs/riemann/riemann_test.go

@@ -0,0 +1,27 @@
+package riemann
+
+import (
+	"testing"
+
+	"github.com/influxdb/telegraf/testutil"
+	"github.com/stretchr/testify/require"
+)
+
+func TestConnectAndWrite(t *testing.T) {
+	if testing.Short() {
+		t.Skip("Skipping integration test in short mode")
+	}
+
+	url := testutil.GetLocalHost() + ":5555"
+
+	r := &Riemann{
+		URL:       url,
+		Transport: "tcp",
+	}
+
+	err := r.Connect()
+	require.NoError(t, err)
+
+	err = r.Write(testutil.MockBatchPoints().Points())
+	require.NoError(t, err)
+}

plugins/aerospike/aerospike.go

@@ -250,8 +250,8 @@ func get(key []byte, host string) (map[string]string, error) {
 func readAerospikeStats(stats map[string]string, acc plugins.Accumulator, host, namespace string) {
 	for key, value := range stats {
 		tags := map[string]string{
-			"host":      host,
-			"namespace": "_service",
+			"aerospike_host": host,
+			"namespace":      "_service",
 		}
 
 		if namespace != "" {

plugins/aerospike/aerospike_test.go

@@ -67,8 +67,8 @@ func TestReadAerospikeStatsNamespace(t *testing.T) {
 	readAerospikeStats(stats, &acc, "host1", "test")
 
 	tags := map[string]string{
-		"host":      "host1",
-		"namespace": "test",
+		"aerospike_host": "host1",
+		"namespace":      "test",
 	}
 	for k := range stats {
 		assert.True(t, acc.ValidateTaggedValue(k, int64(12345), tags) == nil)

plugins/exec/exec.go

@@ -16,7 +16,7 @@ import (
 
 const sampleConfig = `
   # specify commands via an array of tables
-  [[exec.commands]]
+  [[plugins.exec.commands]]
   # the command to run
   command = "/usr/bin/mycollector --foo=bar"
 

plugins/httpjson/httpjson.go

@@ -48,7 +48,7 @@ func (c RealHTTPClient) MakeRequest(req *http.Request) (*http.Response, error) {
 
 var sampleConfig = `
   # Specify services via an array of tables
-  [[httpjson.services]]
+  [[plugins.httpjson.services]]
 
     # a name for the service being polled
     name = "webserver_stats"
@@ -69,7 +69,7 @@ var sampleConfig = `
     # ]
 
     # HTTP parameters (all values must be strings)
-    [httpjson.services.parameters]
+    [plugins.httpjson.services.parameters]
       event_type = "cpu_spike"
       threshold = "0.75"
 `

plugins/jolokia/jolokia.go

@@ -55,7 +55,7 @@ func (j *Jolokia) SampleConfig() string {
     group = "as"
 
   # List of servers exposing jolokia read service
-  [[jolokia.servers]]
+  [[plugins.jolokia.servers]]
     name = "stable"
     host = "192.168.103.2"
     port = "8180"
@@ -63,20 +63,20 @@ func (j *Jolokia) SampleConfig() string {
   # List of metrics collected on above servers
   # Each metric consists in a name, a jmx path and either a pass or drop slice attributes
   # This collect all heap memory usage metrics
-  [[jolokia.metrics]]
+  [[plugins.jolokia.metrics]]
     name = "heap_memory_usage"
     jmx  = "/java.lang:type=Memory/HeapMemoryUsage"
 
 
   # This drops the 'committed' value from Eden space measurement
-  [[jolokia.metrics]]
+  [[plugins.jolokia.metrics]]
     name = "memory_eden"
     jmx  = "/java.lang:type=MemoryPool,name=PS Eden Space/Usage"
     drop = [ "committed" ]
 
 
   # This passes only DaemonThreadCount and ThreadCount
-  [[jolokia.metrics]]
+  [[plugins.jolokia.metrics]]
     name = "heap_threads"
     jmx  = "/java.lang:type=Threading"
     pass = [

plugins/kafka_consumer/kafka_consumer.go

@@ -1,36 +1,51 @@
 package kafka_consumer
 
 import (
-	"os"
-	"os/signal"
-	"time"
+	"log"
+	"strings"
+	"sync"
 
-	"github.com/Shopify/sarama"
 	"github.com/influxdb/influxdb/models"
 	"github.com/influxdb/telegraf/plugins"
+
+	"github.com/Shopify/sarama"
 	"github.com/wvanbergen/kafka/consumergroup"
 )
 
 type Kafka struct {
-	ConsumerGroupName string
-	Topic             string
-	ZookeeperPeers    []string
-	Consumer          *consumergroup.ConsumerGroup
-	BatchSize         int
+	ConsumerGroup  string
+	Topics         []string
+	ZookeeperPeers []string
+	Consumer       *consumergroup.ConsumerGroup
+	PointBuffer    int
+	Offset         string
+
+	sync.Mutex
+
+	// channel for all incoming kafka messages
+	in <-chan *sarama.ConsumerMessage
+	// channel for all kafka consumer errors
+	errs <-chan *sarama.ConsumerError
+	// channel for all incoming parsed kafka points
+	pointChan chan models.Point
+	done      chan struct{}
+
+	// doNotCommitMsgs tells the parser not to call CommitUpTo on the consumer
+	// this is mostly for test purposes, but there may be a use-case for it later.
+	doNotCommitMsgs bool
 }
 
 var sampleConfig = `
-  # topic to consume
-  topic = "topic_with_metrics"
-
-  # the name of the consumer group
-  consumerGroupName = "telegraf_metrics_consumers"
-
+  # topic(s) to consume
+  topics = ["telegraf"]
   # an array of Zookeeper connection strings
-  zookeeperPeers = ["localhost:2181"]
-
-  # Batch size of points sent to InfluxDB
-  batchSize = 1000
+  zookeeper_peers = ["localhost:2181"]
+  # the name of the consumer group
+  consumer_group = "telegraf_metrics_consumers"
+  # Maximum number of points to buffer between collection intervals
+  point_buffer = 100000
+  # Offset (must be either "oldest" or "newest")
+  offset = "oldest"
 `
 
 func (k *Kafka) SampleConfig() string {
@@ -38,127 +53,114 @@ func (k *Kafka) SampleConfig() string {
 }
 
 func (k *Kafka) Description() string {
-	return "read metrics from a Kafka topic"
+	return "Read line-protocol metrics from Kafka topic(s)"
 }
 
-type Metric struct {
-	Measurement string                 `json:"measurement"`
-	Values      map[string]interface{} `json:"values"`
-	Tags        map[string]string      `json:"tags"`
-	Time        time.Time              `json:"time"`
-}
-
-func (k *Kafka) Gather(acc plugins.Accumulator) error {
+func (k *Kafka) Start() error {
+	k.Lock()
+	defer k.Unlock()
 	var consumerErr error
-	metricQueue := make(chan []byte, 200)
 
-	if k.Consumer == nil {
+	config := consumergroup.NewConfig()
+	switch strings.ToLower(k.Offset) {
+	case "oldest", "":
+		config.Offsets.Initial = sarama.OffsetOldest
+	case "newest":
+		config.Offsets.Initial = sarama.OffsetNewest
+	default:
+		log.Printf("WARNING: Kafka consumer invalid offset '%s', using 'oldest'\n",
+			k.Offset)
+		config.Offsets.Initial = sarama.OffsetOldest
+	}
+
+	if k.Consumer == nil || k.Consumer.Closed() {
 		k.Consumer, consumerErr = consumergroup.JoinConsumerGroup(
-			k.ConsumerGroupName,
-			[]string{k.Topic},
+			k.ConsumerGroup,
+			k.Topics,
 			k.ZookeeperPeers,
-			nil,
+			config,
 		)
-
 		if consumerErr != nil {
 			return consumerErr
 		}
 
-		c := make(chan os.Signal, 1)
-		halt := make(chan bool, 1)
-		signal.Notify(c, os.Interrupt)
-		go func() {
-			<-c
-			halt <- true
-			emitMetrics(k, acc, metricQueue)
-			k.Consumer.Close()
-		}()
-
-		go readFromKafka(k.Consumer.Messages(),
-			metricQueue,
-			k.BatchSize,
-			k.Consumer.CommitUpto,
-			halt)
+		// Setup message and error channels
+		k.in = k.Consumer.Messages()
+		k.errs = k.Consumer.Errors()
 	}
 
-	return emitMetrics(k, acc, metricQueue)
+	k.done = make(chan struct{})
+	if k.PointBuffer == 0 {
+		k.PointBuffer = 100000
+	}
+	k.pointChan = make(chan models.Point, k.PointBuffer)
+
+	// Start the kafka message reader
+	go k.parser()
+	log.Printf("Started the kafka consumer service, peers: %v, topics: %v\n",
+		k.ZookeeperPeers, k.Topics)
+	return nil
 }
 
-func emitMetrics(k *Kafka, acc plugins.Accumulator, metricConsumer <-chan []byte) error {
-	timeout := time.After(1 * time.Second)
-
+// parser() reads all incoming messages from the consumer, and parses them into
+// influxdb metric points.
+func (k *Kafka) parser() {
 	for {
 		select {
-		case batch := <-metricConsumer:
-			var points []models.Point
-			var err error
-			if points, err = models.ParsePoints(batch); err != nil {
-				return err
+		case <-k.done:
+			return
+		case err := <-k.errs:
+			log.Printf("Kafka Consumer Error: %s\n", err.Error())
+		case msg := <-k.in:
+			points, err := models.ParsePoints(msg.Value)
+			if err != nil {
+				log.Printf("Could not parse kafka message: %s, error: %s",
+					string(msg.Value), err.Error())
 			}
 
 			for _, point := range points {
-				acc.AddFields(point.Name(), point.Fields(), point.Tags(), point.Time())
+				select {
+				case k.pointChan <- point:
+					continue
+				default:
+					log.Printf("Kafka Consumer buffer is full, dropping a point." +
+						" You may want to increase the point_buffer setting")
+				}
+			}
+
+			if !k.doNotCommitMsgs {
+				// TODO(cam) this locking can be removed if this PR gets merged:
+				// https://github.com/wvanbergen/kafka/pull/84
+				k.Lock()
+				k.Consumer.CommitUpto(msg)
+				k.Unlock()
 			}
-		case <-timeout:
-			return nil
 		}
 	}
 }
 
-const millisecond = 1000000 * time.Nanosecond
-
-type ack func(*sarama.ConsumerMessage) error
-
-func readFromKafka(
-	kafkaMsgs <-chan *sarama.ConsumerMessage,
-	metricProducer chan<- []byte,
-	maxBatchSize int,
-	ackMsg ack,
-	halt <-chan bool,
-) {
-	batch := make([]byte, 0)
-	currentBatchSize := 0
-	timeout := time.After(500 * millisecond)
-	var msg *sarama.ConsumerMessage
-
-	for {
-		select {
-		case msg = <-kafkaMsgs:
-			if currentBatchSize != 0 {
-				batch = append(batch, '\n')
-			}
-
-			batch = append(batch, msg.Value...)
-			currentBatchSize++
-
-			if currentBatchSize == maxBatchSize {
-				metricProducer <- batch
-				currentBatchSize = 0
-				batch = make([]byte, 0)
-				ackMsg(msg)
-			}
-		case <-timeout:
-			if currentBatchSize != 0 {
-				metricProducer <- batch
-				currentBatchSize = 0
-				batch = make([]byte, 0)
-				ackMsg(msg)
-			}
-
-			timeout = time.After(500 * millisecond)
-		case <-halt:
-			if currentBatchSize != 0 {
-				metricProducer <- batch
-				ackMsg(msg)
-			}
-
-			return
-		}
+func (k *Kafka) Stop() {
+	k.Lock()
+	defer k.Unlock()
+	close(k.done)
+	if err := k.Consumer.Close(); err != nil {
+		log.Printf("Error closing kafka consumer: %s\n", err.Error())
 	}
 }
 
+func (k *Kafka) Gather(acc plugins.Accumulator) error {
+	k.Lock()
+	defer k.Unlock()
+	npoints := len(k.pointChan)
+	for i := 0; i < npoints; i++ {
+		point := <-k.pointChan
+		acc.AddFields(point.Name(), point.Fields(), point.Tags(), point.Time())
+	}
+	return nil
+}
+
 func init() {
-	plugins.Add("kafka", func() plugins.Plugin {
+	plugins.Add("kafka_consumer", func() plugins.Plugin {
 		return &Kafka{}
 	})
 }

plugins/kafka_consumer/kafka_consumer_integration_test.go

@@ -15,43 +15,77 @@ func TestReadsMetricsFromKafka(t *testing.T) {
 	if testing.Short() {
 		t.Skip("Skipping integration test in short mode")
 	}
-	var zkPeers, brokerPeers []string
 
-	zkPeers = []string{testutil.GetLocalHost() + ":2181"}
-	brokerPeers = []string{testutil.GetLocalHost() + ":9092"}
-
-	k := &Kafka{
-		ConsumerGroupName: "telegraf_test_consumers",
-		Topic:             fmt.Sprintf("telegraf_test_topic_%d", time.Now().Unix()),
-		ZookeeperPeers:    zkPeers,
-	}
+	brokerPeers := []string{testutil.GetLocalHost() + ":9092"}
+	zkPeers := []string{testutil.GetLocalHost() + ":2181"}
+	testTopic := fmt.Sprintf("telegraf_test_topic_%d", time.Now().Unix())
 
+	// Send a Kafka message to the kafka host
 	msg := "cpu_load_short,direction=in,host=server01,region=us-west value=23422.0 1422568543702900257"
 	producer, err := sarama.NewSyncProducer(brokerPeers, nil)
 	require.NoError(t, err)
-
-	_, _, err = producer.SendMessage(&sarama.ProducerMessage{Topic: k.Topic, Value: sarama.StringEncoder(msg)})
+	_, _, err = producer.SendMessage(
+		&sarama.ProducerMessage{
+			Topic: testTopic,
+			Value: sarama.StringEncoder(msg),
+		})
 	require.NoError(t, err)
+	defer producer.Close()
 
-	producer.Close()
+	// Start the Kafka Consumer
+	k := &Kafka{
+		ConsumerGroup:  "telegraf_test_consumers",
+		Topics:         []string{testTopic},
+		ZookeeperPeers: zkPeers,
+		PointBuffer:    100000,
+		Offset:         "oldest",
+	}
+	if err := k.Start(); err != nil {
+		t.Fatal(err.Error())
+	} else {
+		defer k.Stop()
+	}
 
+	waitForPoint(k, t)
+
+	// Verify that we can now gather the sent message
 	var acc testutil.Accumulator
-
 	// Sanity check
-	assert.Equal(t, 0, len(acc.Points), "there should not be any points")
+	assert.Equal(t, 0, len(acc.Points), "There should not be any points")
 
+	// Gather points
 	err = k.Gather(&acc)
 	require.NoError(t, err)
-
-	assert.Equal(t, 1, len(acc.Points), "there should be a single point")
-
-	point := acc.Points[0]
-	assert.Equal(t, "cpu_load_short", point.Measurement)
-	assert.Equal(t, map[string]interface{}{"value": 23422.0}, point.Fields)
-	assert.Equal(t, map[string]string{
-		"host":      "server01",
-		"direction": "in",
-		"region":    "us-west",
-	}, point.Tags)
-	assert.Equal(t, time.Unix(0, 1422568543702900257).Unix(), point.Time.Unix())
+	if len(acc.Points) == 1 {
+		point := acc.Points[0]
+		assert.Equal(t, "cpu_load_short", point.Measurement)
+		assert.Equal(t, map[string]interface{}{"value": 23422.0}, point.Fields)
+		assert.Equal(t, map[string]string{
+			"host":      "server01",
+			"direction": "in",
+			"region":    "us-west",
+		}, point.Tags)
+		assert.Equal(t, time.Unix(0, 1422568543702900257).Unix(), point.Time.Unix())
+	} else {
+		t.Errorf("No points found in accumulator, expected 1")
+	}
+}
+
+// Waits for the metric that was sent to the kafka broker to arrive at the kafka
+// consumer
+func waitForPoint(k *Kafka, t *testing.T) {
+	// Give the kafka container up to 2 seconds to get the point to the consumer
+	ticker := time.NewTicker(5 * time.Millisecond)
+	counter := 0
+	for {
+		select {
+		case <-ticker.C:
+			counter++
+			if counter > 1000 {
+				t.Fatal("Waited for 5s, point never arrived to consumer")
+			} else if len(k.pointChan) == 1 {
+				return
+			}
+		}
+	}
 }

plugins/kafka_consumer/kafka_consumer_test.go

@@ -1,92 +1,91 @@
 package kafka_consumer
 
 import (
-	"strings"
 	"testing"
 	"time"
 
-	"github.com/Shopify/sarama"
+	"github.com/influxdb/influxdb/models"
 	"github.com/influxdb/telegraf/testutil"
+
+	"github.com/Shopify/sarama"
 	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
 )
 
-const testMsg = "cpu_load_short,direction=in,host=server01,region=us-west value=23422.0 1422568543702900257"
+const (
+	testMsg     = "cpu_load_short,host=server01 value=23422.0 1422568543702900257"
+	invalidMsg  = "cpu_load_short,host=server01 1422568543702900257"
+	pointBuffer = 5
+)
 
-func TestReadFromKafkaBatchesMsgsOnBatchSize(t *testing.T) {
-	halt := make(chan bool, 1)
-	metricChan := make(chan []byte, 1)
-	kafkaChan := make(chan *sarama.ConsumerMessage, 10)
-	for i := 0; i < 10; i++ {
-		kafkaChan <- saramaMsg(testMsg)
+func NewTestKafka() (*Kafka, chan *sarama.ConsumerMessage) {
+	in := make(chan *sarama.ConsumerMessage, pointBuffer)
+	k := Kafka{
+		ConsumerGroup:   "test",
+		Topics:          []string{"telegraf"},
+		ZookeeperPeers:  []string{"localhost:2181"},
+		PointBuffer:     pointBuffer,
+		Offset:          "oldest",
+		in:              in,
+		doNotCommitMsgs: true,
+		errs:            make(chan *sarama.ConsumerError, pointBuffer),
+		done:            make(chan struct{}),
+		pointChan:       make(chan models.Point, pointBuffer),
 	}
-
-	expectedBatch := strings.Repeat(testMsg+"\n", 9) + testMsg
-	readFromKafka(kafkaChan, metricChan, 10, func(msg *sarama.ConsumerMessage) error {
-		batch := <-metricChan
-		assert.Equal(t, expectedBatch, string(batch))
-
-		halt <- true
-
-		return nil
-	}, halt)
+	return &k, in
 }
 
-func TestReadFromKafkaBatchesMsgsOnTimeout(t *testing.T) {
-	halt := make(chan bool, 1)
-	metricChan := make(chan []byte, 1)
-	kafkaChan := make(chan *sarama.ConsumerMessage, 10)
-	for i := 0; i < 3; i++ {
-		kafkaChan <- saramaMsg(testMsg)
-	}
+// Test that the parser parses kafka messages into points
+func TestRunParser(t *testing.T) {
+	k, in := NewTestKafka()
+	defer close(k.done)
 
-	expectedBatch := strings.Repeat(testMsg+"\n", 2) + testMsg
-	readFromKafka(kafkaChan, metricChan, 10, func(msg *sarama.ConsumerMessage) error {
-		batch := <-metricChan
-		assert.Equal(t, expectedBatch, string(batch))
+	go k.parser()
+	in <- saramaMsg(testMsg)
+	time.Sleep(time.Millisecond)
 
-		halt <- true
-
-		return nil
-	}, halt)
+	assert.Equal(t, len(k.pointChan), 1)
 }
 
-func TestEmitMetricsSendMetricsToAcc(t *testing.T) {
-	k := &Kafka{}
-	var acc testutil.Accumulator
-	testChan := make(chan []byte, 1)
-	testChan <- []byte(testMsg)
+// Test that the parser ignores invalid messages
+func TestRunParserInvalidMsg(t *testing.T) {
+	k, in := NewTestKafka()
+	defer close(k.done)
 
-	err := emitMetrics(k, &acc, testChan)
-	require.NoError(t, err)
+	go k.parser()
+	in <- saramaMsg(invalidMsg)
+	time.Sleep(time.Millisecond)
 
-	assert.Equal(t, 1, len(acc.Points), "there should be a single point")
-
-	point := acc.Points[0]
-	assert.Equal(t, "cpu_load_short", point.Measurement)
-	assert.Equal(t, map[string]interface{}{"value": 23422.0}, point.Fields)
-	assert.Equal(t, map[string]string{
-		"host":      "server01",
-		"direction": "in",
-		"region":    "us-west",
-	}, point.Tags)
-
-	if time.Unix(0, 1422568543702900257).Unix() != point.Time.Unix() {
-		t.Errorf("Expected: %v, received %v\n",
-			time.Unix(0, 1422568543702900257).Unix(),
-			point.Time.Unix())
-	}
+	assert.Equal(t, len(k.pointChan), 0)
 }
 
-func TestEmitMetricsTimesOut(t *testing.T) {
-	k := &Kafka{}
-	var acc testutil.Accumulator
-	testChan := make(chan []byte)
+// Test that points are dropped when we hit the buffer limit
+func TestRunParserRespectsBuffer(t *testing.T) {
+	k, in := NewTestKafka()
+	defer close(k.done)
 
-	err := emitMetrics(k, &acc, testChan)
-	require.NoError(t, err)
+	go k.parser()
+	for i := 0; i < pointBuffer+1; i++ {
+		in <- saramaMsg(testMsg)
+	}
+	time.Sleep(time.Millisecond)
 
-	assert.Equal(t, 0, len(acc.Points), "there should not be a any points")
+	assert.Equal(t, len(k.pointChan), 5)
+}
+
+// Test that the parser parses kafka messages into points
+func TestRunParserAndGather(t *testing.T) {
+	k, in := NewTestKafka()
+	defer close(k.done)
+
+	go k.parser()
+	in <- saramaMsg(testMsg)
+	time.Sleep(time.Millisecond)
+
+	acc := testutil.Accumulator{}
+	k.Gather(&acc)
+
+	assert.Equal(t, len(acc.Points), 1)
+	assert.True(t, acc.CheckValue("cpu_load_short", 23422.0))
 }
 
 func saramaMsg(val string) *sarama.ConsumerMessage {

plugins/postgresql/postgresql.go

@@ -25,7 +25,7 @@ var ignoredColumns = map[string]bool{"datid": true, "datname": true, "stats_rese
 
 var sampleConfig = `
   # specify servers via an array of tables
-  [[postgresql.servers]]
+  [[plugins.postgresql.servers]]
 
   # specify address via a url matching:
   #   postgres://[pqgotest[:password]]@localhost[/dbname]?sslmode=[disable|verify-ca|verify-full]
@@ -49,7 +49,7 @@ var sampleConfig = `
 
   # databases = ["app_production", "blah_testing"]
 
-  # [[postgresql.servers]]
+  # [[plugins.postgresql.servers]]
   # address = "influx@remoteserver"
 `
 

plugins/procstat/procstat.go

@@ -18,6 +18,7 @@ type Specification struct {
 	PidFile string `toml:"pid_file"`
 	Exe     string
 	Prefix  string
+	Pattern string
 }
 
 type Procstat struct {
@@ -29,12 +30,15 @@ func NewProcstat() *Procstat {
 }
 
 var sampleConfig = `
-  [[procstat.specifications]]
+  [[plugins.procstat.specifications]]
   prefix = "" # optional string to prefix measurements
-  # Use one of pid_file or exe to find process
+  # Must specify one of: pid_file, exe, or pattern
+  # PID file to monitor process
   pid_file = "/var/run/nginx.pid"
-  # executable name (used by pgrep)
+  # executable name (ie, pgrep <exe>)
   # exe = "nginx"
+  # pattern as argument for pgrep (ie, pgrep -f <pattern>)
+  # pattern = "nginx"
 `
 
 func (_ *Procstat) SampleConfig() string {
@@ -54,8 +58,8 @@ func (p *Procstat) Gather(acc plugins.Accumulator) error {
 			defer wg.Done()
 			procs, err := spec.createProcesses()
 			if err != nil {
-				log.Printf("Error: procstat getting process, exe: [%s] pidfile: [%s] %s",
-					spec.Exe, spec.PidFile, err.Error())
+				log.Printf("Error: procstat getting process, exe: [%s] pidfile: [%s] pattern: [%s] %s",
+					spec.Exe, spec.PidFile, spec.Pattern, err.Error())
 			} else {
 				for _, proc := range procs {
 					p := NewSpecProcessor(spec.Prefix, acc, proc)
@@ -103,8 +107,10 @@ func (spec *Specification) getAllPids() ([]int32, error) {
 		pids, err = pidsFromFile(spec.PidFile)
 	} else if spec.Exe != "" {
 		pids, err = pidsFromExe(spec.Exe)
+	} else if spec.Pattern != "" {
+		pids, err = pidsFromPattern(spec.Pattern)
 	} else {
-		err = fmt.Errorf("Either exe or pid_file has to be specified")
+		err = fmt.Errorf("Either exe, pid_file or pattern has to be specified")
 	}
 
 	return pids, err
@@ -147,6 +153,26 @@ func pidsFromExe(exe string) ([]int32, error) {
 	return out, outerr
 }
 
+func pidsFromPattern(pattern string) ([]int32, error) {
+	var out []int32
+	var outerr error
+	pgrep, err := exec.Command("pgrep", "-f", pattern).Output()
+	if err != nil {
+		return out, fmt.Errorf("Failed to execute pgrep. Error: '%s'", err)
+	} else {
+		pids := strings.Fields(string(pgrep))
+		for _, pid := range pids {
+			ipid, err := strconv.Atoi(pid)
+			if err == nil {
+				out = append(out, int32(ipid))
+			} else {
+				outerr = err
+			}
+		}
+	}
+	return out, outerr
+}
+
 func init() {
 	plugins.Add("procstat", func() plugins.Plugin {
 		return NewProcstat()

plugins/prometheus/prometheus.go

@@ -87,7 +87,8 @@ func (g *Prometheus) gatherURL(url string, acc plugins.Accumulator) error {
 				}
 				tags[string(key)] = string(value)
 			}
-			acc.Add(string(sample.Metric[model.MetricNameLabel]), float64(sample.Value), tags)
+			acc.Add(string(sample.Metric[model.MetricNameLabel]),
+				float64(sample.Value), tags)
 		}
 	}
 

plugins/rabbitmq/rabbitmq.go

@@ -100,7 +100,7 @@ var gatherFunctions = []gatherFunc{gatherOverview, gatherNodes, gatherQueues}
 
 var sampleConfig = `
   # Specify servers via an array of tables
-  [[rabbitmq.servers]]
+  [[plugins.rabbitmq.servers]]
   # name = "rmq-server-1" # optional tag
   # url = "http://localhost:15672"
   # username = "guest"

plugins/statsd/README.md

@@ -26,6 +26,22 @@ implementation. In short, the telegraf statsd listener will accept:
     - `load.time.nanoseconds:1|h`
     - `load.time:200|ms|@0.1` <- sampled 1/10 of the time
 
+It is possible to omit repetitive names and merge individual stats into a
+single line by separating them with additional colons:
+
+  - `users.current.den001.myapp:32|g:+10|g:-10|g`
+  - `deploys.test.myservice:1|c:101|c:1|c|@0.1`
+  - `users.unique:101|s:101|s:102|s`
+  - `load.time:320|ms:200|ms|@0.1`
+
+This also allows for mixed types in a single line:
+
+  - `foo:1|c:200|ms`
+
+The string `foo:1|c:200|ms` is internally split into two individual metrics
+`foo:1|c` and `foo:200|ms` which are added to the aggregator separately.
+
+
 #### Influx Statsd
 
 In order to take advantage of InfluxDB's tagging system, we have made a couple

plugins/statsd/statsd.go

@@ -183,8 +183,6 @@ func (s *Statsd) Gather(acc plugins.Accumulator) error {
 }
 
 func (s *Statsd) Start() error {
-	log.Println("Starting up the statsd service")
-
 	// Make data structures
 	s.done = make(chan struct{})
 	s.in = make(chan string, s.AllowedPendingMessages)
@@ -197,6 +195,7 @@ func (s *Statsd) Start() error {
 	go s.udpListen()
 	// Start the line parser
 	go s.parser()
+	log.Printf("Started the statsd service on %s\n", s.ServiceAddress)
 	return nil
 }
 
@@ -255,101 +254,109 @@ func (s *Statsd) parseStatsdLine(line string) error {
 	s.Lock()
 	defer s.Unlock()
 
-	m := metric{}
-
-	// Validate splitting the line on "|"
-	pipesplit := strings.Split(line, "|")
-	if len(pipesplit) < 2 {
-		log.Printf("Error: splitting '|', Unable to parse metric: %s\n", line)
-		return errors.New("Error Parsing statsd line")
-	} else if len(pipesplit) > 2 {
-		sr := pipesplit[2]
-		errmsg := "Error: parsing sample rate, %s, it must be in format like: " +
-			"@0.1, @0.5, etc. Ignoring sample rate for line: %s\n"
-		if strings.Contains(sr, "@") && len(sr) > 1 {
-			samplerate, err := strconv.ParseFloat(sr[1:], 64)
-			if err != nil {
-				log.Printf(errmsg, err.Error(), line)
-			} else {
-				// sample rate successfully parsed
-				m.samplerate = samplerate
-			}
-		} else {
-			log.Printf(errmsg, "", line)
-		}
-	}
-
-	// Validate metric type
-	switch pipesplit[1] {
-	case "g", "c", "s", "ms", "h":
-		m.mtype = pipesplit[1]
-	default:
-		log.Printf("Error: Statsd Metric type %s unsupported", pipesplit[1])
-		return errors.New("Error Parsing statsd line")
-	}
-
-	// Validate splitting the rest of the line on ":"
-	colonsplit := strings.Split(pipesplit[0], ":")
-	if len(colonsplit) != 2 {
+	// Validate splitting the line on ":"
+	bits := strings.Split(line, ":")
+	if len(bits) < 2 {
 		log.Printf("Error: splitting ':', Unable to parse metric: %s\n", line)
 		return errors.New("Error Parsing statsd line")
 	}
-	m.bucket = colonsplit[0]
 
-	// Parse the value
-	if strings.ContainsAny(colonsplit[1], "-+") {
-		if m.mtype != "g" {
-			log.Printf("Error: +- values are only supported for gauges: %s\n", line)
+	// Extract bucket name from individual metric bits
+	bucketName, bits := bits[0], bits[1:]
+
+	// Add a metric for each bit available
+	for _, bit := range bits {
+		m := metric{}
+
+		m.bucket = bucketName
+
+		// Validate splitting the bit on "|"
+		pipesplit := strings.Split(bit, "|")
+		if len(pipesplit) < 2 {
+			log.Printf("Error: splitting '|', Unable to parse metric: %s\n", line)
+			return errors.New("Error Parsing statsd line")
+		} else if len(pipesplit) > 2 {
+			sr := pipesplit[2]
+			errmsg := "Error: parsing sample rate, %s, it must be in format like: " +
+				"@0.1, @0.5, etc. Ignoring sample rate for line: %s\n"
+			if strings.Contains(sr, "@") && len(sr) > 1 {
+				samplerate, err := strconv.ParseFloat(sr[1:], 64)
+				if err != nil {
+					log.Printf(errmsg, err.Error(), line)
+				} else {
+					// sample rate successfully parsed
+					m.samplerate = samplerate
+				}
+			} else {
+				log.Printf(errmsg, "", line)
+			}
+		}
+
+		// Validate metric type
+		switch pipesplit[1] {
+		case "g", "c", "s", "ms", "h":
+			m.mtype = pipesplit[1]
+		default:
+			log.Printf("Error: Statsd Metric type %s unsupported", pipesplit[1])
 			return errors.New("Error Parsing statsd line")
 		}
-		m.additive = true
-	}
 
-	switch m.mtype {
-	case "g", "ms", "h":
-		v, err := strconv.ParseFloat(colonsplit[1], 64)
-		if err != nil {
-			log.Printf("Error: parsing value to float64: %s\n", line)
-			return errors.New("Error Parsing statsd line")
+		// Parse the value
+		if strings.ContainsAny(pipesplit[0], "-+") {
+			if m.mtype != "g" {
+				log.Printf("Error: +- values are only supported for gauges: %s\n", line)
+				return errors.New("Error Parsing statsd line")
+			}
+			m.additive = true
 		}
-		m.floatvalue = v
-	case "c", "s":
-		v, err := strconv.ParseInt(colonsplit[1], 10, 64)
-		if err != nil {
-			log.Printf("Error: parsing value to int64: %s\n", line)
-			return errors.New("Error Parsing statsd line")
+
+		switch m.mtype {
+		case "g", "ms", "h":
+			v, err := strconv.ParseFloat(pipesplit[0], 64)
+			if err != nil {
+				log.Printf("Error: parsing value to float64: %s\n", line)
+				return errors.New("Error Parsing statsd line")
+			}
+			m.floatvalue = v
+		case "c", "s":
+			v, err := strconv.ParseInt(pipesplit[0], 10, 64)
+			if err != nil {
+				log.Printf("Error: parsing value to int64: %s\n", line)
+				return errors.New("Error Parsing statsd line")
+			}
+			// If a sample rate is given with a counter, divide value by the rate
+			if m.samplerate != 0 && m.mtype == "c" {
+				v = int64(float64(v) / m.samplerate)
+			}
+			m.intvalue = v
 		}
-		// If a sample rate is given with a counter, divide value by the rate
-		if m.samplerate != 0 && m.mtype == "c" {
-			v = int64(float64(v) / m.samplerate)
+
+		// Parse the name & tags from bucket
+		m.name, m.tags = s.parseName(m.bucket)
+		switch m.mtype {
+		case "c":
+			m.tags["metric_type"] = "counter"
+		case "g":
+			m.tags["metric_type"] = "gauge"
+		case "s":
+			m.tags["metric_type"] = "set"
+		case "ms":
+			m.tags["metric_type"] = "timing"
+		case "h":
+			m.tags["metric_type"] = "histogram"
 		}
-		m.intvalue = v
+
+		// Make a unique key for the measurement name/tags
+		var tg []string
+		for k, v := range m.tags {
+			tg = append(tg, fmt.Sprintf("%s=%s", k, v))
+		}
+		sort.Strings(tg)
+		m.hash = fmt.Sprintf("%s%s", strings.Join(tg, ""), m.name)
+
+		s.aggregate(m)
 	}
 
-	// Parse the name & tags from bucket
-	m.name, m.tags = s.parseName(m.bucket)
-	switch m.mtype {
-	case "c":
-		m.tags["metric_type"] = "counter"
-	case "g":
-		m.tags["metric_type"] = "gauge"
-	case "s":
-		m.tags["metric_type"] = "set"
-	case "ms":
-		m.tags["metric_type"] = "timing"
-	case "h":
-		m.tags["metric_type"] = "histogram"
-	}
-
-	// Make a unique key for the measurement name/tags
-	var tg []string
-	for k, v := range m.tags {
-		tg = append(tg, fmt.Sprintf("%s=%s", k, v))
-	}
-	sort.Strings(tg)
-	m.hash = fmt.Sprintf("%s%s", strings.Join(tg, ""), m.name)
-
-	s.aggregate(m)
 	return nil
 }
 

plugins/statsd/statsd_test.go

@@ -326,6 +326,136 @@ func TestParse_MeasurementsWithSameName(t *testing.T) {
 	}
 }
 
+// Test that measurements with multiple bits, are treated as different outputs
+// but are equal to their single-measurement representation
+func TestParse_MeasurementsWithMultipleValues(t *testing.T) {
+	single_lines := []string{
+		"valid.multiple:0|ms|@0.1",
+		"valid.multiple:0|ms|",
+		"valid.multiple:1|ms",
+		"valid.multiple.duplicate:1|c",
+		"valid.multiple.duplicate:1|c",
+		"valid.multiple.duplicate:2|c",
+		"valid.multiple.duplicate:1|c",
+		"valid.multiple.duplicate:1|h",
+		"valid.multiple.duplicate:1|h",
+		"valid.multiple.duplicate:2|h",
+		"valid.multiple.duplicate:1|h",
+		"valid.multiple.duplicate:1|s",
+		"valid.multiple.duplicate:1|s",
+		"valid.multiple.duplicate:2|s",
+		"valid.multiple.duplicate:1|s",
+		"valid.multiple.duplicate:1|g",
+		"valid.multiple.duplicate:1|g",
+		"valid.multiple.duplicate:2|g",
+		"valid.multiple.duplicate:1|g",
+		"valid.multiple.mixed:1|c",
+		"valid.multiple.mixed:1|ms",
+		"valid.multiple.mixed:2|s",
+		"valid.multiple.mixed:1|g",
+	}
+
+	multiple_lines := []string{
+		"valid.multiple:0|ms|@0.1:0|ms|:1|ms",
+		"valid.multiple.duplicate:1|c:1|c:2|c:1|c",
+		"valid.multiple.duplicate:1|h:1|h:2|h:1|h",
+		"valid.multiple.duplicate:1|s:1|s:2|s:1|s",
+		"valid.multiple.duplicate:1|g:1|g:2|g:1|g",
+		"valid.multiple.mixed:1|c:1|ms:2|s:1|g",
+	}
+
+	s_single := NewStatsd()
+	s_multiple := NewStatsd()
+
+	for _, line := range single_lines {
+		err := s_single.parseStatsdLine(line)
+		if err != nil {
+			t.Errorf("Parsing line %s should not have resulted in an error\n", line)
+		}
+	}
+
+	for _, line := range multiple_lines {
+		err := s_multiple.parseStatsdLine(line)
+		if err != nil {
+			t.Errorf("Parsing line %s should not have resulted in an error\n", line)
+		}
+	}
+
+	if len(s_single.timings) != 3 {
+		t.Errorf("Expected 3 measurement, found %d", len(s_single.timings))
+	}
+
+	if cachedtiming, ok := s_single.timings["metric_type=timingvalid_multiple"]; !ok {
+		t.Errorf("Expected cached measurement with hash 'metric_type=timingvalid_multiple' not found")
+	} else {
+		if cachedtiming.name != "valid_multiple" {
+			t.Errorf("Expected the name to be 'valid_multiple', got %s", cachedtiming.name)
+		}
+
+		// A 0 at samplerate 0.1 will add 10 values of 0,
+		// A 0 with invalid samplerate will add a single 0,
+		// plus the last bit of value 1
+		// which adds up to 12 individual datapoints to be cached
+		if cachedtiming.stats.n != 12 {
+			t.Errorf("Expected 11 additions, got %d", cachedtiming.stats.n)
+		}
+
+		if cachedtiming.stats.upper != 1 {
+			t.Errorf("Expected max input to be 1, got %f", cachedtiming.stats.upper)
+		}
+	}
+
+	// test if s_single and s_multiple did compute the same stats for valid.multiple.duplicate
+	if err := test_validate_set("valid_multiple_duplicate", 2, s_single.sets); err != nil {
+		t.Error(err.Error())
+	}
+
+	if err := test_validate_set("valid_multiple_duplicate", 2, s_multiple.sets); err != nil {
+		t.Error(err.Error())
+	}
+
+	if err := test_validate_counter("valid_multiple_duplicate", 5, s_single.counters); err != nil {
+		t.Error(err.Error())
+	}
+
+	if err := test_validate_counter("valid_multiple_duplicate", 5, s_multiple.counters); err != nil {
+		t.Error(err.Error())
+	}
+
+	if err := test_validate_gauge("valid_multiple_duplicate", 1, s_single.gauges); err != nil {
+		t.Error(err.Error())
+	}
+
+	if err := test_validate_gauge("valid_multiple_duplicate", 1, s_multiple.gauges); err != nil {
+		t.Error(err.Error())
+	}
+
+	// test if s_single and s_multiple did compute the same stats for valid.multiple.mixed
+	if err := test_validate_set("valid_multiple_mixed", 1, s_single.sets); err != nil {
+		t.Error(err.Error())
+	}
+
+	if err := test_validate_set("valid_multiple_mixed", 1, s_multiple.sets); err != nil {
+		t.Error(err.Error())
+	}
+
+	if err := test_validate_counter("valid_multiple_mixed", 1, s_single.counters); err != nil {
+		t.Error(err.Error())
+	}
+
+	if err := test_validate_counter("valid_multiple_mixed", 1, s_multiple.counters); err != nil {
+		t.Error(err.Error())
+	}
+
+	if err := test_validate_gauge("valid_multiple_mixed", 1, s_single.gauges); err != nil {
+		t.Error(err.Error())
+	}
+
+	if err := test_validate_gauge("valid_multiple_mixed", 1, s_multiple.gauges); err != nil {
+		t.Error(err.Error())
+	}
+}
+
 // Valid lines should be parsed and their values should be cached
 func TestParse_ValidLines(t *testing.T) {
 	s := NewStatsd()

plugins/twemproxy/twemproxy.go

@@ -22,7 +22,7 @@ type TwemproxyInstance struct {
 }
 
 var sampleConfig = `
-  [[twemproxy.instances]]
+  [[plugins.twemproxy.instances]]
     # Twemproxy stats address and port (no scheme)
     addr = "localhost:22222"
     # Monitor pool name

scripts/package.sh

@@ -37,7 +37,7 @@ CONFIG_ROOT_DIR=/etc/opt/telegraf
 CONFIG_D_DIR=/etc/opt/telegraf/telegraf.d
 LOGROTATE_DIR=/etc/logrotate.d
 
-SAMPLE_CONFIGURATION=etc/config.sample.toml
+SAMPLE_CONFIGURATION=etc/telegraf.conf
 LOGROTATE_CONFIGURATION=etc/logrotate.d/telegraf
 INITD_SCRIPT=scripts/init.sh
 SYSTEMD_SCRIPT=scripts/telegraf.service

testdata/influx.toml

@@ -1,17 +0,0 @@
-[agent]
-interval = "5s"
-http = ":11213"
-debug = true
-
-[outputs]
-[outputs.influxdb]
-url = "http://localhost:8086"
-username = "root"
-password = "root"
-database = "telegraf"
-
-[tags]
-dc = "us-phx-1"
-
-[redis]
-address = ":6379"

testdata/single_plugin.toml

@@ -1,12 +0,0 @@
-[kafka]
-  topic = "topic_with_metrics"
-  consumerGroupName = "telegraf_metrics_consumers"
-  zookeeperPeers = ["test.example.com:2181"]
-  batchSize = 1000
-  pass = ["some", "strings"]
-  drop = ["other", "stuff"]
-  interval = "5s"
-  [kafka.tagpass]
-    goodtag = ["mytag"]
-  [kafka.tagdrop]
-    badtag = ["othertag"]

testdata/subconfig/procstat.conf

@@ -1,5 +0,0 @@
-[procstat]
-  [[procstat.specifications]]
-  pid_file = "/var/run/grafana-server.pid"
-  [[procstat.specifications]]
-  pid_file = "/var/run/influxdb/influxd.pid"

testdata/telegraf-agent.toml

@@ -1,333 +0,0 @@
-# Telegraf configuration
-
-# Telegraf is entirely plugin driven. All metrics are gathered from the
-# declared plugins.
-
-# Even if a plugin has no configuration, it must be declared in here
-# to be active. Declaring a plugin means just specifying the name
-# as a section with no variables. To deactivate a plugin, comment
-# out the name and any variables.
-
-# Use 'telegraf -config telegraf.toml -test' to see what metrics a config
-# file would generate.
-
-# One rule that plugins conform to is wherever a connection string
-# can be passed, the values '' and 'localhost' are treated specially.
-# They indicate to the plugin to use their own builtin configuration to
-# connect to the local system.
-
-# NOTE: The configuration has a few required parameters. They are marked
-# with 'required'. Be sure to edit those to make this configuration work.
-
-# Tags can also be specified via a normal map, but only one form at a time:
-[tags]
-	# dc = "us-east-1"
-
-# Configuration for telegraf agent
-[agent]
-	# Default data collection interval for all plugins
-	interval = "10s"
-
-	# If utc = false, uses local time (utc is highly recommended)
-	utc = true
-
-	# Precision of writes, valid values are n, u, ms, s, m, and h
-	# note: using second precision greatly helps InfluxDB compression
-	precision = "s"
-
-	# run telegraf in debug mode
-	debug = false
-
-	# Override default hostname, if empty use os.Hostname()
-	hostname = ""
-
-
-###############################################################################
-#                                  OUTPUTS                                    #
-###############################################################################
-
-[outputs]
-
-# Configuration for influxdb server to send metrics to
-[[outputs.influxdb]]
-	# The full HTTP endpoint URL for your InfluxDB instance
-	# Multiple urls can be specified for InfluxDB cluster support. Server to
-	# write to will be randomly chosen each interval.
-	urls = ["http://localhost:8086"] # required.
-
-	# The target database for metrics. This database must already exist
-	database = "telegraf" # required.
-
-	# Connection timeout (for the connection with InfluxDB), formatted as a string.
-	# Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h".
-	# If not provided, will default to 0 (no timeout)
-	# timeout = "5s"
-
-	# username = "telegraf"
-	# password = "metricsmetricsmetricsmetrics"
-
-	# Set the user agent for the POSTs (can be useful for log differentiation)
-	# user_agent = "telegraf"
-
-[[outputs.influxdb]]
-  urls = ["udp://localhost:8089"]
-  database = "udp-telegraf"
-
-# Configuration for the Kafka server to send metrics to
-[[outputs.kafka]]
-	# URLs of kafka brokers
-	brokers = ["localhost:9092"]
-	# Kafka topic for producer messages
-	topic = "telegraf"
-	# Telegraf tag to use as a routing key
-	#  ie, if this tag exists, it's value will be used as the routing key
-	routing_tag = "host"
-
-
-###############################################################################
-#                                  PLUGINS                                    #
-###############################################################################
-
-# Read Apache status information (mod_status)
-[apache]
-# An array of Apache status URI to gather stats.
-urls = ["http://localhost/server-status?auto"]
-
-# Read metrics about cpu usage
-[cpu]
-	# Whether to report per-cpu stats or not
-	percpu = true
-	# Whether to report total system cpu stats or not
-	totalcpu = true
-	# Comment this line if you want the raw CPU time metrics
-	drop = ["cpu_time"]
-
-# Read metrics about disk usage by mount point
-[disk]
-	# no configuration
-
-# Read metrics from one or many disque servers
-[disque]
-	# An array of URI to gather stats about. Specify an ip or hostname
-	# with optional port and password. ie disque://localhost, disque://10.10.3.33:18832,
-	# 10.0.0.1:10000, etc.
-	#
-	# If no servers are specified, then localhost is used as the host.
-	servers = ["localhost"]
-
-# Read stats from one or more Elasticsearch servers or clusters
-[elasticsearch]
-	# specify a list of one or more Elasticsearch servers
-	servers = ["http://localhost:9200"]
-
-	# set local to false when you want to read the indices stats from all nodes
-	# within the cluster
-	local = true
-
-# Read flattened metrics from one or more commands that output JSON to stdout
-[exec]
-	# specify commands via an array of tables
-	[[exec.commands]]
-	# the command to run
-	command = "/usr/bin/mycollector --foo=bar"
-
-	# name of the command (used as a prefix for measurements)
-	name = "mycollector"
-
-# Read metrics of haproxy, via socket or csv stats page
-[haproxy]
-	# An array of address to gather stats about. Specify an ip on hostname
-	# with optional port. ie localhost, 10.10.3.33:1936, etc.
-	#
-	# If no servers are specified, then default to 127.0.0.1:1936
-	servers = ["http://myhaproxy.com:1936", "http://anotherhaproxy.com:1936"]
-	# Or you can also use local socket(not work yet)
-	# servers = ["socket:/run/haproxy/admin.sock"]
-
-# Read flattened metrics from one or more JSON HTTP endpoints
-[httpjson]
-	# Specify services via an array of tables
-	[[httpjson.services]]
-
-		# a name for the service being polled
-		name = "webserver_stats"
-
-		# URL of each server in the service's cluster
-		servers = [
-			"http://localhost:9999/stats/",
-			"http://localhost:9998/stats/",
-		]
-
-		# HTTP method to use (case-sensitive)
-		method = "GET"
-
-		# HTTP parameters (all values must be strings)
-		[httpjson.services.parameters]
-			event_type = "cpu_spike"
-			threshold = "0.75"
-
-# Read metrics about disk IO by device
-[io]
-	# no configuration
-
-# read metrics from a Kafka topic
-[kafka]
-	# topic to consume
-	topic = "topic_with_metrics"
-
-	# the name of the consumer group
-	consumerGroupName = "telegraf_metrics_consumers"
-
-	# an array of Zookeeper connection strings
-	zookeeperPeers = ["localhost:2181"]
-
-	# Batch size of points sent to InfluxDB
-	batchSize = 1000
-
-# Read metrics from a LeoFS Server via SNMP
-[leofs]
-	# An array of URI to gather stats about LeoFS.
-	# Specify an ip or hostname with port. ie 127.0.0.1:4020
-	#
-	# If no servers are specified, then 127.0.0.1 is used as the host and 4020 as the port.
-	servers = ["127.0.0.1:4021"]
-
-# Read metrics from local Lustre service on OST, MDS
-[lustre2]
-	# An array of /proc globs to search for Lustre stats
-	# If not specified, the default will work on Lustre 2.5.x
-	#
-	# ost_procfiles = ["/proc/fs/lustre/obdfilter/*/stats", "/proc/fs/lustre/osd-ldiskfs/*/stats"]
-	# mds_procfiles = ["/proc/fs/lustre/mdt/*/md_stats"]
-
-# Read metrics about memory usage
-[mem]
-	# no configuration
-
-# Read metrics from one or many memcached servers
-[memcached]
-	# An array of address to gather stats about. Specify an ip on hostname
-	# with optional port. ie localhost, 10.0.0.1:11211, etc.
-	#
-	# If no servers are specified, then localhost is used as the host.
-	servers = ["localhost"]
-
-# Read metrics from one or many MongoDB servers
-[mongodb]
-	# An array of URI to gather stats about. Specify an ip or hostname
-	# with optional port add password. ie mongodb://user:auth_key@10.10.3.30:27017,
-	# mongodb://10.10.3.33:18832, 10.0.0.1:10000, etc.
-	#
-	# If no servers are specified, then 127.0.0.1 is used as the host and 27107 as the port.
-	servers = ["127.0.0.1:27017"]
-
-# Read metrics from one or many mysql servers
-[mysql]
-	# specify servers via a url matching:
-	#  [username[:password]@][protocol[(address)]]/[?tls=[true|false|skip-verify]]
-	#  e.g.
-	#    servers = ["root:root@http://10.0.0.18/?tls=false"]
-	#    servers = ["root:passwd@tcp(127.0.0.1:3306)/"]
-	#
-	# If no servers are specified, then localhost is used as the host.
-	servers = ["localhost"]
-
-# Read metrics about network interface usage
-[net]
-	# By default, telegraf gathers stats from any up interface (excluding loopback)
-	# Setting interfaces will tell it to gather these explicit interfaces,
-	# regardless of status.
-	#
-	# interfaces = ["eth0", ... ]
-
-# Read Nginx's basic status information (ngx_http_stub_status_module)
-[nginx]
-	# An array of Nginx stub_status URI to gather stats.
-	urls = ["http://localhost/status"]
-
-# Ping given url(s) and return statistics
-[ping]
-	# urls to ping
-	urls = ["www.google.com"] # required
-	# number of pings to send (ping -c <COUNT>)
-	count = 1 # required
-	# interval, in s, at which to ping. 0 == default (ping -i <PING_INTERVAL>)
-	ping_interval = 0.0
-	# ping timeout, in s. 0 == no timeout (ping -t <TIMEOUT>)
-	timeout = 0.0
-	# interface to send ping from (ping -I <INTERFACE>)
-	interface = ""
-
-# Read metrics from one or many postgresql servers
-[postgresql]
-	# specify servers via an array of tables
-	[[postgresql.servers]]
-
-	# specify address via a url matching:
-	#   postgres://[pqgotest[:password]]@localhost[/dbname]?sslmode=[disable|verify-ca|verify-full]
-	# or a simple string:
-	#   host=localhost user=pqotest password=... sslmode=... dbname=app_production
-	#
-	# All connection parameters are optional. By default, the host is localhost
-	# and the user is the currently running user. For localhost, we default
-	# to sslmode=disable as well.
-	#
-	# Without the dbname parameter, the driver will default to a database
-	# with the same name as the user. This dbname is just for instantiating a
-	# connection with the server and doesn't restrict the databases we are trying
-	# to grab metrics for.
-	#
-
-	address = "sslmode=disable"
-
-	# A list of databases to pull metrics about. If not specified, metrics for all
-	# databases are gathered.
-
-	# databases = ["app_production", "blah_testing"]
-
-	# [[postgresql.servers]]
-	# address = "influx@remoteserver"
-
-# Read metrics from one or many prometheus clients
-[prometheus]
-	# An array of urls to scrape metrics from.
-	urls = ["http://localhost:9100/metrics"]
-
-# Read metrics from one or many RabbitMQ servers via the management API
-[rabbitmq]
-	# Specify servers via an array of tables
-	[[rabbitmq.servers]]
-	# name = "rmq-server-1" # optional tag
-	# url = "http://localhost:15672"
-	# username = "guest"
-	# password = "guest"
-
-	# A list of nodes to pull metrics about. If not specified, metrics for
-	# all nodes are gathered.
-	# nodes = ["rabbit@node1", "rabbit@node2"]
-
-# Read metrics from one or many redis servers
-[redis]
-	# An array of URI to gather stats about. Specify an ip or hostname
-	# with optional port add password. ie redis://localhost, redis://10.10.3.33:18832,
-	# 10.0.0.1:10000, etc.
-	#
-	# If no servers are specified, then localhost is used as the host.
-	servers = ["localhost"]
-
-# Read metrics from one or many RethinkDB servers
-[rethinkdb]
-	# An array of URI to gather stats about. Specify an ip or hostname
-	# with optional port add password. ie rethinkdb://user:auth_key@10.10.3.30:28105,
-	# rethinkdb://10.10.3.33:18832, 10.0.0.1:10000, etc.
-	#
-	# If no servers are specified, then 127.0.0.1 is used as the host and 28015 as the port.
-	servers = ["127.0.0.1:28015"]
-
-# Read metrics about swap memory usage
-[swap]
-	# no configuration
-
-# Read metrics about system load & uptime
-[system]
-	# no configuration