PR #59, implementation of multiple outputs

This commit is contained in:
Cameron Sparr 2015-08-11 10:21:00 -06:00
commit 48a075529a
9 changed files with 220 additions and 75 deletions

1
.gitignore vendored
View File

@ -1,3 +1,4 @@
pkg/ pkg/
tivan tivan
.vagrant .vagrant
telegraf

View File

@ -10,10 +10,15 @@ import (
"sync" "sync"
"time" "time"
"github.com/influxdb/influxdb/client" "github.com/influxdb/telegraf/outputs"
"github.com/influxdb/telegraf/plugins" "github.com/influxdb/telegraf/plugins"
) )
type runningOutput struct {
name string
output outputs.Output
}
type runningPlugin struct { type runningPlugin struct {
name string name string
plugin plugins.Plugin plugin plugins.Plugin
@ -32,9 +37,8 @@ type Agent struct {
Config *Config Config *Config
outputs []*runningOutput
plugins []*runningPlugin plugins []*runningPlugin
conn *client.Client
} }
// NewAgent returns an Agent struct based off the given Config // NewAgent returns an Agent struct based off the given Config
@ -66,25 +70,36 @@ func NewAgent(config *Config) (*Agent, error) {
// Connect connects to the agent's config URL // Connect connects to the agent's config URL
func (a *Agent) Connect() error { func (a *Agent) Connect() error {
config := a.Config for _, o := range a.outputs {
err := o.output.Connect(a.Hostname)
u, err := url.Parse(config.URL)
if err != nil { if err != nil {
return err return err
} }
}
c, err := client.NewClient(client.Config{ return nil
URL: *u,
Username: config.Username,
Password: config.Password,
UserAgent: config.UserAgent,
Timeout: config.Timeout.Duration,
})
if err != nil {
return err
} }
func (a *Agent) LoadOutputs() ([]string, error) {
var names []string
for _, name := range a.Config.OutputsDeclared() {
creator, ok := outputs.Outputs[name]
if !ok {
return nil, fmt.Errorf("Undefined but requested output: %s", name)
}
output := creator()
err := a.Config.ApplyOutput(name, output)
if err != nil {
return nil, err
}
a.outputs = append(a.outputs, &runningOutput{name, output})
names = append(names, name)
}
<<<<<<< HEAD
_, err = c.Query(client.Query{ _, err = c.Query(client.Query{
Command: fmt.Sprintf("CREATE DATABASE telegraf"), Command: fmt.Sprintf("CREATE DATABASE telegraf"),
}) })
@ -94,8 +109,11 @@ func (a *Agent) Connect() error {
} }
a.conn = c a.conn = c
=======
sort.Strings(names)
>>>>>>> jipperinbham-outputs-phase1
return nil return names, nil
} }
// LoadPlugins loads the agent's plugins // LoadPlugins loads the agent's plugins
@ -153,17 +171,14 @@ func (a *Agent) crankParallel() error {
close(points) close(points)
var acc BatchPoints var bp BatchPoints
acc.Tags = a.Config.Tags bp.Time = time.Now()
acc.Time = time.Now()
acc.Database = a.Config.Database
for sub := range points { for sub := range points {
acc.Points = append(acc.Points, sub.Points...) bp.Points = append(bp.Points, sub.Points...)
} }
_, err := a.conn.Write(acc.BatchPoints) return a.flush(&bp)
return err
} }
func (a *Agent) crank() error { func (a *Agent) crank() error {
@ -180,12 +195,9 @@ func (a *Agent) crank() error {
} }
} }
acc.Tags = a.Config.Tags
acc.Time = time.Now() acc.Time = time.Now()
acc.Database = a.Config.Database
_, err := a.conn.Write(acc.BatchPoints) return a.flush(&acc)
return err
} }
func (a *Agent) crankSeparate(shutdown chan struct{}, plugin *runningPlugin) error { func (a *Agent) crankSeparate(shutdown chan struct{}, plugin *runningPlugin) error {
@ -207,7 +219,10 @@ func (a *Agent) crankSeparate(shutdown chan struct{}, plugin *runningPlugin) err
acc.Time = time.Now() acc.Time = time.Now()
acc.Database = a.Config.Database acc.Database = a.Config.Database
a.conn.Write(acc.BatchPoints) err = a.flush(&acc)
if err != nil {
return err
}
select { select {
case <-shutdown: case <-shutdown:
@ -218,6 +233,22 @@ func (a *Agent) crankSeparate(shutdown chan struct{}, plugin *runningPlugin) err
} }
} }
func (a *Agent) flush(bp *BatchPoints) error {
var wg sync.WaitGroup
var outerr error
for _, o := range a.outputs {
wg.Add(1)
go func(ro *runningOutput) {
defer wg.Done()
outerr = ro.output.Write(bp.BatchPoints)
}(o)
}
wg.Wait()
return outerr
}
// TestAllPlugins verifies that we can 'Gather' from all plugins with the // TestAllPlugins verifies that we can 'Gather' from all plugins with the
// default configuration // default configuration
func (a *Agent) TestAllPlugins() error { func (a *Agent) TestAllPlugins() error {
@ -276,13 +307,6 @@ func (a *Agent) Test() error {
// Run runs the agent daemon, gathering every Interval // Run runs the agent daemon, gathering every Interval
func (a *Agent) Run(shutdown chan struct{}) error { func (a *Agent) Run(shutdown chan struct{}) error {
if a.conn == nil {
err := a.Connect()
if err != nil {
return err
}
}
var wg sync.WaitGroup var wg sync.WaitGroup
for _, plugin := range a.plugins { for _, plugin := range a.plugins {

View File

@ -9,6 +9,7 @@ import (
"strings" "strings"
"github.com/influxdb/telegraf" "github.com/influxdb/telegraf"
_ "github.com/influxdb/telegraf/outputs/all"
_ "github.com/influxdb/telegraf/plugins/all" _ "github.com/influxdb/telegraf/plugins/all"
) )
@ -61,6 +62,11 @@ func main() {
ag.Debug = true ag.Debug = true
} }
outputs, err := ag.LoadOutputs()
if err != nil {
log.Fatal(err)
}
plugins, err := ag.LoadPlugins() plugins, err := ag.LoadPlugins()
if err != nil { if err != nil {
log.Fatal(err) log.Fatal(err)
@ -100,7 +106,8 @@ func main() {
close(shutdown) close(shutdown)
}() }()
log.Print("InfluxDB Agent running") log.Print("Telegraf Agent running")
log.Printf("Loaded outputs: %s", strings.Join(outputs, " "))
log.Printf("Loaded plugins: %s", strings.Join(plugins, " ")) log.Printf("Loaded plugins: %s", strings.Join(plugins, " "))
if ag.Debug { if ag.Debug {
log.Printf("Debug: enabled") log.Printf("Debug: enabled")
@ -108,11 +115,6 @@ func main() {
ag.Interval, ag.Debug, ag.Hostname) ag.Interval, ag.Debug, ag.Hostname)
} }
if config.URL != "" {
log.Printf("Sending metrics to: %s", config.URL)
log.Printf("Tags enabled: %v", config.ListTags())
}
if *fPidfile != "" { if *fPidfile != "" {
f, err := os.Create(*fPidfile) f, err := os.Create(*fPidfile)
if err != nil { if err != nil {

View File

@ -30,26 +30,28 @@ func (d *Duration) UnmarshalTOML(b []byte) error {
return nil return nil
} }
// Config specifies the URL/user/password for the database that telegraf // Config specifies the outputs that telegraf
// will be logging to, as well as all the plugins that the user has // will be logging to, as well as all the plugins that the user has
// specified // specified
type Config struct { type Config struct {
URL string
Username string
Password string
Database string
UserAgent string
Timeout Duration
Tags map[string]string
agent *ast.Table agent *ast.Table
plugins map[string]*ast.Table plugins map[string]*ast.Table
outputs map[string]*ast.Table
} }
// Plugins returns the configured plugins as a map of name -> plugin toml // Plugins returns the configured plugins as a map of name -> plugin toml
func (c *Config) Plugins() map[string]*ast.Table { func (c *Config) Plugins() map[string]*ast.Table {
return c.plugins return c.plugins
} }
type TagFilter struct {
Name string
Filter []string
}
// Outputs returns the configured outputs as a map of name -> output toml
func (c *Config) Outputs() map[string]*ast.Table {
return c.outputs
}
// The name of a tag, and the values on which to filter // The name of a tag, and the values on which to filter
type TagFilter struct { type TagFilter struct {
@ -64,6 +66,9 @@ type ConfiguredPlugin struct {
Drop []string Drop []string
Pass []string Pass []string
TagDrop []TagFilter
TagPass []TagFilter
TagDrop []TagFilter TagDrop []TagFilter
TagPass []TagFilter TagPass []TagFilter
@ -106,6 +111,10 @@ func (cp *ConfiguredPlugin) ShouldPass(measurement string, tags map[string]strin
return false return false
} }
<<<<<<< HEAD
=======
>>>>>>> jipperinbham-outputs-phase1
if cp.TagDrop != nil { if cp.TagDrop != nil {
for _, pat := range cp.TagDrop { for _, pat := range cp.TagDrop {
if tagval, ok := tags[pat.Name]; ok { if tagval, ok := tags[pat.Name]; ok {
@ -119,7 +128,18 @@ func (cp *ConfiguredPlugin) ShouldPass(measurement string, tags map[string]strin
return true return true
} }
<<<<<<< HEAD
return true return true
=======
return nil
}
// ApplyOutput loads the toml config into the given interface
func (c *Config) ApplyOutput(name string, v interface{}) error {
if c.outputs[name] != nil {
return toml.UnmarshalTable(c.outputs[name], v)
}
>>>>>>> jipperinbham-outputs-phase1
} }
// ApplyAgent loads the toml config into the given interface // ApplyAgent loads the toml config into the given interface
@ -225,15 +245,24 @@ func (c *Config) ApplyPlugin(name string, v interface{}) (*ConfiguredPlugin, err
// PluginsDeclared returns the name of all plugins declared in the config. // PluginsDeclared returns the name of all plugins declared in the config.
func (c *Config) PluginsDeclared() []string { func (c *Config) PluginsDeclared() []string {
var plugins []string return declared(c.plugins)
for name := range c.plugins {
plugins = append(plugins, name)
} }
sort.Strings(plugins) // OutputsDeclared returns the name of all outputs declared in the config.
func (c *Config) OutputsDeclared() []string {
return declared(c.outputs)
}
return plugins func declared(endpoints map[string]*ast.Table) []string {
var names []string
for name, _ := range endpoints {
names = append(names, name)
}
sort.Strings(names)
return names
} }
// DefaultConfig returns an empty default configuration // DefaultConfig returns an empty default configuration
@ -257,6 +286,7 @@ func LoadConfig(path string) (*Config, error) {
c := &Config{ c := &Config{
plugins: make(map[string]*ast.Table), plugins: make(map[string]*ast.Table),
outputs: make(map[string]*ast.Table),
} }
for name, val := range tbl.Fields { for name, val := range tbl.Fields {
@ -266,13 +296,16 @@ func LoadConfig(path string) (*Config, error) {
} }
switch name { switch name {
case "influxdb":
err := toml.UnmarshalTable(subtbl, c)
if err != nil {
return nil, err
}
case "agent": case "agent":
c.agent = subtbl c.agent = subtbl
case "outputs":
for outputName, outputVal := range subtbl.Fields {
outputSubtbl, ok := outputVal.(*ast.Table)
if !ok {
return nil, errInvalidConfig
}
c.outputs[outputName] = outputSubtbl
}
default: default:
c.plugins[name] = subtbl c.plugins[name] = subtbl
} }
@ -327,8 +360,11 @@ var header = `# Telegraf configuration
# NOTE: The configuration has a few required parameters. They are marked # NOTE: The configuration has a few required parameters. They are marked
# with 'required'. Be sure to edit those to make this configuration work. # with 'required'. Be sure to edit those to make this configuration work.
# OUTPUTS
[outputs]
# Configuration for influxdb server to send metrics to # Configuration for influxdb server to send metrics to
[influxdb] [outputs.influxdb]
# The full HTTP endpoint URL for your InfluxDB instance # The full HTTP endpoint URL for your InfluxDB instance
url = "http://localhost:8086" # required. url = "http://localhost:8086" # required.
@ -345,12 +381,11 @@ database = "telegraf" # required.
# Set the user agent for the POSTs (can be useful for log differentiation) # Set the user agent for the POSTs (can be useful for log differentiation)
# user_agent = "telegraf" # user_agent = "telegraf"
# tags = { "dc": "us-east-1" }
# Tags can also be specified via a normal map, but only one form at a time: # Tags can also be specified via a normal map, but only one form at a time:
# [influxdb.tags] # [influxdb.tags]
# dc = "us-east-1" # tags = { "dc" = "us-east-1" }
# Configuration for telegraf itself # Configuration for telegraf itself
# [agent] # [agent]

View File

@ -22,8 +22,11 @@
# NOTE: The configuration has a few required parameters. They are marked # NOTE: The configuration has a few required parameters. They are marked
# with 'required'. Be sure to edit those to make this configuration work. # with 'required'. Be sure to edit those to make this configuration work.
# OUTPUTS
[outputs]
# Configuration for influxdb server to send metrics to # Configuration for influxdb server to send metrics to
[influxdb] [outputs.influxdb]
# The full HTTP endpoint URL for your InfluxDB instance # The full HTTP endpoint URL for your InfluxDB instance
url = "http://localhost:8086" # required. url = "http://localhost:8086" # required.
@ -35,12 +38,8 @@ database = "telegraf" # required.
# Set the user agent for the POSTs (can be useful for log differentiation) # Set the user agent for the POSTs (can be useful for log differentiation)
# user_agent = "telegraf" # user_agent = "telegraf"
# tags = { "dc": "us-east-1" }
# Tags can also be specified via a normal map, but only one form at a time: # tags = { "dc" = "us-east-1" }
# [influxdb.tags]
# dc = "us-east-1"
# Configuration for telegraf itself # Configuration for telegraf itself
# [agent] # [agent]

5
outputs/all/all.go Normal file
View File

@ -0,0 +1,5 @@
package all
import (
_ "github.com/influxdb/telegraf/outputs/influxdb"
)

View File

@ -0,0 +1,60 @@
package influxdb
import (
"net/url"
"github.com/influxdb/influxdb/client"
"github.com/influxdb/telegraf/outputs"
)
type InfluxDB struct {
URL string
Username string
Password string
Database string
UserAgent string
Tags map[string]string
conn *client.Client
}
func (i *InfluxDB) Connect(host string) error {
u, err := url.Parse(i.URL)
if err != nil {
return err
}
c, err := client.NewClient(client.Config{
URL: *u,
Username: i.Username,
Password: i.Password,
UserAgent: i.UserAgent,
})
if err != nil {
return err
}
if i.Tags == nil {
i.Tags = make(map[string]string)
}
i.Tags["host"] = host
i.conn = c
return nil
}
func (i *InfluxDB) Write(bp client.BatchPoints) error {
bp.Database = i.Database
bp.Tags = i.Tags
if _, err := i.conn.Write(bp); err != nil {
return err
}
return nil
}
func init() {
outputs.Add("influxdb", func() outputs.Output {
return &InfluxDB{}
})
}

18
outputs/registry.go Normal file
View File

@ -0,0 +1,18 @@
package outputs
import (
"github.com/influxdb/influxdb/client"
)
type Output interface {
Connect(string) error
Write(client.BatchPoints) error
}
type Creator func() Output
var Outputs = map[string]Creator{}
func Add(name string, creator Creator) {
Outputs[name] = creator
}

View File

@ -3,12 +3,13 @@ interval = "5s"
http = ":11213" http = ":11213"
debug = true debug = true
[influxdb] [outputs]
[outputs.influxdb]
url = "http://localhost:8086" url = "http://localhost:8086"
username = "root" username = "root"
password = "root" password = "root"
database = "telegraf" database = "telegraf"
tags = { dc = "us-phx-1" } tags = { "dc" = "us-phx-1" }
[redis] [redis]
address = ":6379" address = ":6379"