dom
/
telegraf
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Revert much of the newer config file parsing, fix tagdrop/tagpass

This commit is contained in:
Cameron Sparr 2015-11-23 18:00:54 -07:00
parent 224a570a08
commit 8dde60e869
4 changed files with 169 additions and 294 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
accumulator.go
View File

@ -105,7 +105,7 @@ func (ac *accumulator) AddFields(
}
if ac.plugin != nil {
if !ac.plugin.ShouldPass(measurement, tags) {
if !ac.plugin.ShouldPass(measurement) || !ac.plugin.ShouldTagsPass(tags) {
return
}
}

44
agent.go
View File

@ -66,6 +66,8 @@ type Agent struct {
Tags map[string]string
Config *Config
outputs []*runningOutput
plugins []*runningPlugin
}
@ -74,6 +76,7 @@ type Agent struct {
func NewAgent(config *Config) (*Agent, error) {
agent := &Agent{
Tags: make(map[string]string),
Config: config,
Interval: internal.Duration{10 * time.Second},
RoundInterval: true,
FlushInterval: internal.Duration{10 * time.Second},
@ -96,7 +99,11 @@ func NewAgent(config *Config) (*Agent, error) {
agent.Hostname = hostname
}
agent.Tags["host"] = agent.Hostname
if config.Tags == nil {
config.Tags = map[string]string{}
}
config.Tags["host"] = agent.Hostname
return agent, nil
}
@ -146,18 +153,21 @@ func (a *Agent) Close() error {
}
// LoadOutputs loads the agent's outputs
func (a *Agent) LoadOutputs(filters []string, config *Config) ([]string, error) {
func (a *Agent) LoadOutputs(filters []string) ([]string, error) {
var names []string
for name, output := range config.OutputsDeclared() {
for _, name := range a.Config.OutputsDeclared() {
// Trim the ID off the output name for filtering
filtername := strings.TrimRight(name, "-0123456789")
if sliceContains(filtername, filters) || len(filters) == 0 {
if a.Debug {
log.Println("Output Enabled: ", name)
creator, ok := outputs.Outputs[filtername]
if !ok {
return nil, fmt.Errorf("Undefined but requested output: %s", name)
}
err := config.ApplyOutput(name, output)
if sliceContains(filtername, filters) || len(filters) == 0 {
output := creator()
err := a.Config.ApplyOutput(name, output)
if err != nil {
return nil, err
}
@ -173,15 +183,27 @@ func (a *Agent) LoadOutputs(filters []string, config *Config) ([]string, error)
}
// LoadPlugins loads the agent's plugins
func (a *Agent) LoadPlugins(filters []string, config *Config) ([]string, error) {
func (a *Agent) LoadPlugins(filters []string) ([]string, error) {
var names []string
for name, plugin := range config.PluginsDeclared() {
for _, name := range a.Config.PluginsDeclared() {
// Trim the ID off the output name for filtering
filtername := strings.TrimRight(name, "-0123456789")
creator, ok := plugins.Plugins[filtername]
if !ok {
return nil, fmt.Errorf("Undefined but requested plugin: %s", name)
}
if sliceContains(filtername, filters) || len(filters) == 0 {
config := config.GetPluginConfig(name)
a.plugins = append(a.plugins, &runningPlugin{name, filtername, plugin, config})
plugin := creator()
config, err := a.Config.ApplyPlugin(name, plugin)
if err != nil {
return nil, err
}
a.plugins = append(a.plugins,
&runningPlugin{name, filtername, plugin, config})
names = append(names, name)
}
}

4
cmd/telegraf/telegraf.go
View File

@ -102,7 +102,7 @@ func main() {
ag.Debug = true
}
outputs, err := ag.LoadOutputs(outputFilters, config)
outputs, err := ag.LoadOutputs(outputFilters)
if err != nil {
log.Fatal(err)
}
@ -111,7 +111,7 @@ func main() {
os.Exit(1)
}
plugins, err := ag.LoadPlugins(pluginFilters, config)
plugins, err := ag.LoadPlugins(pluginFilters)
if err != nil {
log.Fatal(err)
}

409
config.go
View File

@ -6,7 +6,6 @@ import (
"io/ioutil"
"log"
"path/filepath"
"reflect"
"sort"
"strings"
"time"
@ -21,45 +20,22 @@ import (
// will be logging to, as well as all the plugins that the user has
// specified
type Config struct {
// This lives outside the agent because mergeStruct doesn't need to handle
// maps normally. We just copy the elements manually in ApplyAgent.
Tags map[string]string
agent *Agent
plugins map[string]plugins.Plugin
pluginConfigurations map[string]*ConfiguredPlugin
outputs map[string]outputs.Output
agentFieldsSet []string
pluginFieldsSet map[string][]string
pluginConfigurationFieldsSet map[string][]string
outputFieldsSet map[string][]string
agent *ast.Table
plugins map[string]*ast.Table
outputs map[string]*ast.Table
}
// Returns a new, empty config object.
func NewConfig() *Config {
c := &Config{
Tags: make(map[string]string),
plugins: make(map[string]plugins.Plugin),
pluginConfigurations: make(map[string]*ConfiguredPlugin),
outputs: make(map[string]outputs.Output),
pluginFieldsSet: make(map[string][]string),
pluginConfigurationFieldsSet: make(map[string][]string),
outputFieldsSet: make(map[string][]string),
plugins: make(map[string]*ast.Table),
outputs: make(map[string]*ast.Table),
}
return c
}
// Plugins returns the configured plugins as a map of name -> plugins.Plugin
func (c *Config) Plugins() map[string]plugins.Plugin {
return c.plugins
}
// Outputs returns the configured outputs as a map of name -> outputs.Output
func (c *Config) Outputs() map[string]outputs.Output {
return c.outputs
}
// TagFilter is the name of a tag, and the values on which to filter
type TagFilter struct {
Name string
@ -81,14 +57,14 @@ type ConfiguredPlugin struct {
}
// ShouldPass returns true if the metric should pass, false if should drop
func (cp *ConfiguredPlugin) ShouldPass(measurement string, tags map[string]string) bool {
// based on the drop/pass plugin parameters
func (cp *ConfiguredPlugin) ShouldPass(measurement string) bool {
if cp.Pass != nil {
for _, pat := range cp.Pass {
if strings.HasPrefix(measurement, pat) {
return true
}
}
return false
}
@ -101,7 +77,12 @@ func (cp *ConfiguredPlugin) ShouldPass(measurement string, tags map[string]strin
return true
}
return true
}
// ShouldTagsPass returns true if the metric should pass, false if should drop
// based on the tagdrop/tagpass plugin parameters
func (cp *ConfiguredPlugin) ShouldTagsPass(tags map[string]string) bool {
if cp.TagPass != nil {
for _, pat := range cp.TagPass {
if tagval, ok := tags[pat.Name]; ok {
@ -131,42 +112,136 @@ func (cp *ConfiguredPlugin) ShouldPass(measurement string, tags map[string]strin
return true
}
// ApplyOutput loads the Output struct built from the config into the given Output struct.
// Overrides only values in the given struct that were set in the config.
// ApplyOutput loads the toml config into the given interface
func (c *Config) ApplyOutput(name string, v interface{}) error {
if c.outputs[name] != nil {
return mergeStruct(v, c.outputs[name], c.outputFieldsSet[name])
return toml.UnmarshalTable(c.outputs[name], v)
}
return nil
}
// ApplyAgent loads the Agent struct built from the config into the given Agent struct.
// Overrides only values in the given struct that were set in the config.
// ApplyAgent loads the toml config into the given Agent object, overriding
// defaults (such as collection duration) with the values from the toml config.
func (c *Config) ApplyAgent(a *Agent) error {
if c.agent != nil {
for key, value := range c.Tags {
a.Tags[key] = value
}
return mergeStruct(a, c.agent, c.agentFieldsSet)
return toml.UnmarshalTable(c.agent, a)
}
return nil
}
func (c *Config) GetPluginConfig(name string) *ConfiguredPlugin {
return c.pluginConfigurations[name]
// ApplyPlugin takes defined plugin names and applies them to the given
// interface, returning a ConfiguredPlugin object in the end that can
// be inserted into a runningPlugin by the agent.
func (c *Config) ApplyPlugin(name string, v interface{}) (*ConfiguredPlugin, error) {
cp := &ConfiguredPlugin{Name: name}
if tbl, ok := c.plugins[name]; ok {
if node, ok := tbl.Fields["pass"]; ok {
if kv, ok := node.(*ast.KeyValue); ok {
if ary, ok := kv.Value.(*ast.Array); ok {
for _, elem := range ary.Value {
if str, ok := elem.(*ast.String); ok {
cp.Pass = append(cp.Pass, str.Value)
}
}
}
}
}
if node, ok := tbl.Fields["drop"]; ok {
if kv, ok := node.(*ast.KeyValue); ok {
if ary, ok := kv.Value.(*ast.Array); ok {
for _, elem := range ary.Value {
if str, ok := elem.(*ast.String); ok {
cp.Drop = append(cp.Drop, str.Value)
}
}
}
}
}
if node, ok := tbl.Fields["interval"]; ok {
if kv, ok := node.(*ast.KeyValue); ok {
if str, ok := kv.Value.(*ast.String); ok {
dur, err := time.ParseDuration(str.Value)
if err != nil {
return nil, err
}
cp.Interval = dur
}
}
}
if node, ok := tbl.Fields["tagpass"]; ok {
if subtbl, ok := node.(*ast.Table); ok {
for name, val := range subtbl.Fields {
if kv, ok := val.(*ast.KeyValue); ok {
tagfilter := &TagFilter{Name: name}
if ary, ok := kv.Value.(*ast.Array); ok {
for _, elem := range ary.Value {
if str, ok := elem.(*ast.String); ok {
tagfilter.Filter = append(tagfilter.Filter, str.Value)
}
}
}
cp.TagPass = append(cp.TagPass, *tagfilter)
}
}
}
}
if node, ok := tbl.Fields["tagdrop"]; ok {
if subtbl, ok := node.(*ast.Table); ok {
for name, val := range subtbl.Fields {
if kv, ok := val.(*ast.KeyValue); ok {
tagfilter := &TagFilter{Name: name}
if ary, ok := kv.Value.(*ast.Array); ok {
for _, elem := range ary.Value {
if str, ok := elem.(*ast.String); ok {
tagfilter.Filter = append(tagfilter.Filter, str.Value)
}
}
}
cp.TagDrop = append(cp.TagDrop, *tagfilter)
}
}
}
}
delete(tbl.Fields, "drop")
delete(tbl.Fields, "pass")
delete(tbl.Fields, "interval")
delete(tbl.Fields, "tagdrop")
delete(tbl.Fields, "tagpass")
return cp, toml.UnmarshalTable(tbl, v)
}
return cp, nil
}
// Couldn't figure out how to get this to work with the declared function.
// PluginsDeclared returns the name of all plugins declared in the config.
func (c *Config) PluginsDeclared() map[string]plugins.Plugin {
return c.plugins
func (c *Config) PluginsDeclared() []string {
return declared(c.plugins)
}
// OutputsDeclared returns the name of all outputs declared in the config.
func (c *Config) OutputsDeclared() map[string]outputs.Output {
return c.outputs
func (c *Config) OutputsDeclared() []string {
return declared(c.outputs)
}
func declared(endpoints map[string]*ast.Table) []string {
var names []string
for name := range endpoints {
names = append(names, name)
}
sort.Strings(names)
return names
}
// ListTags returns a string of tags specified in the config,
@ -365,56 +440,6 @@ func PrintOutputConfig(name string) error {
return nil
}
// Find the field with a name matching fieldName, respecting the struct tag and ignoring case and underscores.
// If no field is found, return the zero reflect.Value, which should be checked for with .IsValid().
func findField(fieldName string, value reflect.Value) reflect.Value {
r := strings.NewReplacer("_", "")
vType := value.Type()
for i := 0; i < vType.NumField(); i++ {
fieldType := vType.Field(i)
// if we have toml tag, use it
if tag := fieldType.Tag.Get("toml"); tag != "" {
if tag == "-" { // omit
continue
}
if tag == fieldName {
return value.Field(i)
}
} else {
if strings.ToLower(fieldType.Name) == strings.ToLower(r.Replace(fieldName)) {
return value.Field(i)
}
}
}
return reflect.Value{}
}
// A very limited merge. Merges the fields named in the fields parameter,
// replacing most values, but appending to arrays.
func mergeStruct(base, overlay interface{}, fields []string) error {
baseValue := reflect.ValueOf(base).Elem()
overlayValue := reflect.ValueOf(overlay).Elem()
if baseValue.Kind() != reflect.Struct {
return fmt.Errorf("Tried to merge something that wasn't a struct: type %v was %v",
baseValue.Type(), baseValue.Kind())
}
if baseValue.Type() != overlayValue.Type() {
return fmt.Errorf("Tried to merge two different types: %v and %v",
baseValue.Type(), overlayValue.Type())
}
for _, field := range fields {
overlayFieldValue := findField(field, overlayValue)
if !overlayFieldValue.IsValid() {
return fmt.Errorf("could not find field in %v matching %v",
overlayValue.Type(), field)
}
baseFieldValue := findField(field, baseValue)
baseFieldValue.Set(overlayFieldValue)
}
return nil
}
func (c *Config) LoadDirectory(path string) error {
directoryEntries, err := ioutil.ReadDir(path)
if err != nil {
@ -432,13 +457,10 @@ func (c *Config) LoadDirectory(path string) error {
if err != nil {
return err
}
}
return nil
}
// hazmat area. Keeping the ast parsing here.
// LoadConfig loads the given config file and returns a *Config pointer
func (c *Config) LoadConfig(path string) error {
data, err := ioutil.ReadFile(path)
@ -459,11 +481,7 @@ func (c *Config) LoadConfig(path string) error {
switch name {
case "agent":
err := c.parseAgent(subTable)
if err != nil {
log.Printf("Could not parse [agent] config\n")
return err
}
c.agent = subTable
case "tags":
if err = toml.UnmarshalTable(subTable, c.Tags); err != nil {
log.Printf("Could not parse [tags] config\n")
@ -473,20 +491,11 @@ func (c *Config) LoadConfig(path string) error {
for outputName, outputVal := range subTable.Fields {
switch outputSubTable := outputVal.(type) {
case *ast.Table:
err = c.parseOutput(outputName, outputSubTable, 0)
if err != nil {
log.Printf("Could not parse config for output: %s\n",
outputName)
return err
}
c.outputs[outputName] = outputSubTable
case []*ast.Table:
for id, t := range outputSubTable {
err = c.parseOutput(outputName, t, id)
if err != nil {
log.Printf("Could not parse config for output: %s\n",
outputName)
return err
}
nameID := fmt.Sprintf("%s-%d", outputName, id)
c.outputs[nameID] = t
}
default:
return fmt.Errorf("Unsupported config format: %s",
@ -497,20 +506,11 @@ func (c *Config) LoadConfig(path string) error {
for pluginName, pluginVal := range subTable.Fields {
switch pluginSubTable := pluginVal.(type) {
case *ast.Table:
err = c.parsePlugin(pluginName, pluginSubTable, 0)
if err != nil {
log.Printf("Could not parse config for plugin: %s\n",
pluginName)
return err
}
c.plugins[pluginName] = pluginSubTable
case []*ast.Table:
for id, t := range pluginSubTable {
err = c.parsePlugin(pluginName, t, id)
if err != nil {
log.Printf("Could not parse config for plugin: %s\n",
pluginName)
return err
}
nameID := fmt.Sprintf("%s-%d", pluginName, id)
c.plugins[nameID] = t
}
default:
return fmt.Errorf("Unsupported config format: %s",
@ -520,155 +520,8 @@ func (c *Config) LoadConfig(path string) error {
// Assume it's a plugin for legacy config file support if no other
// identifiers are present
default:
err = c.parsePlugin(name, subTable, 0)
if err != nil {
return err
c.plugins[name] = subTable
}
}
}
return nil
}
// Needs to have the field names, for merging later.
func extractFieldNames(ast *ast.Table) []string {
// A reasonable capacity?
var names []string
for name := range ast.Fields {
names = append(names, name)
}
return names
}
// Parse the agent config out of the given *ast.Table.
func (c *Config) parseAgent(agentAst *ast.Table) error {
c.agentFieldsSet = extractFieldNames(agentAst)
agent := &Agent{}
err := toml.UnmarshalTable(agentAst, agent)
if err != nil {
return err
}
c.agent = agent
return nil
}
// Parse an output config out of the given *ast.Table.
func (c *Config) parseOutput(name string, outputAst *ast.Table, id int) error {
c.outputFieldsSet[name] = extractFieldNames(outputAst)
creator, ok := outputs.Outputs[name]
if !ok {
return fmt.Errorf("Undefined but requested output: %s", name)
}
output := creator()
err := toml.UnmarshalTable(outputAst, output)
if err != nil {
return err
}
c.outputs[fmt.Sprintf("%s-%d", name, id)] = output
return nil
}
// Parse a plugin config, plus plugin meta-config, out of the given *ast.Table.
func (c *Config) parsePlugin(name string, pluginAst *ast.Table, id int) error {
creator, ok := plugins.Plugins[name]
if !ok {
return fmt.Errorf("Undefined but requested plugin: %s", name)
}
plugin := creator()
cp := &ConfiguredPlugin{Name: name}
cpFields := make([]string, 0, 5)
if node, ok := pluginAst.Fields["pass"]; ok {
if kv, ok := node.(*ast.KeyValue); ok {
if ary, ok := kv.Value.(*ast.Array); ok {
for _, elem := range ary.Value {
if str, ok := elem.(*ast.String); ok {
cp.Pass = append(cp.Pass, str.Value)
}
}
cpFields = append(cpFields, "pass")
}
}
}
if node, ok := pluginAst.Fields["drop"]; ok {
if kv, ok := node.(*ast.KeyValue); ok {
if ary, ok := kv.Value.(*ast.Array); ok {
for _, elem := range ary.Value {
if str, ok := elem.(*ast.String); ok {
cp.Drop = append(cp.Drop, str.Value)
}
}
cpFields = append(cpFields, "drop")
}
}
}
if node, ok := pluginAst.Fields["interval"]; ok {
if kv, ok := node.(*ast.KeyValue); ok {
if str, ok := kv.Value.(*ast.String); ok {
dur, err := time.ParseDuration(str.Value)
if err != nil {
return err
}
cp.Interval = dur
cpFields = append(cpFields, "interval")
}
}
}
if node, ok := pluginAst.Fields["tagpass"]; ok {
if subtbl, ok := node.(*ast.Table); ok {
for name, val := range subtbl.Fields {
if kv, ok := val.(*ast.KeyValue); ok {
tagfilter := &TagFilter{Name: name}
if ary, ok := kv.Value.(*ast.Array); ok {
for _, elem := range ary.Value {
if str, ok := elem.(*ast.String); ok {
tagfilter.Filter = append(tagfilter.Filter, str.Value)
}
}
}
cp.TagPass = append(cp.TagPass, *tagfilter)
cpFields = append(cpFields, "tagpass")
}
}
}
}
if node, ok := pluginAst.Fields["tagdrop"]; ok {
if subtbl, ok := node.(*ast.Table); ok {
for name, val := range subtbl.Fields {
if kv, ok := val.(*ast.KeyValue); ok {
tagfilter := &TagFilter{Name: name}
if ary, ok := kv.Value.(*ast.Array); ok {
for _, elem := range ary.Value {
if str, ok := elem.(*ast.String); ok {
tagfilter.Filter = append(tagfilter.Filter, str.Value)
}
}
}
cp.TagDrop = append(cp.TagDrop, *tagfilter)
cpFields = append(cpFields, "tagdrop")
}
}
}
}
delete(pluginAst.Fields, "drop")
delete(pluginAst.Fields, "pass")
delete(pluginAst.Fields, "interval")
delete(pluginAst.Fields, "tagdrop")
delete(pluginAst.Fields, "tagpass")
nameID := fmt.Sprintf("%s-%d", name, id)
c.pluginFieldsSet[nameID] = extractFieldNames(pluginAst)
c.pluginConfigurationFieldsSet[nameID] = cpFields
err := toml.UnmarshalTable(pluginAst, plugin)
if err != nil {
return err
}
c.plugins[nameID] = plugin
c.pluginConfigurations[nameID] = cp
return nil
}