Revert much of the newer config file parsing, fix tagdrop/tagpass

accumulator.go

@@ -105,7 +105,7 @@ func (ac *accumulator) AddFields(
 	}
 
 	if ac.plugin != nil {
-		if !ac.plugin.ShouldPass(measurement, tags) {
+		if !ac.plugin.ShouldPass(measurement) || !ac.plugin.ShouldTagsPass(tags) {
 			return
 		}
 	}

agent.go

@@ -66,6 +66,8 @@ type Agent struct {
 
 	Tags map[string]string
 
+	Config *Config
+
 	outputs []*runningOutput
 	plugins []*runningPlugin
 }
@@ -74,6 +76,7 @@ type Agent struct {
 func NewAgent(config *Config) (*Agent, error) {
 	agent := &Agent{
 		Tags:          make(map[string]string),
+		Config:        config,
 		Interval:      internal.Duration{10 * time.Second},
 		RoundInterval: true,
 		FlushInterval: internal.Duration{10 * time.Second},
@@ -96,7 +99,11 @@ func NewAgent(config *Config) (*Agent, error) {
 		agent.Hostname = hostname
 	}
 
-	agent.Tags["host"] = agent.Hostname
+	if config.Tags == nil {
+		config.Tags = map[string]string{}
+	}
+
+	config.Tags["host"] = agent.Hostname
 
 	return agent, nil
 }
@@ -146,18 +153,21 @@ func (a *Agent) Close() error {
 }
 
 // LoadOutputs loads the agent's outputs
-func (a *Agent) LoadOutputs(filters []string, config *Config) ([]string, error) {
+func (a *Agent) LoadOutputs(filters []string) ([]string, error) {
 	var names []string
 
-	for name, output := range config.OutputsDeclared() {
+	for _, name := range a.Config.OutputsDeclared() {
 		// Trim the ID off the output name for filtering
 		filtername := strings.TrimRight(name, "-0123456789")
-		if sliceContains(filtername, filters) || len(filters) == 0 {
+		creator, ok := outputs.Outputs[filtername]
-			if a.Debug {
+		if !ok {
-				log.Println("Output Enabled: ", name)
+			return nil, fmt.Errorf("Undefined but requested output: %s", name)
 		}
 
-			err := config.ApplyOutput(name, output)
+		if sliceContains(filtername, filters) || len(filters) == 0 {
+			output := creator()
+
+			err := a.Config.ApplyOutput(name, output)
 			if err != nil {
 				return nil, err
 			}
@@ -173,15 +183,27 @@ func (a *Agent) LoadOutputs(filters []string, config *Config) ([]string, error)
 }
 
 // LoadPlugins loads the agent's plugins
-func (a *Agent) LoadPlugins(filters []string, config *Config) ([]string, error) {
+func (a *Agent) LoadPlugins(filters []string) ([]string, error) {
 	var names []string
 
-	for name, plugin := range config.PluginsDeclared() {
+	for _, name := range a.Config.PluginsDeclared() {
 		// Trim the ID off the output name for filtering
 		filtername := strings.TrimRight(name, "-0123456789")
+		creator, ok := plugins.Plugins[filtername]
+		if !ok {
+			return nil, fmt.Errorf("Undefined but requested plugin: %s", name)
+		}
+
 		if sliceContains(filtername, filters) || len(filters) == 0 {
-			config := config.GetPluginConfig(name)
+			plugin := creator()
-			a.plugins = append(a.plugins, &runningPlugin{name, filtername, plugin, config})
+
+			config, err := a.Config.ApplyPlugin(name, plugin)
+			if err != nil {
+				return nil, err
+			}
+
+			a.plugins = append(a.plugins,
+				&runningPlugin{name, filtername, plugin, config})
 			names = append(names, name)
 		}
 	}

cmd/telegraf/telegraf.go

@@ -102,7 +102,7 @@ func main() {
 		ag.Debug = true
 	}
 
-	outputs, err := ag.LoadOutputs(outputFilters, config)
+	outputs, err := ag.LoadOutputs(outputFilters)
 	if err != nil {
 		log.Fatal(err)
 	}
@@ -111,7 +111,7 @@ func main() {
 		os.Exit(1)
 	}
 
-	plugins, err := ag.LoadPlugins(pluginFilters, config)
+	plugins, err := ag.LoadPlugins(pluginFilters)
 	if err != nil {
 		log.Fatal(err)
 	}

config.go

@@ -6,7 +6,6 @@ import (
 	"io/ioutil"
 	"log"
 	"path/filepath"
-	"reflect"
 	"sort"
 	"strings"
 	"time"
@@ -21,45 +20,22 @@ import (
 // will be logging to, as well as all the plugins that the user has
 // specified
 type Config struct {
-	// This lives outside the agent because mergeStruct doesn't need to handle
-	// maps normally. We just copy the elements manually in ApplyAgent.
 	Tags map[string]string
 
-	agent                *Agent
+	agent   *ast.Table
-	plugins              map[string]plugins.Plugin
+	plugins map[string]*ast.Table
-	pluginConfigurations map[string]*ConfiguredPlugin
+	outputs map[string]*ast.Table
-	outputs              map[string]outputs.Output
-
-	agentFieldsSet               []string
-	pluginFieldsSet              map[string][]string
-	pluginConfigurationFieldsSet map[string][]string
-	outputFieldsSet              map[string][]string
 }
 
-// Returns a new, empty config object.
 func NewConfig() *Config {
 	c := &Config{
 		Tags:    make(map[string]string),
-		plugins:                      make(map[string]plugins.Plugin),
+		plugins: make(map[string]*ast.Table),
-		pluginConfigurations:         make(map[string]*ConfiguredPlugin),
+		outputs: make(map[string]*ast.Table),
-		outputs:                      make(map[string]outputs.Output),
-		pluginFieldsSet:              make(map[string][]string),
-		pluginConfigurationFieldsSet: make(map[string][]string),
-		outputFieldsSet:              make(map[string][]string),
 	}
 	return c
 }
 
-// Plugins returns the configured plugins as a map of name -> plugins.Plugin
-func (c *Config) Plugins() map[string]plugins.Plugin {
-	return c.plugins
-}
-
-// Outputs returns the configured outputs as a map of name -> outputs.Output
-func (c *Config) Outputs() map[string]outputs.Output {
-	return c.outputs
-}
-
 // TagFilter is the name of a tag, and the values on which to filter
 type TagFilter struct {
 	Name   string
@@ -81,14 +57,14 @@ type ConfiguredPlugin struct {
 }
 
 // ShouldPass returns true if the metric should pass, false if should drop
-func (cp *ConfiguredPlugin) ShouldPass(measurement string, tags map[string]string) bool {
+// based on the drop/pass plugin parameters
+func (cp *ConfiguredPlugin) ShouldPass(measurement string) bool {
 	if cp.Pass != nil {
 		for _, pat := range cp.Pass {
 			if strings.HasPrefix(measurement, pat) {
 				return true
 			}
 		}
-
 		return false
 	}
 
@@ -101,7 +77,12 @@ func (cp *ConfiguredPlugin) ShouldPass(measurement string, tags map[string]strin
 
 		return true
 	}
+	return true
+}
 
+// ShouldTagsPass returns true if the metric should pass, false if should drop
+// based on the tagdrop/tagpass plugin parameters
+func (cp *ConfiguredPlugin) ShouldTagsPass(tags map[string]string) bool {
 	if cp.TagPass != nil {
 		for _, pat := range cp.TagPass {
 			if tagval, ok := tags[pat.Name]; ok {
@@ -131,42 +112,136 @@ func (cp *ConfiguredPlugin) ShouldPass(measurement string, tags map[string]strin
 	return true
 }
 
-// ApplyOutput loads the Output struct built from the config into the given Output struct.
+// ApplyOutput loads the toml config into the given interface
-// Overrides only values in the given struct that were set in the config.
 func (c *Config) ApplyOutput(name string, v interface{}) error {
 	if c.outputs[name] != nil {
-		return mergeStruct(v, c.outputs[name], c.outputFieldsSet[name])
+		return toml.UnmarshalTable(c.outputs[name], v)
 	}
 	return nil
 }
 
-// ApplyAgent loads the Agent struct built from the config into the given Agent struct.
+// ApplyAgent loads the toml config into the given Agent object, overriding
-// Overrides only values in the given struct that were set in the config.
+// defaults (such as collection duration) with the values from the toml config.
 func (c *Config) ApplyAgent(a *Agent) error {
 	if c.agent != nil {
-		for key, value := range c.Tags {
+		return toml.UnmarshalTable(c.agent, a)
-			a.Tags[key] = value
-		}
-		return mergeStruct(a, c.agent, c.agentFieldsSet)
 	}
 
 	return nil
 }
 
-func (c *Config) GetPluginConfig(name string) *ConfiguredPlugin {
+// ApplyPlugin takes defined plugin names and applies them to the given
-	return c.pluginConfigurations[name]
+// interface, returning a ConfiguredPlugin object in the end that can
+// be inserted into a runningPlugin by the agent.
+func (c *Config) ApplyPlugin(name string, v interface{}) (*ConfiguredPlugin, error) {
+	cp := &ConfiguredPlugin{Name: name}
+
+	if tbl, ok := c.plugins[name]; ok {
+
+		if node, ok := tbl.Fields["pass"]; ok {
+			if kv, ok := node.(*ast.KeyValue); ok {
+				if ary, ok := kv.Value.(*ast.Array); ok {
+					for _, elem := range ary.Value {
+						if str, ok := elem.(*ast.String); ok {
+							cp.Pass = append(cp.Pass, str.Value)
+						}
+					}
+				}
+			}
 		}
 
-// Couldn't figure out how to get this to work with the declared function.
+		if node, ok := tbl.Fields["drop"]; ok {
+			if kv, ok := node.(*ast.KeyValue); ok {
+				if ary, ok := kv.Value.(*ast.Array); ok {
+					for _, elem := range ary.Value {
+						if str, ok := elem.(*ast.String); ok {
+							cp.Drop = append(cp.Drop, str.Value)
+						}
+					}
+				}
+			}
+		}
+
+		if node, ok := tbl.Fields["interval"]; ok {
+			if kv, ok := node.(*ast.KeyValue); ok {
+				if str, ok := kv.Value.(*ast.String); ok {
+					dur, err := time.ParseDuration(str.Value)
+					if err != nil {
+						return nil, err
+					}
+
+					cp.Interval = dur
+				}
+			}
+		}
+
+		if node, ok := tbl.Fields["tagpass"]; ok {
+			if subtbl, ok := node.(*ast.Table); ok {
+				for name, val := range subtbl.Fields {
+					if kv, ok := val.(*ast.KeyValue); ok {
+						tagfilter := &TagFilter{Name: name}
+						if ary, ok := kv.Value.(*ast.Array); ok {
+							for _, elem := range ary.Value {
+								if str, ok := elem.(*ast.String); ok {
+									tagfilter.Filter = append(tagfilter.Filter, str.Value)
+								}
+							}
+						}
+						cp.TagPass = append(cp.TagPass, *tagfilter)
+					}
+				}
+			}
+		}
+
+		if node, ok := tbl.Fields["tagdrop"]; ok {
+			if subtbl, ok := node.(*ast.Table); ok {
+				for name, val := range subtbl.Fields {
+					if kv, ok := val.(*ast.KeyValue); ok {
+						tagfilter := &TagFilter{Name: name}
+						if ary, ok := kv.Value.(*ast.Array); ok {
+							for _, elem := range ary.Value {
+								if str, ok := elem.(*ast.String); ok {
+									tagfilter.Filter = append(tagfilter.Filter, str.Value)
+								}
+							}
+						}
+						cp.TagDrop = append(cp.TagDrop, *tagfilter)
+					}
+				}
+			}
+		}
+
+		delete(tbl.Fields, "drop")
+		delete(tbl.Fields, "pass")
+		delete(tbl.Fields, "interval")
+		delete(tbl.Fields, "tagdrop")
+		delete(tbl.Fields, "tagpass")
+		return cp, toml.UnmarshalTable(tbl, v)
+	}
+
+	return cp, nil
+}
 
 // PluginsDeclared returns the name of all plugins declared in the config.
-func (c *Config) PluginsDeclared() map[string]plugins.Plugin {
+func (c *Config) PluginsDeclared() []string {
-	return c.plugins
+	return declared(c.plugins)
 }
 
 // OutputsDeclared returns the name of all outputs declared in the config.
-func (c *Config) OutputsDeclared() map[string]outputs.Output {
+func (c *Config) OutputsDeclared() []string {
-	return c.outputs
+	return declared(c.outputs)
+}
+
+func declared(endpoints map[string]*ast.Table) []string {
+	var names []string
+
+	for name := range endpoints {
+		names = append(names, name)
+	}
+
+	sort.Strings(names)
+
+	return names
 }
 
 // ListTags returns a string of tags specified in the config,
@@ -365,56 +440,6 @@ func PrintOutputConfig(name string) error {
 	return nil
 }
 
-// Find the field with a name matching fieldName, respecting the struct tag and ignoring case and underscores.
-// If no field is found, return the zero reflect.Value, which should be checked for with .IsValid().
-func findField(fieldName string, value reflect.Value) reflect.Value {
-	r := strings.NewReplacer("_", "")
-	vType := value.Type()
-	for i := 0; i < vType.NumField(); i++ {
-		fieldType := vType.Field(i)
-
-		// if we have toml tag, use it
-		if tag := fieldType.Tag.Get("toml"); tag != "" {
-			if tag == "-" { // omit
-				continue
-			}
-			if tag == fieldName {
-				return value.Field(i)
-			}
-		} else {
-			if strings.ToLower(fieldType.Name) == strings.ToLower(r.Replace(fieldName)) {
-				return value.Field(i)
-			}
-		}
-	}
-	return reflect.Value{}
-}
-
-// A very limited merge. Merges the fields named in the fields parameter,
-// replacing most values, but appending to arrays.
-func mergeStruct(base, overlay interface{}, fields []string) error {
-	baseValue := reflect.ValueOf(base).Elem()
-	overlayValue := reflect.ValueOf(overlay).Elem()
-	if baseValue.Kind() != reflect.Struct {
-		return fmt.Errorf("Tried to merge something that wasn't a struct: type %v was %v",
-			baseValue.Type(), baseValue.Kind())
-	}
-	if baseValue.Type() != overlayValue.Type() {
-		return fmt.Errorf("Tried to merge two different types: %v and %v",
-			baseValue.Type(), overlayValue.Type())
-	}
-	for _, field := range fields {
-		overlayFieldValue := findField(field, overlayValue)
-		if !overlayFieldValue.IsValid() {
-			return fmt.Errorf("could not find field in %v matching %v",
-				overlayValue.Type(), field)
-		}
-		baseFieldValue := findField(field, baseValue)
-		baseFieldValue.Set(overlayFieldValue)
-	}
-	return nil
-}
-
 func (c *Config) LoadDirectory(path string) error {
 	directoryEntries, err := ioutil.ReadDir(path)
 	if err != nil {
@@ -432,13 +457,10 @@ func (c *Config) LoadDirectory(path string) error {
 		if err != nil {
 			return err
 		}
-
 	}
 	return nil
 }
 
-// hazmat area. Keeping the ast parsing here.
-
 // LoadConfig loads the given config file and returns a *Config pointer
 func (c *Config) LoadConfig(path string) error {
 	data, err := ioutil.ReadFile(path)
@@ -459,11 +481,7 @@ func (c *Config) LoadConfig(path string) error {
 
 		switch name {
 		case "agent":
-			err := c.parseAgent(subTable)
+			c.agent = subTable
-			if err != nil {
-				log.Printf("Could not parse [agent] config\n")
-				return err
-			}
 		case "tags":
 			if err = toml.UnmarshalTable(subTable, c.Tags); err != nil {
 				log.Printf("Could not parse [tags] config\n")
@@ -473,20 +491,11 @@ func (c *Config) LoadConfig(path string) error {
 			for outputName, outputVal := range subTable.Fields {
 				switch outputSubTable := outputVal.(type) {
 				case *ast.Table:
-					err = c.parseOutput(outputName, outputSubTable, 0)
+					c.outputs[outputName] = outputSubTable
-					if err != nil {
-						log.Printf("Could not parse config for output: %s\n",
-							outputName)
-						return err
-					}
 				case []*ast.Table:
 					for id, t := range outputSubTable {
-						err = c.parseOutput(outputName, t, id)
+						nameID := fmt.Sprintf("%s-%d", outputName, id)
-						if err != nil {
+						c.outputs[nameID] = t
-							log.Printf("Could not parse config for output: %s\n",
-								outputName)
-							return err
-						}
 					}
 				default:
 					return fmt.Errorf("Unsupported config format: %s",
@@ -497,20 +506,11 @@ func (c *Config) LoadConfig(path string) error {
 			for pluginName, pluginVal := range subTable.Fields {
 				switch pluginSubTable := pluginVal.(type) {
 				case *ast.Table:
-					err = c.parsePlugin(pluginName, pluginSubTable, 0)
+					c.plugins[pluginName] = pluginSubTable
-					if err != nil {
-						log.Printf("Could not parse config for plugin: %s\n",
-							pluginName)
-						return err
-					}
 				case []*ast.Table:
 					for id, t := range pluginSubTable {
-						err = c.parsePlugin(pluginName, t, id)
+						nameID := fmt.Sprintf("%s-%d", pluginName, id)
-						if err != nil {
+						c.plugins[nameID] = t
-							log.Printf("Could not parse config for plugin: %s\n",
-								pluginName)
-							return err
-						}
 					}
 				default:
 					return fmt.Errorf("Unsupported config format: %s",
@@ -520,155 +520,8 @@ func (c *Config) LoadConfig(path string) error {
 		// Assume it's a plugin for legacy config file support if no other
 		// identifiers are present
 		default:
-			err = c.parsePlugin(name, subTable, 0)
+			c.plugins[name] = subTable
-			if err != nil {
-				return err
 		}
 	}
-	}
-
-	return nil
-}
-
-// Needs to have the field names, for merging later.
-func extractFieldNames(ast *ast.Table) []string {
-	// A reasonable capacity?
-	var names []string
-	for name := range ast.Fields {
-		names = append(names, name)
-	}
-	return names
-}
-
-// Parse the agent config out of the given *ast.Table.
-func (c *Config) parseAgent(agentAst *ast.Table) error {
-	c.agentFieldsSet = extractFieldNames(agentAst)
-	agent := &Agent{}
-	err := toml.UnmarshalTable(agentAst, agent)
-	if err != nil {
-		return err
-	}
-	c.agent = agent
-	return nil
-}
-
-// Parse an output config out of the given *ast.Table.
-func (c *Config) parseOutput(name string, outputAst *ast.Table, id int) error {
-	c.outputFieldsSet[name] = extractFieldNames(outputAst)
-	creator, ok := outputs.Outputs[name]
-	if !ok {
-		return fmt.Errorf("Undefined but requested output: %s", name)
-	}
-	output := creator()
-	err := toml.UnmarshalTable(outputAst, output)
-	if err != nil {
-		return err
-	}
-	c.outputs[fmt.Sprintf("%s-%d", name, id)] = output
-	return nil
-}
-
-// Parse a plugin config, plus plugin meta-config, out of the given *ast.Table.
-func (c *Config) parsePlugin(name string, pluginAst *ast.Table, id int) error {
-	creator, ok := plugins.Plugins[name]
-	if !ok {
-		return fmt.Errorf("Undefined but requested plugin: %s", name)
-	}
-	plugin := creator()
-	cp := &ConfiguredPlugin{Name: name}
-	cpFields := make([]string, 0, 5)
-
-	if node, ok := pluginAst.Fields["pass"]; ok {
-		if kv, ok := node.(*ast.KeyValue); ok {
-			if ary, ok := kv.Value.(*ast.Array); ok {
-				for _, elem := range ary.Value {
-					if str, ok := elem.(*ast.String); ok {
-						cp.Pass = append(cp.Pass, str.Value)
-					}
-				}
-				cpFields = append(cpFields, "pass")
-			}
-		}
-	}
-
-	if node, ok := pluginAst.Fields["drop"]; ok {
-		if kv, ok := node.(*ast.KeyValue); ok {
-			if ary, ok := kv.Value.(*ast.Array); ok {
-				for _, elem := range ary.Value {
-					if str, ok := elem.(*ast.String); ok {
-						cp.Drop = append(cp.Drop, str.Value)
-					}
-				}
-				cpFields = append(cpFields, "drop")
-			}
-		}
-	}
-
-	if node, ok := pluginAst.Fields["interval"]; ok {
-		if kv, ok := node.(*ast.KeyValue); ok {
-			if str, ok := kv.Value.(*ast.String); ok {
-				dur, err := time.ParseDuration(str.Value)
-				if err != nil {
-					return err
-				}
-
-				cp.Interval = dur
-				cpFields = append(cpFields, "interval")
-			}
-		}
-	}
-
-	if node, ok := pluginAst.Fields["tagpass"]; ok {
-		if subtbl, ok := node.(*ast.Table); ok {
-			for name, val := range subtbl.Fields {
-				if kv, ok := val.(*ast.KeyValue); ok {
-					tagfilter := &TagFilter{Name: name}
-					if ary, ok := kv.Value.(*ast.Array); ok {
-						for _, elem := range ary.Value {
-							if str, ok := elem.(*ast.String); ok {
-								tagfilter.Filter = append(tagfilter.Filter, str.Value)
-							}
-						}
-					}
-					cp.TagPass = append(cp.TagPass, *tagfilter)
-					cpFields = append(cpFields, "tagpass")
-				}
-			}
-		}
-	}
-
-	if node, ok := pluginAst.Fields["tagdrop"]; ok {
-		if subtbl, ok := node.(*ast.Table); ok {
-			for name, val := range subtbl.Fields {
-				if kv, ok := val.(*ast.KeyValue); ok {
-					tagfilter := &TagFilter{Name: name}
-					if ary, ok := kv.Value.(*ast.Array); ok {
-						for _, elem := range ary.Value {
-							if str, ok := elem.(*ast.String); ok {
-								tagfilter.Filter = append(tagfilter.Filter, str.Value)
-							}
-						}
-					}
-					cp.TagDrop = append(cp.TagDrop, *tagfilter)
-					cpFields = append(cpFields, "tagdrop")
-				}
-			}
-		}
-	}
-
-	delete(pluginAst.Fields, "drop")
-	delete(pluginAst.Fields, "pass")
-	delete(pluginAst.Fields, "interval")
-	delete(pluginAst.Fields, "tagdrop")
-	delete(pluginAst.Fields, "tagpass")
-	nameID := fmt.Sprintf("%s-%d", name, id)
-	c.pluginFieldsSet[nameID] = extractFieldNames(pluginAst)
-	c.pluginConfigurationFieldsSet[nameID] = cpFields
-	err := toml.UnmarshalTable(pluginAst, plugin)
-	if err != nil {
-		return err
-	}
-	c.plugins[nameID] = plugin
-	c.pluginConfigurations[nameID] = cp
 	return nil
 }