From 8dde60e869ca66096a6d56fa4fd4bd92d009ad5b Mon Sep 17 00:00:00 2001 From: Cameron Sparr Date: Mon, 23 Nov 2015 18:00:54 -0700 Subject: [PATCH] Revert much of the newer config file parsing, fix tagdrop/tagpass --- accumulator.go | 2 +- agent.go | 46 +++-- cmd/telegraf/telegraf.go | 4 +- config.go | 411 +++++++++++++-------------------------- 4 files changed, 169 insertions(+), 294 deletions(-) diff --git a/accumulator.go b/accumulator.go index ae6af52e5..e489c1dc9 100644 --- a/accumulator.go +++ b/accumulator.go @@ -105,7 +105,7 @@ func (ac *accumulator) AddFields( } if ac.plugin != nil { - if !ac.plugin.ShouldPass(measurement, tags) { + if !ac.plugin.ShouldPass(measurement) || !ac.plugin.ShouldTagsPass(tags) { return } } diff --git a/agent.go b/agent.go index b671df106..564d43dd6 100644 --- a/agent.go +++ b/agent.go @@ -66,6 +66,8 @@ type Agent struct { Tags map[string]string + Config *Config + outputs []*runningOutput plugins []*runningPlugin } @@ -74,6 +76,7 @@ type Agent struct { func NewAgent(config *Config) (*Agent, error) { agent := &Agent{ Tags: make(map[string]string), + Config: config, Interval: internal.Duration{10 * time.Second}, RoundInterval: true, FlushInterval: internal.Duration{10 * time.Second}, @@ -96,7 +99,11 @@ func NewAgent(config *Config) (*Agent, error) { agent.Hostname = hostname } - agent.Tags["host"] = agent.Hostname + if config.Tags == nil { + config.Tags = map[string]string{} + } + + config.Tags["host"] = agent.Hostname return agent, nil } @@ -146,18 +153,21 @@ func (a *Agent) Close() error { } // LoadOutputs loads the agent's outputs -func (a *Agent) LoadOutputs(filters []string, config *Config) ([]string, error) { +func (a *Agent) LoadOutputs(filters []string) ([]string, error) { var names []string - for name, output := range config.OutputsDeclared() { + for _, name := range a.Config.OutputsDeclared() { // Trim the ID off the output name for filtering filtername := strings.TrimRight(name, "-0123456789") - if sliceContains(filtername, filters) || len(filters) == 0 { - if a.Debug { - log.Println("Output Enabled: ", name) - } + creator, ok := outputs.Outputs[filtername] + if !ok { + return nil, fmt.Errorf("Undefined but requested output: %s", name) + } - err := config.ApplyOutput(name, output) + if sliceContains(filtername, filters) || len(filters) == 0 { + output := creator() + + err := a.Config.ApplyOutput(name, output) if err != nil { return nil, err } @@ -173,15 +183,27 @@ func (a *Agent) LoadOutputs(filters []string, config *Config) ([]string, error) } // LoadPlugins loads the agent's plugins -func (a *Agent) LoadPlugins(filters []string, config *Config) ([]string, error) { +func (a *Agent) LoadPlugins(filters []string) ([]string, error) { var names []string - for name, plugin := range config.PluginsDeclared() { + for _, name := range a.Config.PluginsDeclared() { // Trim the ID off the output name for filtering filtername := strings.TrimRight(name, "-0123456789") + creator, ok := plugins.Plugins[filtername] + if !ok { + return nil, fmt.Errorf("Undefined but requested plugin: %s", name) + } + if sliceContains(filtername, filters) || len(filters) == 0 { - config := config.GetPluginConfig(name) - a.plugins = append(a.plugins, &runningPlugin{name, filtername, plugin, config}) + plugin := creator() + + config, err := a.Config.ApplyPlugin(name, plugin) + if err != nil { + return nil, err + } + + a.plugins = append(a.plugins, + &runningPlugin{name, filtername, plugin, config}) names = append(names, name) } } diff --git a/cmd/telegraf/telegraf.go b/cmd/telegraf/telegraf.go index 57db12407..7f25fc7ac 100644 --- a/cmd/telegraf/telegraf.go +++ b/cmd/telegraf/telegraf.go @@ -102,7 +102,7 @@ func main() { ag.Debug = true } - outputs, err := ag.LoadOutputs(outputFilters, config) + outputs, err := ag.LoadOutputs(outputFilters) if err != nil { log.Fatal(err) } @@ -111,7 +111,7 @@ func main() { os.Exit(1) } - plugins, err := ag.LoadPlugins(pluginFilters, config) + plugins, err := ag.LoadPlugins(pluginFilters) if err != nil { log.Fatal(err) } diff --git a/config.go b/config.go index a17a85209..9ffd0c80f 100644 --- a/config.go +++ b/config.go @@ -6,7 +6,6 @@ import ( "io/ioutil" "log" "path/filepath" - "reflect" "sort" "strings" "time" @@ -21,45 +20,22 @@ import ( // will be logging to, as well as all the plugins that the user has // specified type Config struct { - // This lives outside the agent because mergeStruct doesn't need to handle - // maps normally. We just copy the elements manually in ApplyAgent. Tags map[string]string - agent *Agent - plugins map[string]plugins.Plugin - pluginConfigurations map[string]*ConfiguredPlugin - outputs map[string]outputs.Output - - agentFieldsSet []string - pluginFieldsSet map[string][]string - pluginConfigurationFieldsSet map[string][]string - outputFieldsSet map[string][]string + agent *ast.Table + plugins map[string]*ast.Table + outputs map[string]*ast.Table } -// Returns a new, empty config object. func NewConfig() *Config { c := &Config{ - Tags: make(map[string]string), - plugins: make(map[string]plugins.Plugin), - pluginConfigurations: make(map[string]*ConfiguredPlugin), - outputs: make(map[string]outputs.Output), - pluginFieldsSet: make(map[string][]string), - pluginConfigurationFieldsSet: make(map[string][]string), - outputFieldsSet: make(map[string][]string), + Tags: make(map[string]string), + plugins: make(map[string]*ast.Table), + outputs: make(map[string]*ast.Table), } return c } -// Plugins returns the configured plugins as a map of name -> plugins.Plugin -func (c *Config) Plugins() map[string]plugins.Plugin { - return c.plugins -} - -// Outputs returns the configured outputs as a map of name -> outputs.Output -func (c *Config) Outputs() map[string]outputs.Output { - return c.outputs -} - // TagFilter is the name of a tag, and the values on which to filter type TagFilter struct { Name string @@ -81,14 +57,14 @@ type ConfiguredPlugin struct { } // ShouldPass returns true if the metric should pass, false if should drop -func (cp *ConfiguredPlugin) ShouldPass(measurement string, tags map[string]string) bool { +// based on the drop/pass plugin parameters +func (cp *ConfiguredPlugin) ShouldPass(measurement string) bool { if cp.Pass != nil { for _, pat := range cp.Pass { if strings.HasPrefix(measurement, pat) { return true } } - return false } @@ -101,7 +77,12 @@ func (cp *ConfiguredPlugin) ShouldPass(measurement string, tags map[string]strin return true } + return true +} +// ShouldTagsPass returns true if the metric should pass, false if should drop +// based on the tagdrop/tagpass plugin parameters +func (cp *ConfiguredPlugin) ShouldTagsPass(tags map[string]string) bool { if cp.TagPass != nil { for _, pat := range cp.TagPass { if tagval, ok := tags[pat.Name]; ok { @@ -131,42 +112,136 @@ func (cp *ConfiguredPlugin) ShouldPass(measurement string, tags map[string]strin return true } -// ApplyOutput loads the Output struct built from the config into the given Output struct. -// Overrides only values in the given struct that were set in the config. +// ApplyOutput loads the toml config into the given interface func (c *Config) ApplyOutput(name string, v interface{}) error { if c.outputs[name] != nil { - return mergeStruct(v, c.outputs[name], c.outputFieldsSet[name]) + return toml.UnmarshalTable(c.outputs[name], v) } return nil } -// ApplyAgent loads the Agent struct built from the config into the given Agent struct. -// Overrides only values in the given struct that were set in the config. +// ApplyAgent loads the toml config into the given Agent object, overriding +// defaults (such as collection duration) with the values from the toml config. func (c *Config) ApplyAgent(a *Agent) error { if c.agent != nil { - for key, value := range c.Tags { - a.Tags[key] = value - } - return mergeStruct(a, c.agent, c.agentFieldsSet) + return toml.UnmarshalTable(c.agent, a) } return nil } -func (c *Config) GetPluginConfig(name string) *ConfiguredPlugin { - return c.pluginConfigurations[name] +// ApplyPlugin takes defined plugin names and applies them to the given +// interface, returning a ConfiguredPlugin object in the end that can +// be inserted into a runningPlugin by the agent. +func (c *Config) ApplyPlugin(name string, v interface{}) (*ConfiguredPlugin, error) { + cp := &ConfiguredPlugin{Name: name} + + if tbl, ok := c.plugins[name]; ok { + + if node, ok := tbl.Fields["pass"]; ok { + if kv, ok := node.(*ast.KeyValue); ok { + if ary, ok := kv.Value.(*ast.Array); ok { + for _, elem := range ary.Value { + if str, ok := elem.(*ast.String); ok { + cp.Pass = append(cp.Pass, str.Value) + } + } + } + } + } + + if node, ok := tbl.Fields["drop"]; ok { + if kv, ok := node.(*ast.KeyValue); ok { + if ary, ok := kv.Value.(*ast.Array); ok { + for _, elem := range ary.Value { + if str, ok := elem.(*ast.String); ok { + cp.Drop = append(cp.Drop, str.Value) + } + } + } + } + } + + if node, ok := tbl.Fields["interval"]; ok { + if kv, ok := node.(*ast.KeyValue); ok { + if str, ok := kv.Value.(*ast.String); ok { + dur, err := time.ParseDuration(str.Value) + if err != nil { + return nil, err + } + + cp.Interval = dur + } + } + } + + if node, ok := tbl.Fields["tagpass"]; ok { + if subtbl, ok := node.(*ast.Table); ok { + for name, val := range subtbl.Fields { + if kv, ok := val.(*ast.KeyValue); ok { + tagfilter := &TagFilter{Name: name} + if ary, ok := kv.Value.(*ast.Array); ok { + for _, elem := range ary.Value { + if str, ok := elem.(*ast.String); ok { + tagfilter.Filter = append(tagfilter.Filter, str.Value) + } + } + } + cp.TagPass = append(cp.TagPass, *tagfilter) + } + } + } + } + + if node, ok := tbl.Fields["tagdrop"]; ok { + if subtbl, ok := node.(*ast.Table); ok { + for name, val := range subtbl.Fields { + if kv, ok := val.(*ast.KeyValue); ok { + tagfilter := &TagFilter{Name: name} + if ary, ok := kv.Value.(*ast.Array); ok { + for _, elem := range ary.Value { + if str, ok := elem.(*ast.String); ok { + tagfilter.Filter = append(tagfilter.Filter, str.Value) + } + } + } + cp.TagDrop = append(cp.TagDrop, *tagfilter) + } + } + } + } + + delete(tbl.Fields, "drop") + delete(tbl.Fields, "pass") + delete(tbl.Fields, "interval") + delete(tbl.Fields, "tagdrop") + delete(tbl.Fields, "tagpass") + return cp, toml.UnmarshalTable(tbl, v) + } + + return cp, nil } -// Couldn't figure out how to get this to work with the declared function. - // PluginsDeclared returns the name of all plugins declared in the config. -func (c *Config) PluginsDeclared() map[string]plugins.Plugin { - return c.plugins +func (c *Config) PluginsDeclared() []string { + return declared(c.plugins) } // OutputsDeclared returns the name of all outputs declared in the config. -func (c *Config) OutputsDeclared() map[string]outputs.Output { - return c.outputs +func (c *Config) OutputsDeclared() []string { + return declared(c.outputs) +} + +func declared(endpoints map[string]*ast.Table) []string { + var names []string + + for name := range endpoints { + names = append(names, name) + } + + sort.Strings(names) + + return names } // ListTags returns a string of tags specified in the config, @@ -365,56 +440,6 @@ func PrintOutputConfig(name string) error { return nil } -// Find the field with a name matching fieldName, respecting the struct tag and ignoring case and underscores. -// If no field is found, return the zero reflect.Value, which should be checked for with .IsValid(). -func findField(fieldName string, value reflect.Value) reflect.Value { - r := strings.NewReplacer("_", "") - vType := value.Type() - for i := 0; i < vType.NumField(); i++ { - fieldType := vType.Field(i) - - // if we have toml tag, use it - if tag := fieldType.Tag.Get("toml"); tag != "" { - if tag == "-" { // omit - continue - } - if tag == fieldName { - return value.Field(i) - } - } else { - if strings.ToLower(fieldType.Name) == strings.ToLower(r.Replace(fieldName)) { - return value.Field(i) - } - } - } - return reflect.Value{} -} - -// A very limited merge. Merges the fields named in the fields parameter, -// replacing most values, but appending to arrays. -func mergeStruct(base, overlay interface{}, fields []string) error { - baseValue := reflect.ValueOf(base).Elem() - overlayValue := reflect.ValueOf(overlay).Elem() - if baseValue.Kind() != reflect.Struct { - return fmt.Errorf("Tried to merge something that wasn't a struct: type %v was %v", - baseValue.Type(), baseValue.Kind()) - } - if baseValue.Type() != overlayValue.Type() { - return fmt.Errorf("Tried to merge two different types: %v and %v", - baseValue.Type(), overlayValue.Type()) - } - for _, field := range fields { - overlayFieldValue := findField(field, overlayValue) - if !overlayFieldValue.IsValid() { - return fmt.Errorf("could not find field in %v matching %v", - overlayValue.Type(), field) - } - baseFieldValue := findField(field, baseValue) - baseFieldValue.Set(overlayFieldValue) - } - return nil -} - func (c *Config) LoadDirectory(path string) error { directoryEntries, err := ioutil.ReadDir(path) if err != nil { @@ -432,13 +457,10 @@ func (c *Config) LoadDirectory(path string) error { if err != nil { return err } - } return nil } -// hazmat area. Keeping the ast parsing here. - // LoadConfig loads the given config file and returns a *Config pointer func (c *Config) LoadConfig(path string) error { data, err := ioutil.ReadFile(path) @@ -459,11 +481,7 @@ func (c *Config) LoadConfig(path string) error { switch name { case "agent": - err := c.parseAgent(subTable) - if err != nil { - log.Printf("Could not parse [agent] config\n") - return err - } + c.agent = subTable case "tags": if err = toml.UnmarshalTable(subTable, c.Tags); err != nil { log.Printf("Could not parse [tags] config\n") @@ -473,20 +491,11 @@ func (c *Config) LoadConfig(path string) error { for outputName, outputVal := range subTable.Fields { switch outputSubTable := outputVal.(type) { case *ast.Table: - err = c.parseOutput(outputName, outputSubTable, 0) - if err != nil { - log.Printf("Could not parse config for output: %s\n", - outputName) - return err - } + c.outputs[outputName] = outputSubTable case []*ast.Table: for id, t := range outputSubTable { - err = c.parseOutput(outputName, t, id) - if err != nil { - log.Printf("Could not parse config for output: %s\n", - outputName) - return err - } + nameID := fmt.Sprintf("%s-%d", outputName, id) + c.outputs[nameID] = t } default: return fmt.Errorf("Unsupported config format: %s", @@ -497,20 +506,11 @@ func (c *Config) LoadConfig(path string) error { for pluginName, pluginVal := range subTable.Fields { switch pluginSubTable := pluginVal.(type) { case *ast.Table: - err = c.parsePlugin(pluginName, pluginSubTable, 0) - if err != nil { - log.Printf("Could not parse config for plugin: %s\n", - pluginName) - return err - } + c.plugins[pluginName] = pluginSubTable case []*ast.Table: for id, t := range pluginSubTable { - err = c.parsePlugin(pluginName, t, id) - if err != nil { - log.Printf("Could not parse config for plugin: %s\n", - pluginName) - return err - } + nameID := fmt.Sprintf("%s-%d", pluginName, id) + c.plugins[nameID] = t } default: return fmt.Errorf("Unsupported config format: %s", @@ -520,155 +520,8 @@ func (c *Config) LoadConfig(path string) error { // Assume it's a plugin for legacy config file support if no other // identifiers are present default: - err = c.parsePlugin(name, subTable, 0) - if err != nil { - return err - } + c.plugins[name] = subTable } } - - return nil -} - -// Needs to have the field names, for merging later. -func extractFieldNames(ast *ast.Table) []string { - // A reasonable capacity? - var names []string - for name := range ast.Fields { - names = append(names, name) - } - return names -} - -// Parse the agent config out of the given *ast.Table. -func (c *Config) parseAgent(agentAst *ast.Table) error { - c.agentFieldsSet = extractFieldNames(agentAst) - agent := &Agent{} - err := toml.UnmarshalTable(agentAst, agent) - if err != nil { - return err - } - c.agent = agent - return nil -} - -// Parse an output config out of the given *ast.Table. -func (c *Config) parseOutput(name string, outputAst *ast.Table, id int) error { - c.outputFieldsSet[name] = extractFieldNames(outputAst) - creator, ok := outputs.Outputs[name] - if !ok { - return fmt.Errorf("Undefined but requested output: %s", name) - } - output := creator() - err := toml.UnmarshalTable(outputAst, output) - if err != nil { - return err - } - c.outputs[fmt.Sprintf("%s-%d", name, id)] = output - return nil -} - -// Parse a plugin config, plus plugin meta-config, out of the given *ast.Table. -func (c *Config) parsePlugin(name string, pluginAst *ast.Table, id int) error { - creator, ok := plugins.Plugins[name] - if !ok { - return fmt.Errorf("Undefined but requested plugin: %s", name) - } - plugin := creator() - cp := &ConfiguredPlugin{Name: name} - cpFields := make([]string, 0, 5) - - if node, ok := pluginAst.Fields["pass"]; ok { - if kv, ok := node.(*ast.KeyValue); ok { - if ary, ok := kv.Value.(*ast.Array); ok { - for _, elem := range ary.Value { - if str, ok := elem.(*ast.String); ok { - cp.Pass = append(cp.Pass, str.Value) - } - } - cpFields = append(cpFields, "pass") - } - } - } - - if node, ok := pluginAst.Fields["drop"]; ok { - if kv, ok := node.(*ast.KeyValue); ok { - if ary, ok := kv.Value.(*ast.Array); ok { - for _, elem := range ary.Value { - if str, ok := elem.(*ast.String); ok { - cp.Drop = append(cp.Drop, str.Value) - } - } - cpFields = append(cpFields, "drop") - } - } - } - - if node, ok := pluginAst.Fields["interval"]; ok { - if kv, ok := node.(*ast.KeyValue); ok { - if str, ok := kv.Value.(*ast.String); ok { - dur, err := time.ParseDuration(str.Value) - if err != nil { - return err - } - - cp.Interval = dur - cpFields = append(cpFields, "interval") - } - } - } - - if node, ok := pluginAst.Fields["tagpass"]; ok { - if subtbl, ok := node.(*ast.Table); ok { - for name, val := range subtbl.Fields { - if kv, ok := val.(*ast.KeyValue); ok { - tagfilter := &TagFilter{Name: name} - if ary, ok := kv.Value.(*ast.Array); ok { - for _, elem := range ary.Value { - if str, ok := elem.(*ast.String); ok { - tagfilter.Filter = append(tagfilter.Filter, str.Value) - } - } - } - cp.TagPass = append(cp.TagPass, *tagfilter) - cpFields = append(cpFields, "tagpass") - } - } - } - } - - if node, ok := pluginAst.Fields["tagdrop"]; ok { - if subtbl, ok := node.(*ast.Table); ok { - for name, val := range subtbl.Fields { - if kv, ok := val.(*ast.KeyValue); ok { - tagfilter := &TagFilter{Name: name} - if ary, ok := kv.Value.(*ast.Array); ok { - for _, elem := range ary.Value { - if str, ok := elem.(*ast.String); ok { - tagfilter.Filter = append(tagfilter.Filter, str.Value) - } - } - } - cp.TagDrop = append(cp.TagDrop, *tagfilter) - cpFields = append(cpFields, "tagdrop") - } - } - } - } - - delete(pluginAst.Fields, "drop") - delete(pluginAst.Fields, "pass") - delete(pluginAst.Fields, "interval") - delete(pluginAst.Fields, "tagdrop") - delete(pluginAst.Fields, "tagpass") - nameID := fmt.Sprintf("%s-%d", name, id) - c.pluginFieldsSet[nameID] = extractFieldNames(pluginAst) - c.pluginConfigurationFieldsSet[nameID] = cpFields - err := toml.UnmarshalTable(pluginAst, plugin) - if err != nil { - return err - } - c.plugins[nameID] = plugin - c.pluginConfigurations[nameID] = cp return nil }