Cleanup & standardize config file

changes:

- -sample-config will now comment out all but a few default plugins.
- config file parse errors will output path to bad conf file.
- cleanup 80-char line-length and some other style issues.
- default package conf file will now have all plugins, but commented
  out.

closes #199
closes #944

internal/config/config.go

@@ -22,6 +22,15 @@ import (
 	"github.com/influxdata/toml/ast"
 )
 
+var (
+	// Default input plugins
+	inputDefaults = []string{"cpu", "mem", "swap", "system", "kernel",
+		"processes", "disk", "diskio"}
+
+	// Default output plugins
+	outputDefaults = []string{"influxdb"}
+)
+
 // Config specifies the URL/user/password for the database that telegraf
 // will be logging to, as well as all the plugins that the user has
 // specified
@@ -135,21 +144,23 @@ func (c *Config) ListTags() string {
 }
 
 var header = `# Telegraf Configuration
-
+#
 # Telegraf is entirely plugin driven. All metrics are gathered from the
 # declared inputs, and sent to the declared outputs.
-
+#
 # Plugins must be declared in here to be active.
 # To deactivate a plugin, comment out the name and any variables.
-
+#
 # Use 'telegraf -config telegraf.conf -test' to see what metrics a config
 # file would generate.
 
+
 # Global tags can be specified here in key="value" format.
 [global_tags]
   # dc = "us-east-1" # will tag all metrics with dc=us-east-1
   # rack = "1a"
 
+
 # Configuration for telegraf agent
 [agent]
   ## Default data collection interval for all inputs
@@ -188,55 +199,72 @@ var header = `# Telegraf Configuration
   omit_hostname = false
 
 
-#
+###############################################################################
-# OUTPUTS:
+#                            OUTPUT PLUGINS                                   #
-#
+###############################################################################
-
 `
 
-var pluginHeader = `
+var inputHeader = `
-#
+
-# INPUTS:
+###############################################################################
-#
+#                            INPUT PLUGINS                                    #
+###############################################################################
 `
 
 var serviceInputHeader = `
-#
+
-# SERVICE INPUTS:
+###############################################################################
-#
+#                            SERVICE INPUT PLUGINS                            #
+###############################################################################
 `
 
 // PrintSampleConfig prints the sample config
-func PrintSampleConfig(pluginFilters []string, outputFilters []string) {
+func PrintSampleConfig(inputFilters []string, outputFilters []string) {
 	fmt.Printf(header)
 
-	// Filter outputs
+	if len(outputFilters) != 0 {
-	var onames []string
+		printFilteredOutputs(outputFilters, false)
-	for oname := range outputs.Outputs {
+	} else {
-		if len(outputFilters) == 0 || sliceContains(oname, outputFilters) {
+		printFilteredOutputs(outputDefaults, false)
-			onames = append(onames, oname)
+		// Print non-default outputs, commented
+		var pnames []string
+		for pname := range outputs.Outputs {
+			if !sliceContains(pname, outputDefaults) {
+				pnames = append(pnames, pname)
+			}
 		}
-	}
+		sort.Strings(pnames)
-	sort.Strings(onames)
+		printFilteredOutputs(pnames, true)
-
-	// Print Outputs
-	for _, oname := range onames {
-		creator := outputs.Outputs[oname]
-		output := creator()
-		printConfig(oname, output, "outputs")
 	}
 
+	fmt.Printf(inputHeader)
+	if len(inputFilters) != 0 {
+		printFilteredInputs(inputFilters, false)
+	} else {
+		printFilteredInputs(inputDefaults, false)
+		// Print non-default inputs, commented
+		var pnames []string
+		for pname := range inputs.Inputs {
+			if !sliceContains(pname, inputDefaults) {
+				pnames = append(pnames, pname)
+			}
+		}
+		sort.Strings(pnames)
+		printFilteredInputs(pnames, true)
+	}
+}
+
+func printFilteredInputs(inputFilters []string, commented bool) {
 	// Filter inputs
 	var pnames []string
 	for pname := range inputs.Inputs {
-		if len(pluginFilters) == 0 || sliceContains(pname, pluginFilters) {
+		if sliceContains(pname, inputFilters) {
 			pnames = append(pnames, pname)
 		}
 	}
 	sort.Strings(pnames)
 
 	// Print Inputs
-	fmt.Printf(pluginHeader)
 	servInputs := make(map[string]telegraf.ServiceInput)
 	for _, pname := range pnames {
 		creator := inputs.Inputs[pname]
@@ -248,13 +276,34 @@ func PrintSampleConfig(pluginFilters []string, outputFilters []string) {
 			continue
 		}
 
-		printConfig(pname, input, "inputs")
+		printConfig(pname, input, "inputs", commented)
 	}
 
 	// Print Service Inputs
+	if len(servInputs) == 0 {
+		return
+	}
 	fmt.Printf(serviceInputHeader)
 	for name, input := range servInputs {
-		printConfig(name, input, "inputs")
+		printConfig(name, input, "inputs", commented)
+	}
+}
+
+func printFilteredOutputs(outputFilters []string, commented bool) {
+	// Filter outputs
+	var onames []string
+	for oname := range outputs.Outputs {
+		if sliceContains(oname, outputFilters) {
+			onames = append(onames, oname)
+		}
+	}
+	sort.Strings(onames)
+
+	// Print Outputs
+	for _, oname := range onames {
+		creator := outputs.Outputs[oname]
+		output := creator()
+		printConfig(oname, output, "outputs", commented)
 	}
 }
 
@@ -263,13 +312,26 @@ type printer interface {
 	SampleConfig() string
 }
 
-func printConfig(name string, p printer, op string) {
+func printConfig(name string, p printer, op string, commented bool) {
-	fmt.Printf("\n# %s\n[[%s.%s]]", p.Description(), op, name)
+	comment := ""
+	if commented {
+		comment = "# "
+	}
+	fmt.Printf("\n%s# %s\n%s[[%s.%s]]", comment, p.Description(), comment,
+		op, name)
+
 	config := p.SampleConfig()
 	if config == "" {
-		fmt.Printf("\n  # no configuration\n")
+		fmt.Printf("\n%s  # no configuration\n\n", comment)
 	} else {
-		fmt.Printf(config)
+		lines := strings.Split(config, "\n")
+		for i, line := range lines {
+			if i == 0 || i == len(lines)-1 {
+				fmt.Print("\n")
+				continue
+			}
+			fmt.Print(comment + line + "\n")
+		}
 	}
 }
 
@@ -285,7 +347,7 @@ func sliceContains(name string, list []string) bool {
 // PrintInputConfig prints the config usage of a single input.
 func PrintInputConfig(name string) error {
 	if creator, ok := inputs.Inputs[name]; ok {
-		printConfig(name, creator(), "inputs")
+		printConfig(name, creator(), "inputs", false)
 	} else {
 		return errors.New(fmt.Sprintf("Input %s not found", name))
 	}
@@ -295,7 +357,7 @@ func PrintInputConfig(name string) error {
 // PrintOutputConfig prints the config usage of a single output.
 func PrintOutputConfig(name string) error {
 	if creator, ok := outputs.Outputs[name]; ok {
-		printConfig(name, creator(), "outputs")
+		printConfig(name, creator(), "outputs", false)
 	} else {
 		return errors.New(fmt.Sprintf("Output %s not found", name))
 	}
@@ -327,42 +389,42 @@ func (c *Config) LoadDirectory(path string) error {
 func (c *Config) LoadConfig(path string) error {
 	tbl, err := config.ParseFile(path)
 	if err != nil {
-		return err
+		return fmt.Errorf("Error parsing %s, %s", path, err)
 	}
 
 	for name, val := range tbl.Fields {
 		subTable, ok := val.(*ast.Table)
 		if !ok {
-			return errors.New("invalid configuration")
+			return fmt.Errorf("%s: invalid configuration", path)
 		}
 
 		switch name {
 		case "agent":
 			if err = config.UnmarshalTable(subTable, c.Agent); err != nil {
 				log.Printf("Could not parse [agent] config\n")
-				return err
+				return fmt.Errorf("Error parsing %s, %s", path, err)
 			}
 		case "global_tags", "tags":
 			if err = config.UnmarshalTable(subTable, c.Tags); err != nil {
 				log.Printf("Could not parse [global_tags] config\n")
-				return err
+				return fmt.Errorf("Error parsing %s, %s", path, err)
 			}
 		case "outputs":
 			for pluginName, pluginVal := range subTable.Fields {
 				switch pluginSubTable := pluginVal.(type) {
 				case *ast.Table:
 					if err = c.addOutput(pluginName, pluginSubTable); err != nil {
-						return err
+						return fmt.Errorf("Error parsing %s, %s", path, err)
 					}
 				case []*ast.Table:
 					for _, t := range pluginSubTable {
 						if err = c.addOutput(pluginName, t); err != nil {
-							return err
+							return fmt.Errorf("Error parsing %s, %s", path, err)
 						}
 					}
 				default:
-					return fmt.Errorf("Unsupported config format: %s",
+					return fmt.Errorf("Unsupported config format: %s, file %s",
-						pluginName)
+						pluginName, path)
 				}
 			}
 		case "inputs", "plugins":
@@ -370,24 +432,24 @@ func (c *Config) LoadConfig(path string) error {
 				switch pluginSubTable := pluginVal.(type) {
 				case *ast.Table:
 					if err = c.addInput(pluginName, pluginSubTable); err != nil {
-						return err
+						return fmt.Errorf("Error parsing %s, %s", path, err)
 					}
 				case []*ast.Table:
 					for _, t := range pluginSubTable {
 						if err = c.addInput(pluginName, t); err != nil {
-							return err
+							return fmt.Errorf("Error parsing %s, %s", path, err)
 						}
 					}
 				default:
-					return fmt.Errorf("Unsupported config format: %s",
+					return fmt.Errorf("Unsupported config format: %s, file %s",
-						pluginName)
+						pluginName, path)
 				}
 			}
 		// Assume it's an input input for legacy config file support if no other
 		// identifiers are present
 		default:
 			if err = c.addInput(name, subTable); err != nil {
-				return err
+				return fmt.Errorf("Error parsing %s, %s", path, err)
 			}
 		}
 	}

plugins/inputs/disque/disque.go

@@ -24,9 +24,8 @@ type Disque struct {
 
 var sampleConfig = `
   ## An array of URI to gather stats about. Specify an ip or hostname
-  ## with optional port and password. ie disque://localhost, disque://10.10.3.33:18832,
+  ## with optional port and password.
-  ## 10.0.0.1:10000, etc.
+  ## ie disque://localhost, disque://10.10.3.33:18832, 10.0.0.1:10000, etc.
-
   ## If no servers are specified, then localhost is used as the host.
   servers = ["localhost"]
 `

plugins/inputs/dns_query/dns_query.go

@@ -35,7 +35,8 @@ var sampleConfig = `
   ## Domains or subdomains to query. "."(root) is default
   domains = ["."] # optional
 
-  ## Query record type. Posible values: A, AAAA, CNAME, MX,  NS, PTR, TXT, SOA, SPF, SRV. Default is "NS"
+  ## Query record type. Default is "A"
+  ## Posible values: A, AAAA, CNAME, MX, NS, PTR, TXT, SOA, SPF, SRV.
   record_type = "A" # optional
 
   ## Dns server port. 53 is default

plugins/inputs/exec/exec.go

@@ -22,7 +22,7 @@ const sampleConfig = `
   ## measurement name suffix (for separating different commands)
   name_suffix = "_mycollector"
 
-  ## Data format to consume. This can be "json", "influx", "graphite" or "nagios
+  ## Data format to consume.
   ## Each data format has it's own unique set of configuration options, read
   ## more about them here:
   ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md

plugins/inputs/kafka_consumer/kafka_consumer.go

@@ -56,7 +56,7 @@ var sampleConfig = `
   ## Offset (must be either "oldest" or "newest")
   offset = "oldest"
 
-  ## Data format to consume. This can be "json", "influx" or "graphite"
+  ## Data format to consume.
   ## Each data format has it's own unique set of configuration options, read
   ## more about them here:
   ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md

plugins/inputs/mesos/mesos.go

@@ -34,7 +34,16 @@ var sampleConfig = `
   # A list of Mesos masters, default value is localhost:5050.
   masters = ["localhost:5050"]
   # Metrics groups to be collected, by default, all enabled.
-  master_collections = ["resources","master","system","slaves","frameworks","messages","evqueue","registrar"]
+  master_collections = [
+    "resources",
+    "master",
+    "system",
+    "slaves",
+    "frameworks",
+    "messages",
+    "evqueue",
+    "registrar",
+  ]
 `
 
 // SampleConfig returns a sample configuration block

plugins/inputs/mqtt_consumer/mqtt_consumer.go

@@ -78,7 +78,7 @@ var sampleConfig = `
   ## Use SSL but skip chain & host verification
   # insecure_skip_verify = false
 
-  ## Data format to consume. This can be "json", "influx" or "graphite"
+  ## Data format to consume.
   ## Each data format has it's own unique set of configuration options, read
   ## more about them here:
   ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md

plugins/inputs/nats_consumer/nats_consumer.go

@@ -55,7 +55,7 @@ var sampleConfig = `
   ## name a queue group
   queue_group = "telegraf_consumers"
 
-  ## Data format to consume. This can be "json", "influx" or "graphite"
+  ## Data format to consume.
   ## Each data format has it's own unique set of configuration options, read
   ## more about them here:
   ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md

plugins/inputs/postgresql/postgresql.go

@@ -26,7 +26,8 @@ var ignoredColumns = map[string]bool{"datid": true, "datname": true, "stats_rese
 
 var sampleConfig = `
   ## specify address via a url matching:
-  ##   postgres://[pqgotest[:password]]@localhost[/dbname]?sslmode=[disable|verify-ca|verify-full]
+  ##   postgres://[pqgotest[:password]]@localhost[/dbname]\
+  ##       ?sslmode=[disable|verify-ca|verify-full]
   ## or a simple string:
   ##   host=localhost user=pqotest password=... sslmode=... dbname=app_production
   ##

plugins/inputs/postgresql_extensible/postgresql_extensible.go

@@ -38,38 +38,41 @@ type query []struct {
 var ignoredColumns = map[string]bool{"datid": true, "datname": true, "stats_reset": true}
 
 var sampleConfig = `
-  # specify address via a url matching:
+  ## specify address via a url matching:
-  #   postgres://[pqgotest[:password]]@localhost[/dbname]?sslmode=[disable|verify-ca|verify-full]
+  ##   postgres://[pqgotest[:password]]@localhost[/dbname]\
-  # or a simple string:
+  ##       ?sslmode=[disable|verify-ca|verify-full]
-  #   host=localhost user=pqotest password=... sslmode=... dbname=app_production
+  ## or a simple string:
+  ##   host=localhost user=pqotest password=... sslmode=... dbname=app_production
   #
-  # All connection parameters are optional.  #
+  ## All connection parameters are optional.  #
-  # Without the dbname parameter, the driver will default to a database
+  ## Without the dbname parameter, the driver will default to a database
-  # with the same name as the user. This dbname is just for instantiating a
+  ## with the same name as the user. This dbname is just for instantiating a
-  # connection with the server and doesn't restrict the databases we are trying
+  ## connection with the server and doesn't restrict the databases we are trying
-  # to grab metrics for.
+  ## to grab metrics for.
   #
   address = "host=localhost user=postgres sslmode=disable"
-  # A list of databases to pull metrics about. If not specified, metrics for all
+  ## A list of databases to pull metrics about. If not specified, metrics for all
-  # databases are gathered.
+  ## databases are gathered.
-  # databases = ["app_production", "testing"]
+  ## databases = ["app_production", "testing"]
   #
-  # Define the toml config where the sql queries are stored
+  ## Define the toml config where the sql queries are stored
-  # New queries can be added, if the withdbname is set to true and there is no databases defined
+  ## New queries can be added, if the withdbname is set to true and there is no
-  # in the 'databases field', the sql query is ended by a 'is not null' in order to make the query
+  ## databases defined in the 'databases field', the sql query is ended by a
-  # succeed.
+  ## 'is not null' in order to make the query succeed.
-  # Example :
+  ## Example :
-  # The sqlquery : "SELECT * FROM pg_stat_database where datname" become "SELECT * FROM pg_stat_database where datname IN ('postgres', 'pgbench')"
+  ## The sqlquery : "SELECT * FROM pg_stat_database where datname" become
-  # because the databases variable was set to ['postgres', 'pgbench' ] and the withdbname was true.
+  ## "SELECT * FROM pg_stat_database where datname IN ('postgres', 'pgbench')"
-  # Be careful that if the withdbname is set to false you d'ont have to define the where clause (aka with the dbname)
+  ## because the databases variable was set to ['postgres', 'pgbench' ] and the
-  # the tagvalue field is used to define custom tags (separated by comas)
+  ## withdbname was true. Be careful that if the withdbname is set to false you
+  ## don't have to define the where clause (aka with the dbname) the tagvalue
+  ## field is used to define custom tags (separated by comas)
   #
-  # Structure :
+  ## Structure :
-  # [[inputs.postgresql_extensible.query]]
+  ## [[inputs.postgresql_extensible.query]]
-  #   sqlquery string
+  ##   sqlquery string
-  #   version string
+  ##   version string
-  #   withdbname boolean
+  ##   withdbname boolean
-  #   tagvalue string (coma separated)
+  ##   tagvalue string (coma separated)
   [[inputs.postgresql_extensible.query]]
     sqlquery="SELECT * FROM pg_stat_database"
     version=901

plugins/inputs/prometheus/prometheus.go

@@ -26,10 +26,10 @@ var sampleConfig = `
   ## An array of urls to scrape metrics from.
   urls = ["http://localhost:9100/metrics"]
 
-	### Use SSL but skip chain & host verification
+  ## Use SSL but skip chain & host verification
-	# insecure_skip_verify = false
+  # insecure_skip_verify = false
-	### Use bearer token for authorization
+  ## Use bearer token for authorization
-	# bearer_token = /path/to/bearer/token
+  # bearer_token = /path/to/bearer/token
 `
 
 func (p *Prometheus) SampleConfig() string {

plugins/inputs/snmp/snmp.go

@@ -178,7 +178,6 @@ var sampleConfig = `
     max_repetition = 127
     oid = "ifOutOctets"
 
-
   [[inputs.snmp.host]]
     address = "192.168.2.13:161"
     #address = "127.0.0.1:161"
@@ -221,8 +220,6 @@ var sampleConfig = `
     # if empty get all subtables
     # sub_tables could be not "real subtables"
     sub_tables=[".1.3.6.1.2.1.2.2.1.13", "bytes_recv", "bytes_send"]
-
-
 `
 
 // SampleConfig returns sample configuration message

plugins/inputs/tcp_listener/tcp_listener.go

@@ -53,7 +53,7 @@ const sampleConfig = `
   ## Maximum number of concurrent TCP connections to allow
   max_tcp_connections = 250
 
-  ## Data format to consume. This can be "json", "influx" or "graphite"
+  ## Data format to consume.
   ## Each data format has it's own unique set of configuration options, read
   ## more about them here:
   ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md

plugins/inputs/udp_listener/udp_listener.go

@@ -48,7 +48,7 @@ const sampleConfig = `
   ## usually 1500 bytes, but can be as large as 65,535 bytes.
   udp_packet_size = 1500
 
-  ## Data format to consume. This can be "json", "influx" or "graphite"
+  ## Data format to consume.
   ## Each data format has it's own unique set of configuration options, read
   ## more about them here:
   ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md

plugins/outputs/amqp/amqp.go

@@ -89,7 +89,7 @@ var sampleConfig = `
   ## Use SSL but skip chain & host verification
   # insecure_skip_verify = false
 
-  ## Data format to output. This can be "influx" or "graphite"
+  ## Data format to output.
   ## Each data format has it's own unique set of configuration options, read
   ## more about them here:
   ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_OUTPUT.md

plugins/outputs/file/file.go

@@ -23,7 +23,7 @@ var sampleConfig = `
   ## Files to write to, "stdout" is a specially handled file.
   files = ["stdout", "/tmp/metrics.out"]
 
-  ## Data format to output. This can be "influx" or "graphite"
+  ## Data format to output.
   ## Each data format has it's own unique set of configuration options, read
   ## more about them here:
   ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_OUTPUT.md

plugins/outputs/kafka/kafka.go

@@ -59,16 +59,27 @@ var sampleConfig = `
   ##  ie, if this tag exists, it's value will be used as the routing key
   routing_tag = "host"
 
-  ## CompressionCodec represents the various compression codecs recognized by Kafka in messages.
+  ## CompressionCodec represents the various compression codecs recognized by
+  ## Kafka in messages.
   ##  0 : No compression
   ##  1 : Gzip compression
   ##  2 : Snappy compression
   compression_codec = 0
 
-  ##  RequiredAcks is used in Produce Requests to tell the broker how many replica acknowledgements it must see before responding
+  ##  RequiredAcks is used in Produce Requests to tell the broker how many
-  ##  0 : the producer never waits for an acknowledgement from the broker. This option provides the lowest latency but the weakest durability guarantees (some data will be lost when a server fails).
+  ##  replica acknowledgements it must see before responding
-  ##  1 : the producer gets an acknowledgement after the leader replica has received the data. This option provides better durability as the client waits until the server acknowledges the request as successful (only messages that were written to the now-dead leader but not yet replicated will be lost).
+  ##   0 : the producer never waits for an acknowledgement from the broker.
-  ##  -1 : the producer gets an acknowledgement after all in-sync replicas have received the data. This option provides the best durability, we guarantee that no messages will be lost as long as at least one in sync replica remains.
+  ##       This option provides the lowest latency but the weakest durability
+  ##       guarantees (some data will be lost when a server fails).
+  ##   1 : the producer gets an acknowledgement after the leader replica has
+  ##       received the data. This option provides better durability as the
+  ##       client waits until the server acknowledges the request as successful
+  ##       (only messages that were written to the now-dead leader but not yet
+  ##       replicated will be lost).
+  ##   -1: the producer gets an acknowledgement after all in-sync replicas have
+  ##       received the data. This option provides the best durability, we
+  ##       guarantee that no messages will be lost as long as at least one in
+  ##       sync replica remains.
   required_acks = -1
 
   ##  The total number of times to retry sending a message
@@ -81,7 +92,7 @@ var sampleConfig = `
   ## Use SSL but skip chain & host verification
   # insecure_skip_verify = false
 
-  ## Data format to output. This can be "influx" or "graphite"
+  ## Data format to output.
   ## Each data format has it's own unique set of configuration options, read
   ## more about them here:
   ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_OUTPUT.md

plugins/outputs/mqtt/mqtt.go

@@ -32,7 +32,7 @@ var sampleConfig = `
   ## Use SSL but skip chain & host verification
   # insecure_skip_verify = false
 
-  ## Data format to output. This can be "influx" or "graphite"
+  ## Data format to output.
   ## Each data format has it's own unique set of configuration options, read
   ## more about them here:
   ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_OUTPUT.md

plugins/outputs/nsq/nsq.go

@@ -24,7 +24,7 @@ var sampleConfig = `
   ## NSQ topic for producer messages
   topic = "telegraf"
 
-  ## Data format to output. This can be "influx" or "graphite"
+  ## Data format to output.
   ## Each data format has it's own unique set of configuration options, read
   ## more about them here:
   ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_OUTPUT.md