From 8e041420cd9c34ae8a91e73ae62dab4d76937d0e Mon Sep 17 00:00:00 2001 From: Cameron Sparr Date: Fri, 1 Apr 2016 13:53:34 -0600 Subject: [PATCH] config: parse environment variables in the config file closes #663 --- CHANGELOG.md | 1 + docs/CONFIGURATION.md | 6 +++ etc/telegraf.conf | 48 +++++++++++-------- internal/config/config.go | 47 ++++++++++++++++-- internal/config/config_test.go | 44 +++++++++++++++++ .../testdata/single_plugin_env_vars.toml | 11 +++++ 6 files changed, 132 insertions(+), 25 deletions(-) create mode 100644 internal/config/testdata/single_plugin_env_vars.toml diff --git a/CHANGELOG.md b/CHANGELOG.md index 1095364f5..305fa6d03 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,7 @@ ## v0.12.0 [unreleased] ### Features +- [#951](https://github.com/influxdata/telegraf/pull/951): Parse environment variables in the config file. - [#948](https://github.com/influxdata/telegraf/pull/948): Cleanup config file and make default package version include all plugins (but commented). - [#927](https://github.com/influxdata/telegraf/pull/927): Adds parsing of tags to the statsd input when using DataDog's dogstatsd extension - [#863](https://github.com/influxdata/telegraf/pull/863): AMQP output: allow external auth. Thanks @ekini! diff --git a/docs/CONFIGURATION.md b/docs/CONFIGURATION.md index 810dc9470..0afaa120f 100644 --- a/docs/CONFIGURATION.md +++ b/docs/CONFIGURATION.md @@ -9,6 +9,12 @@ To generate a file with specific inputs and outputs, you can use the -input-filter and -output-filter flags: `telegraf -sample-config -input-filter cpu:mem:net:swap -output-filter influxdb:kafka` +## Environment Variables + +Environment variables can be used anywhere in the config file, simply prepend +them with $. For strings the variable must be within quotes (ie, "$STR_VAR"), +for numbers and booleans they should be plain (ie, $INT_VAR, $BOOL_VAR) + ## `[global_tags]` Configuration Global tags can be specific in the `[global_tags]` section of the config file in diff --git a/etc/telegraf.conf b/etc/telegraf.conf index 43d647beb..633483e22 100644 --- a/etc/telegraf.conf +++ b/etc/telegraf.conf @@ -8,12 +8,18 @@ # # Use 'telegraf -config telegraf.conf -test' to see what metrics a config # file would generate. +# +# Environment variables can be used anywhere in this config file, simply prepend +# them with $. For strings the variable must be within quotes (ie, "$STR_VAR"), +# for numbers and booleans they should be plain (ie, $INT_VAR, $BOOL_VAR) # Global tags can be specified here in key="value" format. [global_tags] # dc = "us-east-1" # will tag all metrics with dc=us-east-1 # rack = "1a" + ## Environment variables can be used as tags, and throughout the config file + # user = "$USER" # Configuration for telegraf agent @@ -1114,27 +1120,6 @@ # SERVICE INPUT PLUGINS # ############################################################################### -# # Generic UDP listener -# [[inputs.udp_listener]] -# ## Address and port to host UDP listener on -# service_address = ":8092" -# -# ## Number of UDP messages allowed to queue up. Once filled, the -# ## UDP listener will start dropping packets. -# allowed_pending_messages = 10000 -# -# ## UDP packet size for the server to listen for. This will depend -# ## on the size of the packets that the client is sending, which is -# ## usually 1500 bytes, but can be as large as 65,535 bytes. -# udp_packet_size = 1500 -# -# ## Data format to consume. -# ## Each data format has it's own unique set of configuration options, read -# ## more about them here: -# ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md -# data_format = "influx" - - # # A Github Webhook Event collector # [[inputs.github_webhooks]] # ## Address and port to host Webhook listener on @@ -1277,3 +1262,24 @@ # ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md # data_format = "influx" + +# # Generic UDP listener +# [[inputs.udp_listener]] +# ## Address and port to host UDP listener on +# service_address = ":8092" +# +# ## Number of UDP messages allowed to queue up. Once filled, the +# ## UDP listener will start dropping packets. +# allowed_pending_messages = 10000 +# +# ## UDP packet size for the server to listen for. This will depend +# ## on the size of the packets that the client is sending, which is +# ## usually 1500 bytes, but can be as large as 65,535 bytes. +# udp_packet_size = 1500 +# +# ## Data format to consume. +# ## Each data format has it's own unique set of configuration options, read +# ## more about them here: +# ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md +# data_format = "influx" + diff --git a/internal/config/config.go b/internal/config/config.go index 715fa777c..1e07234e8 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -1,11 +1,14 @@ package config import ( + "bytes" "errors" "fmt" "io/ioutil" "log" + "os" "path/filepath" + "regexp" "sort" "strings" "time" @@ -19,6 +22,7 @@ import ( "github.com/influxdata/telegraf/plugins/serializers" "github.com/influxdata/config" + "github.com/influxdata/toml" "github.com/influxdata/toml/ast" ) @@ -29,6 +33,9 @@ var ( // Default output plugins outputDefaults = []string{"influxdb"} + + // envVarRe is a regex to find environment variables in the config file + envVarRe = regexp.MustCompile(`\$\w+`) ) // Config specifies the URL/user/password for the database that telegraf @@ -153,12 +160,18 @@ var header = `# Telegraf Configuration # # Use 'telegraf -config telegraf.conf -test' to see what metrics a config # file would generate. +# +# Environment variables can be used anywhere in this config file, simply prepend +# them with $. For strings the variable must be within quotes (ie, "$STR_VAR"), +# for numbers and booleans they should be plain (ie, $INT_VAR, $BOOL_VAR) # Global tags can be specified here in key="value" format. [global_tags] # dc = "us-east-1" # will tag all metrics with dc=us-east-1 # rack = "1a" + ## Environment variables can be used as tags, and throughout the config file + # user = "$USER" # Configuration for telegraf agent @@ -264,8 +277,12 @@ func printFilteredInputs(inputFilters []string, commented bool) { } sort.Strings(pnames) - // Print Inputs + // cache service inputs to print them at the end servInputs := make(map[string]telegraf.ServiceInput) + // for alphabetical looping: + servInputNames := []string{} + + // Print Inputs for _, pname := range pnames { creator := inputs.Inputs[pname] input := creator() @@ -273,6 +290,7 @@ func printFilteredInputs(inputFilters []string, commented bool) { switch p := input.(type) { case telegraf.ServiceInput: servInputs[pname] = p + servInputNames = append(servInputNames, pname) continue } @@ -283,9 +301,10 @@ func printFilteredInputs(inputFilters []string, commented bool) { if len(servInputs) == 0 { return } + sort.Strings(servInputNames) fmt.Printf(serviceInputHeader) - for name, input := range servInputs { - printConfig(name, input, "inputs", commented) + for _, name := range servInputNames { + printConfig(name, servInputs[name], "inputs", commented) } } @@ -387,7 +406,7 @@ func (c *Config) LoadDirectory(path string) error { // LoadConfig loads the given config file and applies it to c func (c *Config) LoadConfig(path string) error { - tbl, err := config.ParseFile(path) + tbl, err := parseFile(path) if err != nil { return fmt.Errorf("Error parsing %s, %s", path, err) } @@ -456,6 +475,26 @@ func (c *Config) LoadConfig(path string) error { return nil } +// parseFile loads a TOML configuration from a provided path and +// returns the AST produced from the TOML parser. When loading the file, it +// will find environment variables and replace them. +func parseFile(fpath string) (*ast.Table, error) { + contents, err := ioutil.ReadFile(fpath) + if err != nil { + return nil, err + } + + env_vars := envVarRe.FindAll(contents, -1) + for _, env_var := range env_vars { + env_val := os.Getenv(strings.TrimPrefix(string(env_var), "$")) + if env_val != "" { + contents = bytes.Replace(contents, env_var, []byte(env_val), 1) + } + } + + return toml.Parse(contents) +} + func (c *Config) addOutput(name string, table *ast.Table) error { if len(c.OutputFilters) > 0 && !sliceContains(name, c.OutputFilters) { return nil diff --git a/internal/config/config_test.go b/internal/config/config_test.go index f0add8b98..d78a8d6b8 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -1,6 +1,7 @@ package config import ( + "os" "testing" "time" @@ -10,9 +11,52 @@ import ( "github.com/influxdata/telegraf/plugins/inputs/memcached" "github.com/influxdata/telegraf/plugins/inputs/procstat" "github.com/influxdata/telegraf/plugins/parsers" + "github.com/stretchr/testify/assert" ) +func TestConfig_LoadSingleInputWithEnvVars(t *testing.T) { + c := NewConfig() + err := os.Setenv("MY_TEST_SERVER", "192.168.1.1") + assert.NoError(t, err) + err = os.Setenv("TEST_INTERVAL", "10s") + assert.NoError(t, err) + c.LoadConfig("./testdata/single_plugin_env_vars.toml") + + memcached := inputs.Inputs["memcached"]().(*memcached.Memcached) + memcached.Servers = []string{"192.168.1.1"} + + mConfig := &internal_models.InputConfig{ + Name: "memcached", + Filter: internal_models.Filter{ + NameDrop: []string{"metricname2"}, + NamePass: []string{"metricname1"}, + FieldDrop: []string{"other", "stuff"}, + FieldPass: []string{"some", "strings"}, + TagDrop: []internal_models.TagFilter{ + internal_models.TagFilter{ + Name: "badtag", + Filter: []string{"othertag"}, + }, + }, + TagPass: []internal_models.TagFilter{ + internal_models.TagFilter{ + Name: "goodtag", + Filter: []string{"mytag"}, + }, + }, + IsActive: true, + }, + Interval: 10 * time.Second, + } + mConfig.Tags = make(map[string]string) + + assert.Equal(t, memcached, c.Inputs[0].Input, + "Testdata did not produce a correct memcached struct.") + assert.Equal(t, mConfig, c.Inputs[0].Config, + "Testdata did not produce correct memcached metadata.") +} + func TestConfig_LoadSingleInput(t *testing.T) { c := NewConfig() c.LoadConfig("./testdata/single_plugin.toml") diff --git a/internal/config/testdata/single_plugin_env_vars.toml b/internal/config/testdata/single_plugin_env_vars.toml new file mode 100644 index 000000000..6600a77b3 --- /dev/null +++ b/internal/config/testdata/single_plugin_env_vars.toml @@ -0,0 +1,11 @@ +[[inputs.memcached]] + servers = ["$MY_TEST_SERVER"] + namepass = ["metricname1"] + namedrop = ["metricname2"] + fieldpass = ["some", "strings"] + fielddrop = ["other", "stuff"] + interval = "$TEST_INTERVAL" + [inputs.memcached.tagpass] + goodtag = ["mytag"] + [inputs.memcached.tagdrop] + badtag = ["othertag"]