config: parse environment variables in the config file

closes #663
This commit is contained in:
Cameron Sparr 2016-04-01 13:53:34 -06:00
parent 9211d22b2b
commit 8e041420cd
6 changed files with 132 additions and 25 deletions

View File

@ -1,6 +1,7 @@
## v0.12.0 [unreleased] ## v0.12.0 [unreleased]
### Features ### Features
- [#951](https://github.com/influxdata/telegraf/pull/951): Parse environment variables in the config file.
- [#948](https://github.com/influxdata/telegraf/pull/948): Cleanup config file and make default package version include all plugins (but commented). - [#948](https://github.com/influxdata/telegraf/pull/948): Cleanup config file and make default package version include all plugins (but commented).
- [#927](https://github.com/influxdata/telegraf/pull/927): Adds parsing of tags to the statsd input when using DataDog's dogstatsd extension - [#927](https://github.com/influxdata/telegraf/pull/927): Adds parsing of tags to the statsd input when using DataDog's dogstatsd extension
- [#863](https://github.com/influxdata/telegraf/pull/863): AMQP output: allow external auth. Thanks @ekini! - [#863](https://github.com/influxdata/telegraf/pull/863): AMQP output: allow external auth. Thanks @ekini!

View File

@ -9,6 +9,12 @@ To generate a file with specific inputs and outputs, you can use the
-input-filter and -output-filter flags: -input-filter and -output-filter flags:
`telegraf -sample-config -input-filter cpu:mem:net:swap -output-filter influxdb:kafka` `telegraf -sample-config -input-filter cpu:mem:net:swap -output-filter influxdb:kafka`
## Environment Variables
Environment variables can be used anywhere in the config file, simply prepend
them with $. For strings the variable must be within quotes (ie, "$STR_VAR"),
for numbers and booleans they should be plain (ie, $INT_VAR, $BOOL_VAR)
## `[global_tags]` Configuration ## `[global_tags]` Configuration
Global tags can be specific in the `[global_tags]` section of the config file in Global tags can be specific in the `[global_tags]` section of the config file in

View File

@ -8,12 +8,18 @@
# #
# Use 'telegraf -config telegraf.conf -test' to see what metrics a config # Use 'telegraf -config telegraf.conf -test' to see what metrics a config
# file would generate. # file would generate.
#
# Environment variables can be used anywhere in this config file, simply prepend
# them with $. For strings the variable must be within quotes (ie, "$STR_VAR"),
# for numbers and booleans they should be plain (ie, $INT_VAR, $BOOL_VAR)
# Global tags can be specified here in key="value" format. # Global tags can be specified here in key="value" format.
[global_tags] [global_tags]
# dc = "us-east-1" # will tag all metrics with dc=us-east-1 # dc = "us-east-1" # will tag all metrics with dc=us-east-1
# rack = "1a" # rack = "1a"
## Environment variables can be used as tags, and throughout the config file
# user = "$USER"
# Configuration for telegraf agent # Configuration for telegraf agent
@ -1114,27 +1120,6 @@
# SERVICE INPUT PLUGINS # # SERVICE INPUT PLUGINS #
############################################################################### ###############################################################################
# # Generic UDP listener
# [[inputs.udp_listener]]
# ## Address and port to host UDP listener on
# service_address = ":8092"
#
# ## Number of UDP messages allowed to queue up. Once filled, the
# ## UDP listener will start dropping packets.
# allowed_pending_messages = 10000
#
# ## UDP packet size for the server to listen for. This will depend
# ## on the size of the packets that the client is sending, which is
# ## usually 1500 bytes, but can be as large as 65,535 bytes.
# udp_packet_size = 1500
#
# ## Data format to consume.
# ## Each data format has it's own unique set of configuration options, read
# ## more about them here:
# ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md
# data_format = "influx"
# # A Github Webhook Event collector # # A Github Webhook Event collector
# [[inputs.github_webhooks]] # [[inputs.github_webhooks]]
# ## Address and port to host Webhook listener on # ## Address and port to host Webhook listener on
@ -1277,3 +1262,24 @@
# ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md # ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md
# data_format = "influx" # data_format = "influx"
# # Generic UDP listener
# [[inputs.udp_listener]]
# ## Address and port to host UDP listener on
# service_address = ":8092"
#
# ## Number of UDP messages allowed to queue up. Once filled, the
# ## UDP listener will start dropping packets.
# allowed_pending_messages = 10000
#
# ## UDP packet size for the server to listen for. This will depend
# ## on the size of the packets that the client is sending, which is
# ## usually 1500 bytes, but can be as large as 65,535 bytes.
# udp_packet_size = 1500
#
# ## Data format to consume.
# ## Each data format has it's own unique set of configuration options, read
# ## more about them here:
# ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md
# data_format = "influx"

View File

@ -1,11 +1,14 @@
package config package config
import ( import (
"bytes"
"errors" "errors"
"fmt" "fmt"
"io/ioutil" "io/ioutil"
"log" "log"
"os"
"path/filepath" "path/filepath"
"regexp"
"sort" "sort"
"strings" "strings"
"time" "time"
@ -19,6 +22,7 @@ import (
"github.com/influxdata/telegraf/plugins/serializers" "github.com/influxdata/telegraf/plugins/serializers"
"github.com/influxdata/config" "github.com/influxdata/config"
"github.com/influxdata/toml"
"github.com/influxdata/toml/ast" "github.com/influxdata/toml/ast"
) )
@ -29,6 +33,9 @@ var (
// Default output plugins // Default output plugins
outputDefaults = []string{"influxdb"} outputDefaults = []string{"influxdb"}
// envVarRe is a regex to find environment variables in the config file
envVarRe = regexp.MustCompile(`\$\w+`)
) )
// Config specifies the URL/user/password for the database that telegraf // Config specifies the URL/user/password for the database that telegraf
@ -153,12 +160,18 @@ var header = `# Telegraf Configuration
# #
# Use 'telegraf -config telegraf.conf -test' to see what metrics a config # Use 'telegraf -config telegraf.conf -test' to see what metrics a config
# file would generate. # file would generate.
#
# Environment variables can be used anywhere in this config file, simply prepend
# them with $. For strings the variable must be within quotes (ie, "$STR_VAR"),
# for numbers and booleans they should be plain (ie, $INT_VAR, $BOOL_VAR)
# Global tags can be specified here in key="value" format. # Global tags can be specified here in key="value" format.
[global_tags] [global_tags]
# dc = "us-east-1" # will tag all metrics with dc=us-east-1 # dc = "us-east-1" # will tag all metrics with dc=us-east-1
# rack = "1a" # rack = "1a"
## Environment variables can be used as tags, and throughout the config file
# user = "$USER"
# Configuration for telegraf agent # Configuration for telegraf agent
@ -264,8 +277,12 @@ func printFilteredInputs(inputFilters []string, commented bool) {
} }
sort.Strings(pnames) sort.Strings(pnames)
// Print Inputs // cache service inputs to print them at the end
servInputs := make(map[string]telegraf.ServiceInput) servInputs := make(map[string]telegraf.ServiceInput)
// for alphabetical looping:
servInputNames := []string{}
// Print Inputs
for _, pname := range pnames { for _, pname := range pnames {
creator := inputs.Inputs[pname] creator := inputs.Inputs[pname]
input := creator() input := creator()
@ -273,6 +290,7 @@ func printFilteredInputs(inputFilters []string, commented bool) {
switch p := input.(type) { switch p := input.(type) {
case telegraf.ServiceInput: case telegraf.ServiceInput:
servInputs[pname] = p servInputs[pname] = p
servInputNames = append(servInputNames, pname)
continue continue
} }
@ -283,9 +301,10 @@ func printFilteredInputs(inputFilters []string, commented bool) {
if len(servInputs) == 0 { if len(servInputs) == 0 {
return return
} }
sort.Strings(servInputNames)
fmt.Printf(serviceInputHeader) fmt.Printf(serviceInputHeader)
for name, input := range servInputs { for _, name := range servInputNames {
printConfig(name, input, "inputs", commented) printConfig(name, servInputs[name], "inputs", commented)
} }
} }
@ -387,7 +406,7 @@ func (c *Config) LoadDirectory(path string) error {
// LoadConfig loads the given config file and applies it to c // LoadConfig loads the given config file and applies it to c
func (c *Config) LoadConfig(path string) error { func (c *Config) LoadConfig(path string) error {
tbl, err := config.ParseFile(path) tbl, err := parseFile(path)
if err != nil { if err != nil {
return fmt.Errorf("Error parsing %s, %s", path, err) return fmt.Errorf("Error parsing %s, %s", path, err)
} }
@ -456,6 +475,26 @@ func (c *Config) LoadConfig(path string) error {
return nil return nil
} }
// parseFile loads a TOML configuration from a provided path and
// returns the AST produced from the TOML parser. When loading the file, it
// will find environment variables and replace them.
func parseFile(fpath string) (*ast.Table, error) {
contents, err := ioutil.ReadFile(fpath)
if err != nil {
return nil, err
}
env_vars := envVarRe.FindAll(contents, -1)
for _, env_var := range env_vars {
env_val := os.Getenv(strings.TrimPrefix(string(env_var), "$"))
if env_val != "" {
contents = bytes.Replace(contents, env_var, []byte(env_val), 1)
}
}
return toml.Parse(contents)
}
func (c *Config) addOutput(name string, table *ast.Table) error { func (c *Config) addOutput(name string, table *ast.Table) error {
if len(c.OutputFilters) > 0 && !sliceContains(name, c.OutputFilters) { if len(c.OutputFilters) > 0 && !sliceContains(name, c.OutputFilters) {
return nil return nil

View File

@ -1,6 +1,7 @@
package config package config
import ( import (
"os"
"testing" "testing"
"time" "time"
@ -10,9 +11,52 @@ import (
"github.com/influxdata/telegraf/plugins/inputs/memcached" "github.com/influxdata/telegraf/plugins/inputs/memcached"
"github.com/influxdata/telegraf/plugins/inputs/procstat" "github.com/influxdata/telegraf/plugins/inputs/procstat"
"github.com/influxdata/telegraf/plugins/parsers" "github.com/influxdata/telegraf/plugins/parsers"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
) )
func TestConfig_LoadSingleInputWithEnvVars(t *testing.T) {
c := NewConfig()
err := os.Setenv("MY_TEST_SERVER", "192.168.1.1")
assert.NoError(t, err)
err = os.Setenv("TEST_INTERVAL", "10s")
assert.NoError(t, err)
c.LoadConfig("./testdata/single_plugin_env_vars.toml")
memcached := inputs.Inputs["memcached"]().(*memcached.Memcached)
memcached.Servers = []string{"192.168.1.1"}
mConfig := &internal_models.InputConfig{
Name: "memcached",
Filter: internal_models.Filter{
NameDrop: []string{"metricname2"},
NamePass: []string{"metricname1"},
FieldDrop: []string{"other", "stuff"},
FieldPass: []string{"some", "strings"},
TagDrop: []internal_models.TagFilter{
internal_models.TagFilter{
Name: "badtag",
Filter: []string{"othertag"},
},
},
TagPass: []internal_models.TagFilter{
internal_models.TagFilter{
Name: "goodtag",
Filter: []string{"mytag"},
},
},
IsActive: true,
},
Interval: 10 * time.Second,
}
mConfig.Tags = make(map[string]string)
assert.Equal(t, memcached, c.Inputs[0].Input,
"Testdata did not produce a correct memcached struct.")
assert.Equal(t, mConfig, c.Inputs[0].Config,
"Testdata did not produce correct memcached metadata.")
}
func TestConfig_LoadSingleInput(t *testing.T) { func TestConfig_LoadSingleInput(t *testing.T) {
c := NewConfig() c := NewConfig()
c.LoadConfig("./testdata/single_plugin.toml") c.LoadConfig("./testdata/single_plugin.toml")

View File

@ -0,0 +1,11 @@
[[inputs.memcached]]
servers = ["$MY_TEST_SERVER"]
namepass = ["metricname1"]
namedrop = ["metricname2"]
fieldpass = ["some", "strings"]
fielddrop = ["other", "stuff"]
interval = "$TEST_INTERVAL"
[inputs.memcached.tagpass]
goodtag = ["mytag"]
[inputs.memcached.tagdrop]
badtag = ["othertag"]