dom/telegraf
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

add support for streaming processors (#7634)

Browse Source
This commit is contained in:
Steven Soroka
2020-06-05 10:43:43 -04:00
committed by GitHub
parent b99e3bc63d
commit 741ea839d2
12 changed files with 913 additions and 579 deletions
Download Patch File Download Diff File Expand all files Collapse all files

109
config/config.go
View File

@@ -65,7 +65,8 @@ type Config struct {
Outputs []*models.RunningOutput
Aggregators []*models.RunningAggregator
// Processors have a slice wrapper type because they need to be sorted
Processors models.RunningProcessors
Processors models.RunningProcessors
AggProcessors models.RunningProcessors
}
func NewConfig() *Config {
@@ -83,6 +84,7 @@ func NewConfig() *Config {
Inputs: make([]*models.RunningInput, 0),
Outputs: make([]*models.RunningOutput, 0),
Processors: make([]*models.RunningProcessor, 0),
AggProcessors: make([]*models.RunningProcessor, 0),
InputFilters: make([]string, 0),
OutputFilters: make([]string, 0),
}
@@ -561,12 +563,7 @@ func printFilteredGlobalSections(sectionFilters []string) {
}
}
type printer interface {
Description() string
SampleConfig() string
}
func printConfig(name string, p printer, op string, commented bool) {
func printConfig(name string, p telegraf.PluginDescriber, op string, commented bool) {
comment := ""
if commented {
comment = "# "
@@ -684,12 +681,20 @@ func (c *Config) LoadConfig(path string) error {
}
data, err := loadConfig(path)
if err != nil {
return fmt.Errorf("Error loading %s, %s", path, err)
return fmt.Errorf("Error loading config file %s: %w", path, err)
}
if err = c.LoadConfigData(data); err != nil {
return fmt.Errorf("Error loading config file %s: %w", path, err)
}
return nil
}
// LoadConfigData loads TOML-formatted config data
func (c *Config) LoadConfigData(data []byte) error {
tbl, err := parseConfig(data)
if err != nil {
return fmt.Errorf("Error parsing %s, %s", path, err)
return fmt.Errorf("Error parsing data: %s", err)
}
// Parse tags tables first:
@@ -697,11 +702,10 @@ func (c *Config) LoadConfig(path string) error {
if val, ok := tbl.Fields[tableName]; ok {
subTable, ok := val.(*ast.Table)
if !ok {
return fmt.Errorf("%s: invalid configuration", path)
return fmt.Errorf("invalid configuration, bad table name %q", tableName)
}
if err = toml.UnmarshalTable(subTable, c.Tags); err != nil {
log.Printf("E! Could not parse [global_tags] config\n")
return fmt.Errorf("Error parsing %s, %s", path, err)
return fmt.Errorf("error parsing table name %q: %w", tableName, err)
}
}
}
@@ -710,11 +714,10 @@ func (c *Config) LoadConfig(path string) error {
if val, ok := tbl.Fields["agent"]; ok {
subTable, ok := val.(*ast.Table)
if !ok {
return fmt.Errorf("%s: invalid configuration", path)
return fmt.Errorf("invalid configuration, error parsing agent table")
}
if err = toml.UnmarshalTable(subTable, c.Agent); err != nil {
log.Printf("E! Could not parse [agent] config\n")
return fmt.Errorf("Error parsing %s, %s", path, err)
return fmt.Errorf("error parsing agent table: %w", err)
}
}
@@ -735,7 +738,7 @@ func (c *Config) LoadConfig(path string) error {
for name, val := range tbl.Fields {
subTable, ok := val.(*ast.Table)
if !ok {
return fmt.Errorf("%s: invalid configuration", path)
return fmt.Errorf("invalid configuration, error parsing field %q as table", name)
}
switch name {
@@ -746,17 +749,17 @@ func (c *Config) LoadConfig(path string) error {
// legacy [outputs.influxdb] support
case *ast.Table:
if err = c.addOutput(pluginName, pluginSubTable); err != nil {
return fmt.Errorf("Error parsing %s, %s", path, err)
return fmt.Errorf("Error parsing %s, %s", pluginName, err)
}
case []*ast.Table:
for _, t := range pluginSubTable {
if err = c.addOutput(pluginName, t); err != nil {
return fmt.Errorf("Error parsing %s, %s", path, err)
return fmt.Errorf("Error parsing %s array, %s", pluginName, err)
}
}
default:
return fmt.Errorf("Unsupported config format: %s, file %s",
pluginName, path)
return fmt.Errorf("Unsupported config format: %s",
pluginName)
}
}
case "inputs", "plugins":
@@ -765,17 +768,17 @@ func (c *Config) LoadConfig(path string) error {
// legacy [inputs.cpu] support
case *ast.Table:
if err = c.addInput(pluginName, pluginSubTable); err != nil {
return fmt.Errorf("Error parsing %s, %s", path, err)
return fmt.Errorf("Error parsing %s, %s", pluginName, err)
}
case []*ast.Table:
for _, t := range pluginSubTable {
if err = c.addInput(pluginName, t); err != nil {
return fmt.Errorf("Error parsing %s, %s", path, err)
return fmt.Errorf("Error parsing %s, %s", pluginName, err)
}
}
default:
return fmt.Errorf("Unsupported config format: %s, file %s",
pluginName, path)
return fmt.Errorf("Unsupported config format: %s",
pluginName)
}
}
case "processors":
@@ -784,12 +787,12 @@ func (c *Config) LoadConfig(path string) error {
case []*ast.Table:
for _, t := range pluginSubTable {
if err = c.addProcessor(pluginName, t); err != nil {
return fmt.Errorf("Error parsing %s, %s", path, err)
return fmt.Errorf("Error parsing %s, %s", pluginName, err)
}
}
default:
return fmt.Errorf("Unsupported config format: %s, file %s",
pluginName, path)
return fmt.Errorf("Unsupported config format: %s",
pluginName)
}
}
case "aggregators":
@@ -798,19 +801,19 @@ func (c *Config) LoadConfig(path string) error {
case []*ast.Table:
for _, t := range pluginSubTable {
if err = c.addAggregator(pluginName, t); err != nil {
return fmt.Errorf("Error parsing %s, %s", path, err)
return fmt.Errorf("Error parsing %s, %s", pluginName, err)
}
}
default:
return fmt.Errorf("Unsupported config format: %s, file %s",
pluginName, path)
return fmt.Errorf("Unsupported config format: %s",
pluginName)
}
}
// Assume it's an input input for legacy config file support if no other
// identifiers are present
default:
if err = c.addInput(name, subTable); err != nil {
return fmt.Errorf("Error parsing %s, %s", path, err)
return fmt.Errorf("Error parsing %s, %s", name, err)
}
}
}
@@ -929,21 +932,48 @@ func (c *Config) addProcessor(name string, table *ast.Table) error {
if !ok {
return fmt.Errorf("Undefined but requested processor: %s", name)
}
processor := creator()
processorConfig, err := buildProcessor(name, table)
if err != nil {
return err
}
if err := toml.UnmarshalTable(table, processor); err != nil {
rf, err := c.newRunningProcessor(creator, processorConfig, name, table)
if err != nil {
return err
}
c.Processors = append(c.Processors, rf)
// save a copy for the aggregator
rf, err = c.newRunningProcessor(creator, processorConfig, name, table)
if err != nil {
return err
}
c.AggProcessors = append(c.AggProcessors, rf)
return nil
}
func (c *Config) newRunningProcessor(
creator processors.StreamingCreator,
processorConfig *models.ProcessorConfig,
name string,
table *ast.Table,
) (*models.RunningProcessor, error) {
processor := creator()
if p, ok := processor.(unwrappable); ok {
if err := toml.UnmarshalTable(table, p.Unwrap()); err != nil {
return nil, err
}
} else {
if err := toml.UnmarshalTable(table, processor); err != nil {
return nil, err
}
}
rf := models.NewRunningProcessor(processor, processorConfig)
c.Processors = append(c.Processors, rf)
return nil
return rf, nil
}
func (c *Config) addOutput(name string, table *ast.Table) error {
@@ -2195,3 +2225,10 @@ func buildOutput(name string, tbl *ast.Table) (*models.OutputConfig, error) {
return oc, nil
}
// unwrappable lets you retrieve the original telegraf.Processor from the
// StreamingProcessor. This is necessary because the toml Unmarshaller won't
// look inside composed types.
type unwrappable interface {
Unwrap() telegraf.Processor
}

8
config/config_test.go
View File

@@ -207,7 +207,7 @@ func TestConfig_FieldNotDefined(t *testing.T) {
c := NewConfig()
err := c.LoadConfig("./testdata/invalid_field.toml")
require.Error(t, err, "invalid field name")
assert.Equal(t, "Error parsing ./testdata/invalid_field.toml, line 2: field corresponding to `not_a_field' is not defined in http_listener_v2.HTTPListenerV2", err.Error())
assert.Equal(t, "Error loading config file ./testdata/invalid_field.toml: Error parsing http_listener_v2, line 2: field corresponding to `not_a_field' is not defined in http_listener_v2.HTTPListenerV2", err.Error())
}
@@ -215,12 +215,12 @@ func TestConfig_WrongFieldType(t *testing.T) {
c := NewConfig()
err := c.LoadConfig("./testdata/wrong_field_type.toml")
require.Error(t, err, "invalid field type")
assert.Equal(t, "Error parsing ./testdata/wrong_field_type.toml, line 2: (http_listener_v2.HTTPListenerV2.Port) cannot unmarshal TOML string into int", err.Error())
assert.Equal(t, "Error loading config file ./testdata/wrong_field_type.toml: Error parsing http_listener_v2, line 2: (http_listener_v2.HTTPListenerV2.Port) cannot unmarshal TOML string into int", err.Error())
c = NewConfig()
err = c.LoadConfig("./testdata/wrong_field_type2.toml")
require.Error(t, err, "invalid field type2")
assert.Equal(t, "Error parsing ./testdata/wrong_field_type2.toml, line 2: (http_listener_v2.HTTPListenerV2.Methods) cannot unmarshal TOML string into []string", err.Error())
assert.Equal(t, "Error loading config file ./testdata/wrong_field_type2.toml: Error parsing http_listener_v2, line 2: (http_listener_v2.HTTPListenerV2.Methods) cannot unmarshal TOML string into []string", err.Error())
}
func TestConfig_InlineTables(t *testing.T) {
@@ -255,5 +255,5 @@ func TestConfig_BadOrdering(t *testing.T) {
c := NewConfig()
err := c.LoadConfig("./testdata/non_slice_slice.toml")
require.Error(t, err, "bad ordering")
assert.Equal(t, "Error parsing ./testdata/non_slice_slice.toml, line 4: cannot unmarshal TOML array into string (need slice)", err.Error())
assert.Equal(t, "Error loading config file ./testdata/non_slice_slice.toml: Error parsing http array, line 4: cannot unmarshal TOML array into string (need slice)", err.Error())
}