From 8cbd39501ba01ca30fa56cfefde03aa756b19ec9 Mon Sep 17 00:00:00 2001 From: Bo Zhao Date: Wed, 26 Sep 2018 19:30:02 -0700 Subject: [PATCH] Add replace function to strings processor (#4686) --- plugins/processors/strings/README.md | 18 ++++++ plugins/processors/strings/strings.go | 74 ++++++++++++++++------ plugins/processors/strings/strings_test.go | 49 ++++++++++++++ 3 files changed, 122 insertions(+), 19 deletions(-) diff --git a/plugins/processors/strings/README.md b/plugins/processors/strings/README.md index f1e7361fe..06bffaee8 100644 --- a/plugins/processors/strings/README.md +++ b/plugins/processors/strings/README.md @@ -10,11 +10,14 @@ Implemented functions are: - trim_right - trim_prefix - trim_suffix +- replace Please note that in this implementation these are processed in the order that they appear above. Specify the `measurement`, `tag` or `field` that you want processed in each section and optionally a `dest` if you want the result stored in a new tag or field. You can specify lots of transformations on data with a single strings processor. +If you'd like to apply the change to every `tag`, `field`, or `measurement`, use the value "*" for each respective field. Note that the `dest` field will be ignored if "*" is used + ### Configuration: ```toml @@ -45,6 +48,11 @@ Specify the `measurement`, `tag` or `field` that you want processed in each sect # [[processors.strings.trim_suffix]] # field = "read_count" # suffix = "_count" + + # [[processors.strings.replace]] + # measurement = "*" + # old = ":" + # new = "_" ``` #### Trim, TrimLeft, TrimRight @@ -56,6 +64,16 @@ The `trim`, `trim_left`, and `trim_right` functions take an optional parameter: The `trim_prefix` and `trim_suffix` functions remote the given `prefix` or `suffix` respectively from the string. +#### Replace + +The `replace` function does a substring replacement across the entire +string to allow for different conventions between various input and output +plugins. Some example usages are eliminating disallowed characters in +field names or replacing separators between different separators. +Can also be used to eliminate unneeded chars that were in metrics. +If the entire name would be deleted, it will refuse to perform +the operation and keep the old name. + ### Example **Config** ```toml diff --git a/plugins/processors/strings/strings.go b/plugins/processors/strings/strings.go index 8e68dbc52..69e89f025 100644 --- a/plugins/processors/strings/strings.go +++ b/plugins/processors/strings/strings.go @@ -16,6 +16,7 @@ type Strings struct { TrimRight []converter `toml:"trim_right"` TrimPrefix []converter `toml:"trim_prefix"` TrimSuffix []converter `toml:"trim_suffix"` + Replace []converter `toml:"replace"` converters []converter init bool @@ -31,6 +32,8 @@ type converter struct { Cutset string Suffix string Prefix string + Old string + New string fn ConvertFunc } @@ -68,6 +71,12 @@ const sampleConfig = ` # [[processors.strings.trim_suffix]] # field = "read_count" # suffix = "_count" + + ## Replace substrings within field names + # [[processors.strings.trim_suffix]] + # measurement = "*" + # old = ":" + # new = "_" ` func (s *Strings) SampleConfig() string { @@ -79,37 +88,53 @@ func (s *Strings) Description() string { } func (c *converter) convertTag(metric telegraf.Metric) { - tv, ok := metric.GetTag(c.Tag) - if !ok { - return + var tags map[string]string + if c.Tag == "*" { + tags = metric.Tags() + } else { + tags = make(map[string]string) + tv, ok := metric.GetTag(c.Tag) + if !ok { + return + } + tags[c.Tag] = tv } - dest := c.Tag - if c.Dest != "" { - dest = c.Dest + for tag, value := range tags { + dest := tag + if c.Tag != "*" && c.Dest != "" { + dest = c.Dest + } + metric.AddTag(dest, c.fn(value)) } - - metric.AddTag(dest, c.fn(tv)) } func (c *converter) convertField(metric telegraf.Metric) { - fv, ok := metric.GetField(c.Field) - if !ok { - return + var fields map[string]interface{} + if c.Field == "*" { + fields = metric.Fields() + } else { + fields = make(map[string]interface{}) + fv, ok := metric.GetField(c.Field) + if !ok { + return + } + fields[c.Field] = fv } - dest := c.Field - if c.Dest != "" { - dest = c.Dest - } - - if fv, ok := fv.(string); ok { - metric.AddField(dest, c.fn(fv)) + for tag, value := range fields { + dest := tag + if c.Tag != "*" && c.Dest != "" { + dest = c.Dest + } + if fv, ok := value.(string); ok { + metric.AddField(dest, c.fn(fv)) + } } } func (c *converter) convertMeasurement(metric telegraf.Metric) { - if metric.Name() != c.Measurement { + if metric.Name() != c.Measurement && c.Measurement != "*" { return } @@ -176,6 +201,17 @@ func (s *Strings) initOnce() { c.fn = func(s string) string { return strings.TrimSuffix(s, c.Suffix) } s.converters = append(s.converters, c) } + for _, c := range s.Replace { + c.fn = func(s string) string { + newString := strings.Replace(s, c.Old, c.New, -1) + if newString == "" { + return s + } else { + return newString + } + } + s.converters = append(s.converters, c) + } s.init = true } diff --git a/plugins/processors/strings/strings_test.go b/plugins/processors/strings/strings_test.go index 2097ac5a8..a4a16d7dd 100644 --- a/plugins/processors/strings/strings_test.go +++ b/plugins/processors/strings/strings_test.go @@ -481,3 +481,52 @@ func TestReadmeExample(t *testing.T) { assert.Equal(t, expectedFields, processed[0].Fields()) assert.Equal(t, expectedTags, processed[0].Tags()) } + +func newMetric(name string) telegraf.Metric { + tags := map[string]string{} + fields := map[string]interface{}{} + m, _ := metric.New(name, tags, fields, time.Now()) + return m +} + +func TestMeasurementReplace(t *testing.T) { + plugin := &Strings{ + Replace: []converter{ + converter{ + Old: "_", + New: "-", + Measurement: "*", + }, + }, + } + metrics := []telegraf.Metric{ + newMetric("foo:some_value:bar"), + newMetric("average:cpu:usage"), + newMetric("average_cpu_usage"), + } + results := plugin.Apply(metrics...) + assert.Equal(t, "foo:some-value:bar", results[0].Name(), "`_` was not changed to `-`") + assert.Equal(t, "average:cpu:usage", results[1].Name(), "Input name should have been unchanged") + assert.Equal(t, "average-cpu-usage", results[2].Name(), "All instances of `_` should have been changed to `-`") +} + +func TestMeasurementCharDeletion(t *testing.T) { + plugin := &Strings{ + Replace: []converter{ + converter{ + Old: "foo", + New: "", + Measurement: "*", + }, + }, + } + metrics := []telegraf.Metric{ + newMetric("foo:bar:baz"), + newMetric("foofoofoo"), + newMetric("barbarbar"), + } + results := plugin.Apply(metrics...) + assert.Equal(t, ":bar:baz", results[0].Name(), "Should have deleted the initial `foo`") + assert.Equal(t, "foofoofoo", results[1].Name(), "Should have refused to delete the whole string") + assert.Equal(t, "barbarbar", results[2].Name(), "Should not have changed the input") +}