From bdb4598b3f219f2245352e0c907d19be17fb521a Mon Sep 17 00:00:00 2001 From: Greg <2653109+glinton@users.noreply.github.com> Date: Fri, 19 Jul 2019 14:16:54 -0600 Subject: [PATCH] Add support for field/tag keys to strings processor (#6129) --- plugins/processors/strings/README.md | 52 +++- plugins/processors/strings/strings.go | 52 ++++ plugins/processors/strings/strings_test.go | 317 +++++++++++++++++++++ 3 files changed, 411 insertions(+), 10 deletions(-) diff --git a/plugins/processors/strings/README.md b/plugins/processors/strings/README.md index 06bffaee8..30d2cbb08 100644 --- a/plugins/processors/strings/README.md +++ b/plugins/processors/strings/README.md @@ -14,41 +14,50 @@ Implemented functions are: Please note that in this implementation these are processed in the order that they appear above. -Specify the `measurement`, `tag` or `field` that you want processed in each section and optionally a `dest` if you want the result stored in a new tag or field. You can specify lots of transformations on data with a single strings processor. +Specify the `measurement`, `tag`, `tag_key`, `field`, or `field_key` that you want processed in each section and optionally a `dest` if you want the result stored in a new tag or field. You can specify lots of transformations on data with a single strings processor. -If you'd like to apply the change to every `tag`, `field`, or `measurement`, use the value "*" for each respective field. Note that the `dest` field will be ignored if "*" is used +If you'd like to apply the change to every `tag`, `tag_key`, `field`, `field_key`, or `measurement`, use the value `"*"` for each respective field. Note that the `dest` field will be ignored if `"*"` is used. + +If you'd like to apply multiple processings to the same `tag_key` or `field_key`, note the process order stated above. See [Example 2]() for an example. ### Configuration: ```toml [[processors.strings]] - # [[processors.strings.uppercase]] - # tag = "method" - + ## Convert a field value to lowercase and store in a new field # [[processors.strings.lowercase]] # field = "uri_stem" # dest = "uri_stem_normalised" - ## Convert a tag value to lowercase + ## Convert a tag value to uppercase + # [[processors.strings.uppercase]] + # tag = "method" + + ## Trim leading and trailing whitespace using the default cutset # [[processors.strings.trim]] # field = "message" + ## Trim leading characters in cutset # [[processors.strings.trim_left]] # field = "message" # cutset = "\t" + ## Trim trailing characters in cutset # [[processors.strings.trim_right]] # field = "message" # cutset = "\r\n" + ## Trim the given prefix from the field # [[processors.strings.trim_prefix]] # field = "my_value" # prefix = "my_" + ## Trim the given suffix from the field # [[processors.strings.trim_suffix]] # field = "read_count" # suffix = "_count" + ## Replace all non-overlapping instances of old with new # [[processors.strings.replace]] # measurement = "*" # old = ":" @@ -79,10 +88,10 @@ the operation and keep the old name. ```toml [[processors.strings]] [[processors.strings.lowercase]] - field = "uri-stem" + tag = "uri_stem" [[processors.strings.trim_prefix]] - field = "uri_stem" + tag = "uri_stem" prefix = "/api/" [[processors.strings.uppercase]] @@ -92,10 +101,33 @@ the operation and keep the old name. **Input** ``` -iis_log,method=get,uri_stem=/API/HealthCheck cs-host="MIXEDCASE_host",referrer="-",ident="-",http_version=1.1,agent="UserAgent",resp_bytes=270i 1519652321000000000 +iis_log,method=get,uri_stem=/API/HealthCheck cs-host="MIXEDCASE_host",http_version=1.1 1519652321000000000 ``` **Output** ``` -iis_log,method=get,uri_stem=healthcheck cs-host="MIXEDCASE_host",cs-host_normalised="MIXEDCASE_HOST",referrer="-",ident="-",http_version=1.1,agent="UserAgent",resp_bytes=270i 1519652321000000000 +iis_log,method=get,uri_stem=healthcheck cs-host="MIXEDCASE_host",http_version=1.1,cs-host_normalised="MIXEDCASE_HOST" 1519652321000000000 +``` + +### Example 2 +**Config** +```toml +[[processors.strings]] + [[processors.strings.lowercase]] + tag_key = "URI-Stem" + + [[processors.strings.replace]] + tag_key = "uri-stem" + old = "-" + new = "_" +``` + +**Input** +``` +iis_log,URI-Stem=/API/HealthCheck http_version=1.1 1519652321000000000 +``` + +**Output** +``` +iis_log,uri_stem=/API/HealthCheck http_version=1.1 1519652321000000000 ``` diff --git a/plugins/processors/strings/strings.go b/plugins/processors/strings/strings.go index 00c7d99b1..56bcf1b2c 100644 --- a/plugins/processors/strings/strings.go +++ b/plugins/processors/strings/strings.go @@ -26,7 +26,9 @@ type ConvertFunc func(s string) string type converter struct { Field string + FieldKey string Tag string + TagKey string Measurement string Dest string Cutset string @@ -109,6 +111,27 @@ func (c *converter) convertTag(metric telegraf.Metric) { } } +func (c *converter) convertTagKey(metric telegraf.Metric) { + var tags map[string]string + if c.TagKey == "*" { + tags = metric.Tags() + } else { + tags = make(map[string]string) + tv, ok := metric.GetTag(c.TagKey) + if !ok { + return + } + tags[c.TagKey] = tv + } + + for key, value := range tags { + if k := c.fn(key); k != "" { + metric.RemoveTag(key) + metric.AddTag(k, value) + } + } +} + func (c *converter) convertField(metric telegraf.Metric) { var fields map[string]interface{} if c.Field == "*" { @@ -133,6 +156,27 @@ func (c *converter) convertField(metric telegraf.Metric) { } } +func (c *converter) convertFieldKey(metric telegraf.Metric) { + var fields map[string]interface{} + if c.FieldKey == "*" { + fields = metric.Fields() + } else { + fields = make(map[string]interface{}) + fv, ok := metric.GetField(c.FieldKey) + if !ok { + return + } + fields[c.FieldKey] = fv + } + + for key, value := range fields { + if k := c.fn(key); k != "" { + metric.RemoveField(key) + metric.AddField(k, value) + } + } +} + func (c *converter) convertMeasurement(metric telegraf.Metric) { if metric.Name() != c.Measurement && c.Measurement != "*" { return @@ -146,10 +190,18 @@ func (c *converter) convert(metric telegraf.Metric) { c.convertField(metric) } + if c.FieldKey != "" { + c.convertFieldKey(metric) + } + if c.Tag != "" { c.convertTag(metric) } + if c.TagKey != "" { + c.convertTagKey(metric) + } + if c.Measurement != "" { c.convertMeasurement(metric) } diff --git a/plugins/processors/strings/strings_test.go b/plugins/processors/strings/strings_test.go index e108c04f7..c89ab7b66 100644 --- a/plugins/processors/strings/strings_test.go +++ b/plugins/processors/strings/strings_test.go @@ -25,6 +25,22 @@ func newM1() telegraf.Metric { return m1 } +func newM2() telegraf.Metric { + m1, _ := metric.New("IIS_log", + map[string]string{ + "verb": "GET", + "S-ComputerName": "MIXEDCASE_hostname", + }, + map[string]interface{}{ + "Request": "/mixed/CASE/paTH/?from=-1D&to=now", + "req/sec": 5, + " whitespace ": " whitespace\t", + }, + time.Now(), + ) + return m1 +} + func TestFieldConversions(t *testing.T) { tests := []struct { name string @@ -253,6 +269,226 @@ func TestFieldConversions(t *testing.T) { } } +func TestFieldKeyConversions(t *testing.T) { + tests := []struct { + name string + plugin *Strings + check func(t *testing.T, actual telegraf.Metric) + }{ + { + name: "Should change existing field key to lowercase", + plugin: &Strings{ + Lowercase: []converter{ + { + FieldKey: "Request", + }, + }, + }, + check: func(t *testing.T, actual telegraf.Metric) { + fv, ok := actual.GetField("request") + require.True(t, ok) + require.Equal(t, "/mixed/CASE/paTH/?from=-1D&to=now", fv) + }, + }, + { + name: "Should change existing field key to uppercase", + plugin: &Strings{ + Uppercase: []converter{ + { + FieldKey: "Request", + }, + }, + }, + check: func(t *testing.T, actual telegraf.Metric) { + fv, ok := actual.GetField("Request") + require.False(t, ok) + + fv, ok = actual.GetField("REQUEST") + require.True(t, ok) + require.Equal(t, "/mixed/CASE/paTH/?from=-1D&to=now", fv) + }, + }, + { + name: "Should trim from both sides", + plugin: &Strings{ + Trim: []converter{ + { + FieldKey: "Request", + Cutset: "eR", + }, + }, + }, + check: func(t *testing.T, actual telegraf.Metric) { + fv, ok := actual.GetField("quest") + require.True(t, ok) + require.Equal(t, "/mixed/CASE/paTH/?from=-1D&to=now", fv) + }, + }, + { + name: "Should trim from both sides but not make lowercase", + plugin: &Strings{ + // Tag/field key multiple executions occur in the following order: (initOnce) + // Lowercase + // Uppercase + // Trim + // TrimLeft + // TrimRight + // TrimPrefix + // TrimSuffix + // Replace + Lowercase: []converter{ + { + FieldKey: "Request", + }, + }, + Trim: []converter{ + { + FieldKey: "request", + Cutset: "tse", + }, + }, + }, + check: func(t *testing.T, actual telegraf.Metric) { + fv, ok := actual.GetField("requ") + require.True(t, ok) + require.Equal(t, "/mixed/CASE/paTH/?from=-1D&to=now", fv) + }, + }, + { + name: "Should trim from left side", + plugin: &Strings{ + TrimLeft: []converter{ + { + FieldKey: "req/sec", + Cutset: "req/", + }, + }, + }, + check: func(t *testing.T, actual telegraf.Metric) { + fv, ok := actual.GetField("sec") + require.True(t, ok) + require.Equal(t, int64(5), fv) + }, + }, + { + name: "Should trim from right side", + plugin: &Strings{ + TrimRight: []converter{ + { + FieldKey: "req/sec", + Cutset: "req/", + }, + }, + }, + check: func(t *testing.T, actual telegraf.Metric) { + fv, ok := actual.GetField("req/sec") + require.True(t, ok) + require.Equal(t, int64(5), fv) + }, + }, + { + name: "Should trim prefix 'req/'", + plugin: &Strings{ + TrimPrefix: []converter{ + { + FieldKey: "req/sec", + Prefix: "req/", + }, + }, + }, + check: func(t *testing.T, actual telegraf.Metric) { + fv, ok := actual.GetField("sec") + require.True(t, ok) + require.Equal(t, int64(5), fv) + }, + }, + { + name: "Should trim suffix '/sec'", + plugin: &Strings{ + TrimSuffix: []converter{ + { + FieldKey: "req/sec", + Suffix: "/sec", + }, + }, + }, + check: func(t *testing.T, actual telegraf.Metric) { + fv, ok := actual.GetField("req") + require.True(t, ok) + require.Equal(t, int64(5), fv) + }, + }, + { + name: "Trim without cutset removes whitespace", + plugin: &Strings{ + Trim: []converter{ + { + FieldKey: " whitespace ", + }, + }, + }, + check: func(t *testing.T, actual telegraf.Metric) { + fv, ok := actual.GetField("whitespace") + require.True(t, ok) + require.Equal(t, " whitespace\t", fv) + }, + }, + { + name: "Trim left without cutset removes whitespace", + plugin: &Strings{ + TrimLeft: []converter{ + { + FieldKey: " whitespace ", + }, + }, + }, + check: func(t *testing.T, actual telegraf.Metric) { + fv, ok := actual.GetField("whitespace ") + require.True(t, ok) + require.Equal(t, " whitespace\t", fv) + }, + }, + { + name: "Trim right without cutset removes whitespace", + plugin: &Strings{ + TrimRight: []converter{ + { + FieldKey: " whitespace ", + }, + }, + }, + check: func(t *testing.T, actual telegraf.Metric) { + fv, ok := actual.GetField(" whitespace") + require.True(t, ok) + require.Equal(t, " whitespace\t", fv) + }, + }, + { + name: "No change if field missing", + plugin: &Strings{ + Lowercase: []converter{ + { + FieldKey: "xyzzy", + Suffix: "-1D&to=now", + }, + }, + }, + check: func(t *testing.T, actual telegraf.Metric) { + fv, ok := actual.GetField("Request") + require.True(t, ok) + require.Equal(t, "/mixed/CASE/paTH/?from=-1D&to=now", fv) + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + metrics := tt.plugin.Apply(newM2()) + require.Len(t, metrics, 1) + tt.check(t, metrics[0]) + }) + } +} + func TestTagConversions(t *testing.T) { tests := []struct { name string @@ -337,6 +573,87 @@ func TestTagConversions(t *testing.T) { } } +func TestTagKeyConversions(t *testing.T) { + tests := []struct { + name string + plugin *Strings + check func(t *testing.T, actual telegraf.Metric) + }{ + { + name: "Should change existing tag key to lowercase", + plugin: &Strings{ + Lowercase: []converter{ + { + Tag: "S-ComputerName", + TagKey: "S-ComputerName", + }, + }, + }, + check: func(t *testing.T, actual telegraf.Metric) { + tv, ok := actual.GetTag("verb") + require.True(t, ok) + require.Equal(t, "GET", tv) + + tv, ok = actual.GetTag("s-computername") + require.True(t, ok) + require.Equal(t, "mixedcase_hostname", tv) + }, + }, + { + name: "Should add new lowercase tag key", + plugin: &Strings{ + Lowercase: []converter{ + { + TagKey: "S-ComputerName", + }, + }, + }, + check: func(t *testing.T, actual telegraf.Metric) { + tv, ok := actual.GetTag("verb") + require.True(t, ok) + require.Equal(t, "GET", tv) + + tv, ok = actual.GetTag("S-ComputerName") + require.False(t, ok) + + tv, ok = actual.GetTag("s-computername") + require.True(t, ok) + require.Equal(t, "MIXEDCASE_hostname", tv) + }, + }, + { + name: "Should add new uppercase tag key", + plugin: &Strings{ + Uppercase: []converter{ + { + TagKey: "S-ComputerName", + }, + }, + }, + check: func(t *testing.T, actual telegraf.Metric) { + tv, ok := actual.GetTag("verb") + require.True(t, ok) + require.Equal(t, "GET", tv) + + tv, ok = actual.GetTag("S-ComputerName") + require.False(t, ok) + + tv, ok = actual.GetTag("S-COMPUTERNAME") + require.True(t, ok) + require.Equal(t, "MIXEDCASE_hostname", tv) + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + metrics := tt.plugin.Apply(newM2()) + require.Len(t, metrics, 1) + tt.check(t, metrics[0]) + }) + } +} + func TestMeasurementConversions(t *testing.T) { tests := []struct { name string