From d17f3a1eea16ca541806eb4066b59208d729746f Mon Sep 17 00:00:00 2001 From: Cameron Sparr Date: Thu, 17 Mar 2016 18:01:01 -0600 Subject: [PATCH] Value parser, for parsing a single value into a metric closes #849 --- docs/DATA_FORMATS_INPUT.md | 39 +++++ internal/config/config.go | 9 + plugins/parsers/registry.go | 23 ++- plugins/parsers/value/parser.go | 68 ++++++++ plugins/parsers/value/parser_test.go | 238 +++++++++++++++++++++++++++ 5 files changed, 375 insertions(+), 2 deletions(-) create mode 100644 plugins/parsers/value/parser.go create mode 100644 plugins/parsers/value/parser_test.go diff --git a/docs/DATA_FORMATS_INPUT.md b/docs/DATA_FORMATS_INPUT.md index 79528a962..12c4d4cde 100644 --- a/docs/DATA_FORMATS_INPUT.md +++ b/docs/DATA_FORMATS_INPUT.md @@ -1,5 +1,12 @@ # Telegraf Input Data Formats +Telegraf is able to parse the following input data formats into metrics: + +1. InfluxDB Line Protocol +1. JSON +1. Graphite +1. Value, ie 45 or "booyah" + Telegraf metrics, like InfluxDB [points](https://docs.influxdata.com/influxdb/v0.10/write_protocols/line/), are a combination of four basic parts: @@ -134,6 +141,38 @@ Your Telegraf metrics would get tagged with "my_tag_1" exec_mycollector,my_tag_1=foo a=5,b_c=6 ``` +## Value: + +The "value" data format translates single values into Telegraf metrics. This +is done by assigning a measurement name (which can be overridden using the +`name_override` config option), and setting a single field ("value") as the +parsed metric. + +#### Value Configuration: + +You can tell Telegraf what type of metric to collect by using the `data_type` +configuration option. + +It is also recommended that you set `name_override` to a measurement name that +makes sense for your metric, otherwise it will just be set to the name of the +plugin. + +```toml +[[inputs.exec]] + ## Commands array + commands = ["cat /proc/sys/kernel/random/entropy_avail"] + + ## override the default metric name of "exec" + name_override = "entropy_available" + + ## Data format to consume. This can be "json", "value", influx" or "graphite" + ## Each data format has it's own unique set of configuration options, read + ## more about them here: + ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md + data_format = "value" + data_type = "integer" +``` + ## Graphite: The Graphite data format translates graphite _dot_ buckets directly into diff --git a/internal/config/config.go b/internal/config/config.go index f64e0a56a..6990b2db7 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -701,12 +701,21 @@ func buildParser(name string, tbl *ast.Table) (parsers.Parser, error) { } } + if node, ok := tbl.Fields["data_type"]; ok { + if kv, ok := node.(*ast.KeyValue); ok { + if str, ok := kv.Value.(*ast.String); ok { + c.DataType = str.Value + } + } + } + c.MetricName = name delete(tbl.Fields, "data_format") delete(tbl.Fields, "separator") delete(tbl.Fields, "templates") delete(tbl.Fields, "tag_keys") + delete(tbl.Fields, "data_type") return parsers.NewParser(c) } diff --git a/plugins/parsers/registry.go b/plugins/parsers/registry.go index 982b6bb80..b86b61c18 100644 --- a/plugins/parsers/registry.go +++ b/plugins/parsers/registry.go @@ -8,6 +8,7 @@ import ( "github.com/influxdata/telegraf/plugins/parsers/graphite" "github.com/influxdata/telegraf/plugins/parsers/influx" "github.com/influxdata/telegraf/plugins/parsers/json" + "github.com/influxdata/telegraf/plugins/parsers/value" ) // ParserInput is an interface for input plugins that are able to parse @@ -38,7 +39,7 @@ type Parser interface { // Config is a struct that covers the data types needed for all parser types, // and can be used to instantiate _any_ of the parsers. type Config struct { - // Dataformat can be one of: json, influx, graphite + // Dataformat can be one of: json, influx, graphite, value DataFormat string // Separator only applied to Graphite data. @@ -48,9 +49,12 @@ type Config struct { // TagKeys only apply to JSON data TagKeys []string - // MetricName only applies to JSON data. This will be the name of the measurement. + // MetricName applies to JSON & value. This will be the name of the measurement. MetricName string + // DataType only applies to value, this will be the type to parse value to + DataType string + // DefaultTags are the default tags that will be added to all parsed metrics. DefaultTags map[string]string } @@ -63,6 +67,9 @@ func NewParser(config *Config) (Parser, error) { case "json": parser, err = NewJSONParser(config.MetricName, config.TagKeys, config.DefaultTags) + case "value": + parser, err = NewValueParser(config.MetricName, + config.DataType, config.DefaultTags) case "influx": parser, err = NewInfluxParser() case "graphite": @@ -98,3 +105,15 @@ func NewGraphiteParser( ) (Parser, error) { return graphite.NewGraphiteParser(separator, templates, defaultTags) } + +func NewValueParser( + metricName string, + dataType string, + defaultTags map[string]string, +) (Parser, error) { + return &value.ValueParser{ + MetricName: metricName, + DataType: dataType, + DefaultTags: defaultTags, + }, nil +} diff --git a/plugins/parsers/value/parser.go b/plugins/parsers/value/parser.go new file mode 100644 index 000000000..00673eced --- /dev/null +++ b/plugins/parsers/value/parser.go @@ -0,0 +1,68 @@ +package value + +import ( + "bytes" + "fmt" + "strconv" + "time" + + "github.com/influxdata/telegraf" +) + +type ValueParser struct { + MetricName string + DataType string + DefaultTags map[string]string +} + +func (v *ValueParser) Parse(buf []byte) ([]telegraf.Metric, error) { + // separate out any fields in the buffer, ignore anything but the last. + values := bytes.Fields(buf) + if len(values) < 1 { + return []telegraf.Metric{}, nil + } + valueStr := string(values[len(values)-1]) + + var value interface{} + var err error + switch v.DataType { + case "", "int", "integer": + value, err = strconv.Atoi(valueStr) + case "float", "long": + value, err = strconv.ParseFloat(valueStr, 64) + case "str", "string": + value = valueStr + case "bool", "boolean": + value, err = strconv.ParseBool(valueStr) + } + if err != nil { + return nil, err + } + + fields := map[string]interface{}{"value": value} + metric, err := telegraf.NewMetric(v.MetricName, v.DefaultTags, + fields, time.Now().UTC()) + if err != nil { + return nil, err + } + + return []telegraf.Metric{metric}, nil +} + +func (v *ValueParser) ParseLine(line string) (telegraf.Metric, error) { + metrics, err := v.Parse([]byte(line)) + + if err != nil { + return nil, err + } + + if len(metrics) < 1 { + return nil, fmt.Errorf("Can not parse the line: %s, for data format: value", line) + } + + return metrics[0], nil +} + +func (v *ValueParser) SetDefaultTags(tags map[string]string) { + v.DefaultTags = tags +} diff --git a/plugins/parsers/value/parser_test.go b/plugins/parsers/value/parser_test.go new file mode 100644 index 000000000..f60787491 --- /dev/null +++ b/plugins/parsers/value/parser_test.go @@ -0,0 +1,238 @@ +package value + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestParseValidValues(t *testing.T) { + parser := ValueParser{ + MetricName: "value_test", + DataType: "integer", + } + metrics, err := parser.Parse([]byte("55")) + assert.NoError(t, err) + assert.Len(t, metrics, 1) + assert.Equal(t, "value_test", metrics[0].Name()) + assert.Equal(t, map[string]interface{}{ + "value": int64(55), + }, metrics[0].Fields()) + assert.Equal(t, map[string]string{}, metrics[0].Tags()) + + parser = ValueParser{ + MetricName: "value_test", + DataType: "float", + } + metrics, err = parser.Parse([]byte("64")) + assert.NoError(t, err) + assert.Len(t, metrics, 1) + assert.Equal(t, "value_test", metrics[0].Name()) + assert.Equal(t, map[string]interface{}{ + "value": float64(64), + }, metrics[0].Fields()) + assert.Equal(t, map[string]string{}, metrics[0].Tags()) + + parser = ValueParser{ + MetricName: "value_test", + DataType: "string", + } + metrics, err = parser.Parse([]byte("foobar")) + assert.NoError(t, err) + assert.Len(t, metrics, 1) + assert.Equal(t, "value_test", metrics[0].Name()) + assert.Equal(t, map[string]interface{}{ + "value": "foobar", + }, metrics[0].Fields()) + assert.Equal(t, map[string]string{}, metrics[0].Tags()) + + parser = ValueParser{ + MetricName: "value_test", + DataType: "boolean", + } + metrics, err = parser.Parse([]byte("true")) + assert.NoError(t, err) + assert.Len(t, metrics, 1) + assert.Equal(t, "value_test", metrics[0].Name()) + assert.Equal(t, map[string]interface{}{ + "value": true, + }, metrics[0].Fields()) + assert.Equal(t, map[string]string{}, metrics[0].Tags()) +} + +func TestParseMultipleValues(t *testing.T) { + parser := ValueParser{ + MetricName: "value_test", + DataType: "integer", + } + metrics, err := parser.Parse([]byte(`55 +45 +223 +12 +999 +`)) + assert.NoError(t, err) + assert.Len(t, metrics, 1) + assert.Equal(t, "value_test", metrics[0].Name()) + assert.Equal(t, map[string]interface{}{ + "value": int64(999), + }, metrics[0].Fields()) + assert.Equal(t, map[string]string{}, metrics[0].Tags()) +} + +func TestParseLineValidValues(t *testing.T) { + parser := ValueParser{ + MetricName: "value_test", + DataType: "integer", + } + metric, err := parser.ParseLine("55") + assert.NoError(t, err) + assert.Equal(t, "value_test", metric.Name()) + assert.Equal(t, map[string]interface{}{ + "value": int64(55), + }, metric.Fields()) + assert.Equal(t, map[string]string{}, metric.Tags()) + + parser = ValueParser{ + MetricName: "value_test", + DataType: "float", + } + metric, err = parser.ParseLine("64") + assert.NoError(t, err) + assert.Equal(t, "value_test", metric.Name()) + assert.Equal(t, map[string]interface{}{ + "value": float64(64), + }, metric.Fields()) + assert.Equal(t, map[string]string{}, metric.Tags()) + + parser = ValueParser{ + MetricName: "value_test", + DataType: "string", + } + metric, err = parser.ParseLine("foobar") + assert.NoError(t, err) + assert.Equal(t, "value_test", metric.Name()) + assert.Equal(t, map[string]interface{}{ + "value": "foobar", + }, metric.Fields()) + assert.Equal(t, map[string]string{}, metric.Tags()) + + parser = ValueParser{ + MetricName: "value_test", + DataType: "boolean", + } + metric, err = parser.ParseLine("true") + assert.NoError(t, err) + assert.Equal(t, "value_test", metric.Name()) + assert.Equal(t, map[string]interface{}{ + "value": true, + }, metric.Fields()) + assert.Equal(t, map[string]string{}, metric.Tags()) +} + +func TestParseInvalidValues(t *testing.T) { + parser := ValueParser{ + MetricName: "value_test", + DataType: "integer", + } + metrics, err := parser.Parse([]byte("55.0")) + assert.Error(t, err) + assert.Len(t, metrics, 0) + + parser = ValueParser{ + MetricName: "value_test", + DataType: "float", + } + metrics, err = parser.Parse([]byte("foobar")) + assert.Error(t, err) + assert.Len(t, metrics, 0) + + parser = ValueParser{ + MetricName: "value_test", + DataType: "boolean", + } + metrics, err = parser.Parse([]byte("213")) + assert.Error(t, err) + assert.Len(t, metrics, 0) +} + +func TestParseLineInvalidValues(t *testing.T) { + parser := ValueParser{ + MetricName: "value_test", + DataType: "integer", + } + _, err := parser.ParseLine("55.0") + assert.Error(t, err) + + parser = ValueParser{ + MetricName: "value_test", + DataType: "float", + } + _, err = parser.ParseLine("foobar") + assert.Error(t, err) + + parser = ValueParser{ + MetricName: "value_test", + DataType: "boolean", + } + _, err = parser.ParseLine("213") + assert.Error(t, err) +} + +func TestParseValidValuesDefaultTags(t *testing.T) { + parser := ValueParser{ + MetricName: "value_test", + DataType: "integer", + } + parser.SetDefaultTags(map[string]string{"test": "tag"}) + metrics, err := parser.Parse([]byte("55")) + assert.NoError(t, err) + assert.Len(t, metrics, 1) + assert.Equal(t, "value_test", metrics[0].Name()) + assert.Equal(t, map[string]interface{}{ + "value": int64(55), + }, metrics[0].Fields()) + assert.Equal(t, map[string]string{"test": "tag"}, metrics[0].Tags()) + + parser = ValueParser{ + MetricName: "value_test", + DataType: "float", + } + parser.SetDefaultTags(map[string]string{"test": "tag"}) + metrics, err = parser.Parse([]byte("64")) + assert.NoError(t, err) + assert.Len(t, metrics, 1) + assert.Equal(t, "value_test", metrics[0].Name()) + assert.Equal(t, map[string]interface{}{ + "value": float64(64), + }, metrics[0].Fields()) + assert.Equal(t, map[string]string{"test": "tag"}, metrics[0].Tags()) + + parser = ValueParser{ + MetricName: "value_test", + DataType: "string", + } + parser.SetDefaultTags(map[string]string{"test": "tag"}) + metrics, err = parser.Parse([]byte("foobar")) + assert.NoError(t, err) + assert.Len(t, metrics, 1) + assert.Equal(t, "value_test", metrics[0].Name()) + assert.Equal(t, map[string]interface{}{ + "value": "foobar", + }, metrics[0].Fields()) + assert.Equal(t, map[string]string{"test": "tag"}, metrics[0].Tags()) + + parser = ValueParser{ + MetricName: "value_test", + DataType: "boolean", + } + parser.SetDefaultTags(map[string]string{"test": "tag"}) + metrics, err = parser.Parse([]byte("true")) + assert.NoError(t, err) + assert.Len(t, metrics, 1) + assert.Equal(t, "value_test", metrics[0].Name()) + assert.Equal(t, map[string]interface{}{ + "value": true, + }, metrics[0].Fields()) + assert.Equal(t, map[string]string{"test": "tag"}, metrics[0].Tags()) +}