Rename formdata parser to form_urlencoded
This commit is contained in:
Daniel Nelson
57
plugins/parsers/form_urlencoded/README.md
Normal file
57
plugins/parsers/form_urlencoded/README.md
Normal file
@@ -0,0 +1,57 @@
|
||||
# Form Urlencoded
|
||||
|
||||
|
||||
The `form-urlencoded` data format parses `application/x-www-form-urlencoded`
|
||||
data, such as commonly used in the [query string][].
|
||||
|
||||
A common use case is to pair it with [http_listener_v2][] input plugin to parse
|
||||
request body or query params.
|
||||
|
||||
### Configuration
|
||||
|
||||
```toml
|
||||
[[inputs.http_listener_v2]]
|
||||
## Address and port to host HTTP listener on
|
||||
service_address = ":8080"
|
||||
|
||||
## Part of the request to consume. Available options are "body" and
|
||||
## "query".
|
||||
data_source = "body"
|
||||
|
||||
## Data format to consume.
|
||||
## Each data format has its own unique set of configuration options, read
|
||||
## more about them here:
|
||||
## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md
|
||||
data_format = "form_urlencoded"
|
||||
|
||||
## Array of key names which should be collected as tags.
|
||||
## By default, keys with string value are ignored if not marked as tags.
|
||||
form_urlencoded_tag_keys = ["tag1"]
|
||||
```
|
||||
|
||||
### Examples
|
||||
|
||||
#### Basic parsing
|
||||
|
||||
Config:
|
||||
```toml
|
||||
[[inputs.http_listener_v2]]
|
||||
name_override = "mymetric"
|
||||
service_address = ":8080"
|
||||
data_source = "query"
|
||||
data_format = "form_urlencoded"
|
||||
form_urlencoded_tag_keys = ["tag1"]
|
||||
```
|
||||
|
||||
Request:
|
||||
```bash
|
||||
curl -i -XGET 'http://localhost:8080/telegraf?tag1=foo&field1=0.42&field2=42'
|
||||
```
|
||||
|
||||
Output:
|
||||
```
|
||||
mymetric,tag1=foo field1=0.42,field2=42
|
||||
```
|
||||
|
||||
[query_string]: https://en.wikipedia.org/wiki/Query_string
|
||||
[http_listener_v2]: /plugins/inputs/http_listener_v2
|
||||
125
plugins/parsers/form_urlencoded/parser.go
Normal file
125
plugins/parsers/form_urlencoded/parser.go
Normal file
@@ -0,0 +1,125 @@
|
||||
package form_urlencoded
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/influxdata/telegraf"
|
||||
"github.com/influxdata/telegraf/metric"
|
||||
)
|
||||
|
||||
var (
|
||||
// ErrNoMetric is returned when no metric is found in input line
|
||||
ErrNoMetric = fmt.Errorf("no metric in line")
|
||||
)
|
||||
|
||||
// Parser decodes "application/x-www-form-urlencoded" data into metrics
|
||||
type Parser struct {
|
||||
MetricName string
|
||||
DefaultTags map[string]string
|
||||
TagKeys []string
|
||||
AllowedKeys []string
|
||||
}
|
||||
|
||||
// Parse converts a slice of bytes in "application/x-www-form-urlencoded" format into metrics
|
||||
func (p Parser) Parse(buf []byte) ([]telegraf.Metric, error) {
|
||||
buf = bytes.TrimSpace(buf)
|
||||
if len(buf) == 0 {
|
||||
return make([]telegraf.Metric, 0), nil
|
||||
}
|
||||
|
||||
values, err := url.ParseQuery(string(buf))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if len(p.AllowedKeys) > 0 {
|
||||
values = p.filterAllowedKeys(values)
|
||||
}
|
||||
|
||||
tags := p.extractTags(values)
|
||||
fields := p.parseFields(values)
|
||||
|
||||
for key, value := range p.DefaultTags {
|
||||
tags[key] = value
|
||||
}
|
||||
|
||||
metric, err := metric.New(p.MetricName, tags, fields, time.Now().UTC())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return []telegraf.Metric{metric}, nil
|
||||
}
|
||||
|
||||
// ParseLine delegates a single line of text to the Parse function
|
||||
func (p Parser) ParseLine(line string) (telegraf.Metric, error) {
|
||||
metrics, err := p.Parse([]byte(line))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if len(metrics) < 1 {
|
||||
return nil, ErrNoMetric
|
||||
}
|
||||
|
||||
return metrics[0], nil
|
||||
}
|
||||
|
||||
// SetDefaultTags sets the default tags for every metric
|
||||
func (p *Parser) SetDefaultTags(tags map[string]string) {
|
||||
p.DefaultTags = tags
|
||||
}
|
||||
|
||||
func (p Parser) filterAllowedKeys(original url.Values) url.Values {
|
||||
result := make(url.Values)
|
||||
|
||||
for _, key := range p.AllowedKeys {
|
||||
value, exists := original[key]
|
||||
if !exists {
|
||||
continue
|
||||
}
|
||||
|
||||
result[key] = value
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
func (p Parser) extractTags(values url.Values) map[string]string {
|
||||
tags := make(map[string]string)
|
||||
for _, key := range p.TagKeys {
|
||||
value, exists := values[key]
|
||||
|
||||
if !exists || len(key) == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
tags[key] = value[0]
|
||||
delete(values, key)
|
||||
}
|
||||
|
||||
return tags
|
||||
}
|
||||
|
||||
func (p Parser) parseFields(values url.Values) map[string]interface{} {
|
||||
fields := make(map[string]interface{})
|
||||
|
||||
for key, value := range values {
|
||||
if len(key) == 0 || len(value) == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
field, err := strconv.ParseFloat(value[0], 64)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
fields[key] = field
|
||||
}
|
||||
|
||||
return fields
|
||||
}
|
||||
172
plugins/parsers/form_urlencoded/parser_test.go
Normal file
172
plugins/parsers/form_urlencoded/parser_test.go
Normal file
@@ -0,0 +1,172 @@
|
||||
package form_urlencoded
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
const (
|
||||
validFormData = "tag1=foo&tag2=bar&tag3=baz&field1=42&field2=69"
|
||||
encodedFormData = "tag1=%24%24%24&field1=1e%2B3"
|
||||
notEscapedProperlyFormData = "invalid=%Y5"
|
||||
blankKeyFormData = "=42&field2=69"
|
||||
emptyFormData = ""
|
||||
)
|
||||
|
||||
func TestParseValidFormData(t *testing.T) {
|
||||
parser := Parser{
|
||||
MetricName: "form_urlencoded_test",
|
||||
}
|
||||
|
||||
metrics, err := parser.Parse([]byte(validFormData))
|
||||
require.NoError(t, err)
|
||||
require.Len(t, metrics, 1)
|
||||
require.Equal(t, "form_urlencoded_test", metrics[0].Name())
|
||||
require.Equal(t, map[string]string{}, metrics[0].Tags())
|
||||
require.Equal(t, map[string]interface{}{
|
||||
"field1": float64(42),
|
||||
"field2": float64(69),
|
||||
}, metrics[0].Fields())
|
||||
}
|
||||
|
||||
func TestParseLineValidFormData(t *testing.T) {
|
||||
parser := Parser{
|
||||
MetricName: "form_urlencoded_test",
|
||||
}
|
||||
|
||||
metric, err := parser.ParseLine(validFormData)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "form_urlencoded_test", metric.Name())
|
||||
require.Equal(t, map[string]string{}, metric.Tags())
|
||||
require.Equal(t, map[string]interface{}{
|
||||
"field1": float64(42),
|
||||
"field2": float64(69),
|
||||
}, metric.Fields())
|
||||
}
|
||||
|
||||
func TestParseValidFormDataWithTags(t *testing.T) {
|
||||
parser := Parser{
|
||||
MetricName: "form_urlencoded_test",
|
||||
TagKeys: []string{"tag1", "tag2"},
|
||||
}
|
||||
|
||||
metrics, err := parser.Parse([]byte(validFormData))
|
||||
require.NoError(t, err)
|
||||
require.Len(t, metrics, 1)
|
||||
require.Equal(t, "form_urlencoded_test", metrics[0].Name())
|
||||
require.Equal(t, map[string]string{
|
||||
"tag1": "foo",
|
||||
"tag2": "bar",
|
||||
}, metrics[0].Tags())
|
||||
require.Equal(t, map[string]interface{}{
|
||||
"field1": float64(42),
|
||||
"field2": float64(69),
|
||||
}, metrics[0].Fields())
|
||||
}
|
||||
|
||||
func TestParseValidFormDataDefaultTags(t *testing.T) {
|
||||
parser := Parser{
|
||||
MetricName: "form_urlencoded_test",
|
||||
TagKeys: []string{"tag1", "tag2"},
|
||||
DefaultTags: map[string]string{"tag4": "default"},
|
||||
}
|
||||
|
||||
metrics, err := parser.Parse([]byte(validFormData))
|
||||
require.NoError(t, err)
|
||||
require.Len(t, metrics, 1)
|
||||
require.Equal(t, "form_urlencoded_test", metrics[0].Name())
|
||||
require.Equal(t, map[string]string{
|
||||
"tag1": "foo",
|
||||
"tag2": "bar",
|
||||
"tag4": "default",
|
||||
}, metrics[0].Tags())
|
||||
require.Equal(t, map[string]interface{}{
|
||||
"field1": float64(42),
|
||||
"field2": float64(69),
|
||||
}, metrics[0].Fields())
|
||||
}
|
||||
|
||||
func TestParseValidFormDataDefaultTagsOverride(t *testing.T) {
|
||||
parser := Parser{
|
||||
MetricName: "form_urlencoded_test",
|
||||
TagKeys: []string{"tag1", "tag2"},
|
||||
DefaultTags: map[string]string{"tag1": "default"},
|
||||
}
|
||||
|
||||
metrics, err := parser.Parse([]byte(validFormData))
|
||||
require.NoError(t, err)
|
||||
require.Len(t, metrics, 1)
|
||||
require.Equal(t, "form_urlencoded_test", metrics[0].Name())
|
||||
require.Equal(t, map[string]string{
|
||||
"tag1": "default",
|
||||
"tag2": "bar",
|
||||
}, metrics[0].Tags())
|
||||
require.Equal(t, map[string]interface{}{
|
||||
"field1": float64(42),
|
||||
"field2": float64(69),
|
||||
}, metrics[0].Fields())
|
||||
}
|
||||
|
||||
func TestParseEncodedFormData(t *testing.T) {
|
||||
parser := Parser{
|
||||
MetricName: "form_urlencoded_test",
|
||||
TagKeys: []string{"tag1"},
|
||||
}
|
||||
|
||||
metrics, err := parser.Parse([]byte(encodedFormData))
|
||||
require.NoError(t, err)
|
||||
require.Len(t, metrics, 1)
|
||||
require.Equal(t, "form_urlencoded_test", metrics[0].Name())
|
||||
require.Equal(t, map[string]string{
|
||||
"tag1": "$$$",
|
||||
}, metrics[0].Tags())
|
||||
require.Equal(t, map[string]interface{}{
|
||||
"field1": float64(1000),
|
||||
}, metrics[0].Fields())
|
||||
}
|
||||
|
||||
func TestParseInvalidFormDataError(t *testing.T) {
|
||||
parser := Parser{
|
||||
MetricName: "form_urlencoded_test",
|
||||
}
|
||||
|
||||
metrics, err := parser.Parse([]byte(notEscapedProperlyFormData))
|
||||
require.Error(t, err)
|
||||
require.Len(t, metrics, 0)
|
||||
}
|
||||
|
||||
func TestParseInvalidFormDataEmptyKey(t *testing.T) {
|
||||
parser := Parser{
|
||||
MetricName: "form_urlencoded_test",
|
||||
}
|
||||
|
||||
// Empty key for field
|
||||
metrics, err := parser.Parse([]byte(blankKeyFormData))
|
||||
require.NoError(t, err)
|
||||
require.Len(t, metrics, 1)
|
||||
require.Equal(t, map[string]string{}, metrics[0].Tags())
|
||||
require.Equal(t, map[string]interface{}{
|
||||
"field2": float64(69),
|
||||
}, metrics[0].Fields())
|
||||
|
||||
// Empty key for tag
|
||||
parser.TagKeys = []string{""}
|
||||
metrics, err = parser.Parse([]byte(blankKeyFormData))
|
||||
require.NoError(t, err)
|
||||
require.Len(t, metrics, 1)
|
||||
require.Equal(t, map[string]string{}, metrics[0].Tags())
|
||||
require.Equal(t, map[string]interface{}{
|
||||
"field2": float64(69),
|
||||
}, metrics[0].Fields())
|
||||
}
|
||||
|
||||
func TestParseInvalidFormDataEmptyString(t *testing.T) {
|
||||
parser := Parser{
|
||||
MetricName: "form_urlencoded_test",
|
||||
}
|
||||
|
||||
metrics, err := parser.Parse([]byte(emptyFormData))
|
||||
require.NoError(t, err)
|
||||
require.Len(t, metrics, 0)
|
||||
}
|
||||
Reference in New Issue
Block a user