diff --git a/internal/config/config.go b/internal/config/config.go index 0e4c6f23f..a5315b9b6 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -1726,12 +1726,12 @@ func getParserConfig(name string, tbl *ast.Table) (*parsers.Config, error) { } } - if node, ok := tbl.Fields["form_data_tag_keys"]; ok { + if node, ok := tbl.Fields["form_urlencoded_tag_keys"]; ok { if kv, ok := node.(*ast.KeyValue); ok { if ary, ok := kv.Value.(*ast.Array); ok { for _, elem := range ary.Value { if str, ok := elem.(*ast.String); ok { - c.FormDataTagKeys = append(c.FormDataTagKeys, str.Value) + c.FormUrlencodedTagKeys = append(c.FormUrlencodedTagKeys, str.Value) } } } @@ -1779,7 +1779,7 @@ func getParserConfig(name string, tbl *ast.Table) (*parsers.Config, error) { delete(tbl.Fields, "csv_timestamp_column") delete(tbl.Fields, "csv_timestamp_format") delete(tbl.Fields, "csv_trim_space") - delete(tbl.Fields, "form_data_tag_keys") + delete(tbl.Fields, "form_urlencoded_tag_keys") return c, nil } diff --git a/plugins/inputs/http_listener_v2/README.md b/plugins/inputs/http_listener_v2/README.md index 4829e044d..b40e3554f 100644 --- a/plugins/inputs/http_listener_v2/README.md +++ b/plugins/inputs/http_listener_v2/README.md @@ -31,6 +31,10 @@ This is a sample configuration for the plugin. ## 0 means to use the default of 524,288,000 bytes (500 mebibytes) # max_body_size = "500MB" + ## Part of the request to consume. Available options are "body" and + ## "query". + # data_source = "body" + ## Set one or more allowed client CA certificate file names to ## enable mutually authenticated TLS connections # tls_allowed_cacerts = ["/etc/telegraf/clientca.pem"] @@ -49,12 +53,6 @@ This is a sample configuration for the plugin. ## more about them here: ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md data_format = "influx" - - ## Part of the request to consume. - ## Available options are "body" and "query". - ## Note that the data source and data format are independent properties. - ## To consume standard query params and POST forms - use "formdata" as a data_format. - # data_source = "body" ``` ### Metrics: diff --git a/plugins/inputs/http_listener_v2/http_listener_v2.go b/plugins/inputs/http_listener_v2/http_listener_v2.go index 44581f11b..5427b384d 100644 --- a/plugins/inputs/http_listener_v2/http_listener_v2.go +++ b/plugins/inputs/http_listener_v2/http_listener_v2.go @@ -35,21 +35,18 @@ type TimeFunc func() time.Time // HTTPListenerV2 is an input plugin that collects external metrics sent via HTTP type HTTPListenerV2 struct { - ServiceAddress string - Path string - Methods []string - DataSource string - - ReadTimeout internal.Duration - WriteTimeout internal.Duration - MaxBodySize internal.Size - Port int - + ServiceAddress string `toml:"service_address"` + Path string `toml:"path"` + Methods []string `toml:"methods"` + DataSource string `toml:"data_source"` + ReadTimeout internal.Duration `toml:"read_timeout"` + WriteTimeout internal.Duration `toml:"write_timeout"` + MaxBodySize internal.Size `toml:"max_body_size"` + Port int `toml:"port"` + BasicUsername string `toml:"basic_username"` + BasicPassword string `toml:"basic_password"` tlsint.ServerConfig - BasicUsername string - BasicPassword string - TimeFunc wg sync.WaitGroup @@ -79,7 +76,11 @@ const sampleConfig = ` ## 0 means to use the default of 524,288,00 bytes (500 mebibytes) # max_body_size = "500MB" - ## Set one or more allowed client CA certificate file names to + ## Part of the request to consume. Available options are "body" and + ## "query". + # data_source = "body" + + ## Set one or more allowed client CA certificate file names to ## enable mutually authenticated TLS connections # tls_allowed_cacerts = ["/etc/telegraf/clientca.pem"] @@ -97,12 +98,6 @@ const sampleConfig = ` ## more about them here: ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md data_format = "influx" - - ## Part of the request to consume. - ## Available options are "body" and "query". - ## Note that the data source and data format are independent properties. - ## To consume standard query params and POST forms - use "formdata" as a data_format. - # data_source = "body" ` func (h *HTTPListenerV2) SampleConfig() string { @@ -167,7 +162,7 @@ func (h *HTTPListenerV2) Start(acc telegraf.Accumulator) error { server.Serve(h.listener) }() - log.Printf("I! Started HTTP listener V2 service on %s\n", h.ServiceAddress) + log.Printf("I! [inputs.http_listener_v2] Listening on %s", listener.Addr().String()) return nil } @@ -176,8 +171,6 @@ func (h *HTTPListenerV2) Start(acc telegraf.Accumulator) error { func (h *HTTPListenerV2) Stop() { h.listener.Close() h.wg.Wait() - - log.Println("I! Stopped HTTP listener V2 service on ", h.ServiceAddress) } func (h *HTTPListenerV2) ServeHTTP(res http.ResponseWriter, req *http.Request) { @@ -226,13 +219,13 @@ func (h *HTTPListenerV2) serveWrite(res http.ResponseWriter, req *http.Request) metrics, err := h.Parse(bytes) if err != nil { - log.Println("D! " + err.Error()) + log.Printf("D! [inputs.http_listener_v2] Parse error: %v", err) badRequest(res) return } for _, m := range metrics { - h.acc.AddFields(m.Name(), m.Fields(), m.Tags(), m.Time()) + h.acc.AddMetric(m) } res.WriteHeader(http.StatusNoContent) @@ -268,7 +261,7 @@ func (h *HTTPListenerV2) collectQuery(res http.ResponseWriter, req *http.Request query, err := url.QueryUnescape(rawQuery) if err != nil { - log.Println("D! " + err.Error()) + log.Printf("D! [inputs.http_listener_v2] Error parsing query: %v", err) badRequest(res) return nil, false } diff --git a/plugins/parsers/form_urlencoded/README.md b/plugins/parsers/form_urlencoded/README.md new file mode 100644 index 000000000..0a07b7b99 --- /dev/null +++ b/plugins/parsers/form_urlencoded/README.md @@ -0,0 +1,57 @@ +# Form Urlencoded + + +The `form-urlencoded` data format parses `application/x-www-form-urlencoded` +data, such as commonly used in the [query string][]. + +A common use case is to pair it with [http_listener_v2][] input plugin to parse +request body or query params. + +### Configuration + +```toml +[[inputs.http_listener_v2]] + ## Address and port to host HTTP listener on + service_address = ":8080" + + ## Part of the request to consume. Available options are "body" and + ## "query". + data_source = "body" + + ## Data format to consume. + ## Each data format has its own unique set of configuration options, read + ## more about them here: + ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md + data_format = "form_urlencoded" + + ## Array of key names which should be collected as tags. + ## By default, keys with string value are ignored if not marked as tags. + form_urlencoded_tag_keys = ["tag1"] +``` + +### Examples + +#### Basic parsing + +Config: +```toml +[[inputs.http_listener_v2]] + name_override = "mymetric" + service_address = ":8080" + data_source = "query" + data_format = "form_urlencoded" + form_urlencoded_tag_keys = ["tag1"] +``` + +Request: +```bash +curl -i -XGET 'http://localhost:8080/telegraf?tag1=foo&field1=0.42&field2=42' +``` + +Output: +``` +mymetric,tag1=foo field1=0.42,field2=42 +``` + +[query_string]: https://en.wikipedia.org/wiki/Query_string +[http_listener_v2]: /plugins/inputs/http_listener_v2 diff --git a/plugins/parsers/formdata/parser.go b/plugins/parsers/form_urlencoded/parser.go similarity index 99% rename from plugins/parsers/formdata/parser.go rename to plugins/parsers/form_urlencoded/parser.go index 0fc4c572b..f38d87a80 100644 --- a/plugins/parsers/formdata/parser.go +++ b/plugins/parsers/form_urlencoded/parser.go @@ -1,4 +1,4 @@ -package formdata +package form_urlencoded import ( "bytes" @@ -32,7 +32,6 @@ func (p Parser) Parse(buf []byte) ([]telegraf.Metric, error) { } values, err := url.ParseQuery(string(buf)) - if err != nil { return nil, err } @@ -49,7 +48,6 @@ func (p Parser) Parse(buf []byte) ([]telegraf.Metric, error) { } metric, err := metric.New(p.MetricName, tags, fields, time.Now().UTC()) - if err != nil { return nil, err } @@ -60,7 +58,6 @@ func (p Parser) Parse(buf []byte) ([]telegraf.Metric, error) { // ParseLine delegates a single line of text to the Parse function func (p Parser) ParseLine(line string) (telegraf.Metric, error) { metrics, err := p.Parse([]byte(line)) - if err != nil { return nil, err } @@ -82,7 +79,6 @@ func (p Parser) filterAllowedKeys(original url.Values) url.Values { for _, key := range p.AllowedKeys { value, exists := original[key] - if !exists { continue } @@ -118,7 +114,6 @@ func (p Parser) parseFields(values url.Values) map[string]interface{} { } field, err := strconv.ParseFloat(value[0], 64) - if err != nil { continue } diff --git a/plugins/parsers/formdata/parser_test.go b/plugins/parsers/form_urlencoded/parser_test.go similarity index 84% rename from plugins/parsers/formdata/parser_test.go rename to plugins/parsers/form_urlencoded/parser_test.go index cd837a057..931d5a4ca 100644 --- a/plugins/parsers/formdata/parser_test.go +++ b/plugins/parsers/form_urlencoded/parser_test.go @@ -1,4 +1,4 @@ -package formdata +package form_urlencoded import ( "testing" @@ -16,13 +16,13 @@ const ( func TestParseValidFormData(t *testing.T) { parser := Parser{ - MetricName: "formdata_test", + MetricName: "form_urlencoded_test", } metrics, err := parser.Parse([]byte(validFormData)) require.NoError(t, err) require.Len(t, metrics, 1) - require.Equal(t, "formdata_test", metrics[0].Name()) + require.Equal(t, "form_urlencoded_test", metrics[0].Name()) require.Equal(t, map[string]string{}, metrics[0].Tags()) require.Equal(t, map[string]interface{}{ "field1": float64(42), @@ -32,12 +32,12 @@ func TestParseValidFormData(t *testing.T) { func TestParseLineValidFormData(t *testing.T) { parser := Parser{ - MetricName: "formdata_test", + MetricName: "form_urlencoded_test", } metric, err := parser.ParseLine(validFormData) require.NoError(t, err) - require.Equal(t, "formdata_test", metric.Name()) + require.Equal(t, "form_urlencoded_test", metric.Name()) require.Equal(t, map[string]string{}, metric.Tags()) require.Equal(t, map[string]interface{}{ "field1": float64(42), @@ -47,14 +47,14 @@ func TestParseLineValidFormData(t *testing.T) { func TestParseValidFormDataWithTags(t *testing.T) { parser := Parser{ - MetricName: "formdata_test", + MetricName: "form_urlencoded_test", TagKeys: []string{"tag1", "tag2"}, } metrics, err := parser.Parse([]byte(validFormData)) require.NoError(t, err) require.Len(t, metrics, 1) - require.Equal(t, "formdata_test", metrics[0].Name()) + require.Equal(t, "form_urlencoded_test", metrics[0].Name()) require.Equal(t, map[string]string{ "tag1": "foo", "tag2": "bar", @@ -67,7 +67,7 @@ func TestParseValidFormDataWithTags(t *testing.T) { func TestParseValidFormDataDefaultTags(t *testing.T) { parser := Parser{ - MetricName: "formdata_test", + MetricName: "form_urlencoded_test", TagKeys: []string{"tag1", "tag2"}, DefaultTags: map[string]string{"tag4": "default"}, } @@ -75,7 +75,7 @@ func TestParseValidFormDataDefaultTags(t *testing.T) { metrics, err := parser.Parse([]byte(validFormData)) require.NoError(t, err) require.Len(t, metrics, 1) - require.Equal(t, "formdata_test", metrics[0].Name()) + require.Equal(t, "form_urlencoded_test", metrics[0].Name()) require.Equal(t, map[string]string{ "tag1": "foo", "tag2": "bar", @@ -89,7 +89,7 @@ func TestParseValidFormDataDefaultTags(t *testing.T) { func TestParseValidFormDataDefaultTagsOverride(t *testing.T) { parser := Parser{ - MetricName: "formdata_test", + MetricName: "form_urlencoded_test", TagKeys: []string{"tag1", "tag2"}, DefaultTags: map[string]string{"tag1": "default"}, } @@ -97,7 +97,7 @@ func TestParseValidFormDataDefaultTagsOverride(t *testing.T) { metrics, err := parser.Parse([]byte(validFormData)) require.NoError(t, err) require.Len(t, metrics, 1) - require.Equal(t, "formdata_test", metrics[0].Name()) + require.Equal(t, "form_urlencoded_test", metrics[0].Name()) require.Equal(t, map[string]string{ "tag1": "default", "tag2": "bar", @@ -110,14 +110,14 @@ func TestParseValidFormDataDefaultTagsOverride(t *testing.T) { func TestParseEncodedFormData(t *testing.T) { parser := Parser{ - MetricName: "formdata_test", + MetricName: "form_urlencoded_test", TagKeys: []string{"tag1"}, } metrics, err := parser.Parse([]byte(encodedFormData)) require.NoError(t, err) require.Len(t, metrics, 1) - require.Equal(t, "formdata_test", metrics[0].Name()) + require.Equal(t, "form_urlencoded_test", metrics[0].Name()) require.Equal(t, map[string]string{ "tag1": "$$$", }, metrics[0].Tags()) @@ -128,7 +128,7 @@ func TestParseEncodedFormData(t *testing.T) { func TestParseInvalidFormDataError(t *testing.T) { parser := Parser{ - MetricName: "formdata_test", + MetricName: "form_urlencoded_test", } metrics, err := parser.Parse([]byte(notEscapedProperlyFormData)) @@ -138,7 +138,7 @@ func TestParseInvalidFormDataError(t *testing.T) { func TestParseInvalidFormDataEmptyKey(t *testing.T) { parser := Parser{ - MetricName: "formdata_test", + MetricName: "form_urlencoded_test", } // Empty key for field @@ -163,7 +163,7 @@ func TestParseInvalidFormDataEmptyKey(t *testing.T) { func TestParseInvalidFormDataEmptyString(t *testing.T) { parser := Parser{ - MetricName: "formdata_test", + MetricName: "form_urlencoded_test", } metrics, err := parser.Parse([]byte(emptyFormData)) diff --git a/plugins/parsers/formdata/README.md b/plugins/parsers/formdata/README.md deleted file mode 100644 index 4d3c6af31..000000000 --- a/plugins/parsers/formdata/README.md +++ /dev/null @@ -1,76 +0,0 @@ -# FormData - -The FormData data format parses a [query string/x-www-form-urlencoded][query_string] data into metric fields. - -Common use case is to pair it with http listener input plugin to parse request body or query params. - -### Configuration - -```toml -[[inputs.http_listener_v2]] - ## Address and port to host HTTP listener on - service_address = ":8080" - - ## Part of the request to consume. - ## Available options are "body" and "query". - ## To consume standard query params or application/x-www-form-urlencoded body, - ## set the data_format option to "formdata". - data_source = "body" - - ## Data format to consume. - ## Each data format has its own unique set of configuration options, read - ## more about them here: - ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md - data_format = "formdata" - - ## Array of key names which should be collected as tags. - ## By default, keys with string value are ignored if not marked as tags. - form_data_tag_keys = ["tag1"] -``` - -### Examples - -#### Basic parsing -Config: -```toml -[[inputs.http_listener_v2]] - service_address = ":8080" - data_source = "query" - data_format = "formdata" - name_override = "mymetric" -``` - -Request: -```bash -curl -i -XGET 'http://localhost:8080/telegraf?field=0.42' -``` - -Output: -``` -mymetric field=0.42 -``` - -#### Tags and key filter - -Config: -```toml -[[inputs.http_listener_v2]] - service_address = ":8080" - data_source = "query" - data_format = "formdata" - name_override = "mymetric" - fielddrop = ["tag2", "field2"] - form_data_tag_keys = ["tag1"] -``` - -Request: -```bash -curl -i -XGET 'http://localhost:8080/telegraf?tag1=foo&tag2=bar&field1=42&field2=69' -``` - -Output: -``` -mymetric,tag1=foo field1=42 -``` - -[query_string]: https://en.wikipedia.org/wiki/Query_string diff --git a/plugins/parsers/registry.go b/plugins/parsers/registry.go index 3511a99d7..2e8d20819 100644 --- a/plugins/parsers/registry.go +++ b/plugins/parsers/registry.go @@ -8,7 +8,7 @@ import ( "github.com/influxdata/telegraf/plugins/parsers/collectd" "github.com/influxdata/telegraf/plugins/parsers/csv" "github.com/influxdata/telegraf/plugins/parsers/dropwizard" - "github.com/influxdata/telegraf/plugins/parsers/formdata" + "github.com/influxdata/telegraf/plugins/parsers/form_urlencoded" "github.com/influxdata/telegraf/plugins/parsers/graphite" "github.com/influxdata/telegraf/plugins/parsers/grok" "github.com/influxdata/telegraf/plugins/parsers/influx" @@ -144,7 +144,7 @@ type Config struct { CSVTrimSpace bool `toml:"csv_trim_space"` // FormData configuration - FormDataTagKeys []string `toml:"form_data_tag_keys"` + FormUrlencodedTagKeys []string `toml:"form_urlencoded_tag_keys"` } // NewParser returns a Parser interface based on the given config. @@ -213,11 +213,11 @@ func NewParser(config *Config) (Parser, error) { config.DefaultTags) case "logfmt": parser, err = NewLogFmtParser(config.MetricName, config.DefaultTags) - case "formdata": - parser, err = NewFormDataParser( + case "form_urlencoded": + parser, err = NewFormUrlencodedParser( config.MetricName, config.DefaultTags, - config.FormDataTagKeys, + config.FormUrlencodedTagKeys, ) default: err = fmt.Errorf("Invalid data format: %s", config.DataFormat) @@ -411,12 +411,12 @@ func NewWavefrontParser(defaultTags map[string]string) (Parser, error) { return wavefront.NewWavefrontParser(defaultTags), nil } -func NewFormDataParser( +func NewFormUrlencodedParser( metricName string, defaultTags map[string]string, tagKeys []string, ) (Parser, error) { - return &formdata.Parser{ + return &form_urlencoded.Parser{ MetricName: metricName, DefaultTags: defaultTags, TagKeys: tagKeys,