diff --git a/CHANGELOG.md b/CHANGELOG.md index 3cdd544d0..283c5ccba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,8 +27,15 @@ The previous riemann output will still be available using `outputs.riemann_legacy` if needed, but that will eventually be deprecated. It is highly recommended that all users migrate to the new riemann output plugin. +- Generic [socket_listener](./plugins/inputs/socket_listener) and +[socket_writer](./plugins/outputs/socket_writer) plugins have been implemented +for receiving and sending UDP, TCP, unix, & unix-datagram data. These plugins +will replace udp_listener and tcp_listener, which are still available but will +be deprecated eventually. + ### Features +- [#2094](https://github.com/influxdata/telegraf/pull/2094): Add generic socket listener & writer. - [#2204](https://github.com/influxdata/telegraf/pull/2204): Extend http_response to support searching for a substring in response. Return 1 if found, else 0. - [#2137](https://github.com/influxdata/telegraf/pull/2137): Added userstats to mysql input plugin. - [#2179](https://github.com/influxdata/telegraf/pull/2179): Added more InnoDB metric to MySQL plugin. @@ -112,7 +119,6 @@ plugins, not just statsd. - [#1980](https://github.com/influxdata/telegraf/issues/1980): Hide username/password from elasticsearch error log messages. - [#2097](https://github.com/influxdata/telegraf/issues/2097): Configurable HTTP timeouts in Jolokia plugin - [#2255](https://github.com/influxdata/telegraf/pull/2255): Allow changing jolokia attribute delimiter -- [#2094](https://github.com/influxdata/telegraf/pull/2094): Add generic socket listener & writer. ### Bugfixes diff --git a/README.md b/README.md index 9b8a9ddd1..462076dca 100644 --- a/README.md +++ b/README.md @@ -184,8 +184,8 @@ Telegraf can also collect metrics via the following service plugins: * [statsd](./plugins/inputs/statsd) * [socket_listener](./plugins/inputs/socket_listener) * [tail](./plugins/inputs/tail) -* [tcp_listener](./plugins/inputs/tcp_listener) -* [udp_listener](./plugins/inputs/udp_listener) +* [tcp_listener](./plugins/inputs/socket_listener) +* [udp_listener](./plugins/inputs/socket_listener) * [webhooks](./plugins/inputs/webhooks) * [filestack](./plugins/inputs/webhooks/filestack) * [github](./plugins/inputs/webhooks/github) @@ -220,9 +220,11 @@ Telegraf can also collect metrics via the following service plugins: * [nsq](./plugins/outputs/nsq) * [opentsdb](./plugins/outputs/opentsdb) * [prometheus](./plugins/outputs/prometheus_client) -* [socket_writer](./plugins/outputs/socket_writer) * [riemann](./plugins/outputs/riemann) * [riemann_legacy](./plugins/outputs/riemann_legacy) +* [socket_writer](./plugins/outputs/socket_writer) +* [tcp](./plugins/outputs/socket_writer) +* [udp](./plugins/outputs/socket_writer) ## Contributing diff --git a/plugins/inputs/socket_listener/README.md b/plugins/inputs/socket_listener/README.md new file mode 100644 index 000000000..e73296804 --- /dev/null +++ b/plugins/inputs/socket_listener/README.md @@ -0,0 +1,112 @@ +# socket listener service input plugin + +The Socket Listener is a service input plugin that listens for messages from +streaming (tcp, unix) or datagram (udp, unixgram) protocols. + +The plugin expects messages in the +[Telegraf Input Data Formats](https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md). + +### Configuration: + +This is a sample configuration for the plugin. + +```toml +# Generic socket listener capable of handling multiple socket types. +[[inputs.socket_listener]] + ## URL to listen on + # service_address = "tcp://:8094" + # service_address = "tcp://127.0.0.1:http" + # service_address = "tcp4://:8094" + # service_address = "tcp6://:8094" + # service_address = "tcp6://[2001:db8::1]:8094" + # service_address = "udp://:8094" + # service_address = "udp4://:8094" + # service_address = "udp6://:8094" + # service_address = "unix:///tmp/telegraf.sock" + # service_address = "unixgram:///tmp/telegraf.sock" + + ## Maximum number of concurrent connections. + ## Only applies to stream sockets (e.g. TCP). + ## 0 (default) is unlimited. + # max_connections = 1024 + + ## Maximum socket buffer size in bytes. + ## For stream sockets, once the buffer fills up, the sender will start backing up. + ## For datagram sockets, once the buffer fills up, metrics will start dropping. + ## Defaults to the OS default. + # read_buffer_size = 65535 + + ## Data format to consume. + ## Each data format has it's own unique set of configuration options, read + ## more about them here: + ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md + # data_format = "influx" +``` + +## A Note on UDP OS Buffer Sizes + +The `read_buffer_size` config option can be used to adjust the size of the socket +buffer, but this number is limited by OS settings. On Linux, `read_buffer_size` +will default to `rmem_default` and will be capped by `rmem_max`. On BSD systems, +`read_buffer_size` is capped by `maxsockbuf`, and there is no OS default +setting. + +Instructions on how to adjust these OS settings are available below. + +Some OSes (most notably, Linux) place very restricive limits on the performance +of UDP protocols. It is _highly_ recommended that you increase these OS limits to +at least 8MB before trying to run large amounts of UDP traffic to your instance. +8MB is just a recommendation, and can be adjusted higher. + +### Linux +Check the current UDP/IP receive buffer limit & default by typing the following +commands: + +``` +sysctl net.core.rmem_max +sysctl net.core.rmem_default +``` + +If the values are less than 8388608 bytes you should add the following lines to +the /etc/sysctl.conf file: + +``` +net.core.rmem_max=8388608 +net.core.rmem_default=8388608 +``` + +Changes to /etc/sysctl.conf do not take effect until reboot. +To update the values immediately, type the following commands as root: + +``` +sysctl -w net.core.rmem_max=8388608 +sysctl -w net.core.rmem_default=8388608 +``` + +### BSD/Darwin + +On BSD/Darwin systems you need to add about a 15% padding to the kernel limit +socket buffer. Meaning if you want an 8MB buffer (8388608 bytes) you need to set +the kernel limit to `8388608*1.15 = 9646900`. This is not documented anywhere but +happens +[in the kernel here.](https://github.com/freebsd/freebsd/blob/master/sys/kern/uipc_sockbuf.c#L63-L64) + +Check the current UDP/IP buffer limit by typing the following command: + +``` +sysctl kern.ipc.maxsockbuf +``` + +If the value is less than 9646900 bytes you should add the following lines +to the /etc/sysctl.conf file (create it if necessary): + +``` +kern.ipc.maxsockbuf=9646900 +``` + +Changes to /etc/sysctl.conf do not take effect until reboot. +To update the values immediately, type the following command as root: + +``` +sysctl -w kern.ipc.maxsockbuf=9646900 +``` diff --git a/plugins/inputs/tcp_listener/README.md b/plugins/inputs/tcp_listener/README.md index 0066ea801..f858c7179 100644 --- a/plugins/inputs/tcp_listener/README.md +++ b/plugins/inputs/tcp_listener/README.md @@ -1,30 +1,4 @@ # TCP listener service input plugin -The TCP listener is a service input plugin that listens for messages on a TCP -socket and adds those messages to InfluxDB. -The plugin expects messages in the -[Telegraf Input Data Formats](https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md). - -### Configuration: - -This is a sample configuration for the plugin. - -```toml -# Generic TCP listener -[[inputs.tcp_listener]] - ## Address and port to host TCP listener on - service_address = ":8094" - - ## Number of TCP messages allowed to queue up. Once filled, the - ## TCP listener will start dropping packets. - allowed_pending_messages = 10000 - - ## Maximum number of concurrent TCP connections to allow - max_tcp_connections = 250 - - ## Data format to consume. - ## Each data format has it's own unique set of configuration options, read - ## more about them here: - ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md - data_format = "influx" -``` +> DEPRECATED: As of version 1.3 the TCP listener plugin has been deprecated in favor of the +> [socket_listener plugin](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/socket_listener) diff --git a/plugins/inputs/tcp_listener/tcp_listener.go b/plugins/inputs/tcp_listener/tcp_listener.go index 3ce4d87b4..544f36bd6 100644 --- a/plugins/inputs/tcp_listener/tcp_listener.go +++ b/plugins/inputs/tcp_listener/tcp_listener.go @@ -58,21 +58,9 @@ var malformedwarn = "E! tcp_listener has received %d malformed packets" + " thus far." const sampleConfig = ` - ## Address and port to host TCP listener on - # service_address = ":8094" - - ## Number of TCP messages allowed to queue up. Once filled, the - ## TCP listener will start dropping packets. - # allowed_pending_messages = 10000 - - ## Maximum number of concurrent TCP connections to allow - # max_tcp_connections = 250 - - ## Data format to consume. - ## Each data format has it's own unique set of configuration options, read - ## more about them here: - ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md - data_format = "influx" + # DEPRECATED: the TCP listener plugin has been deprecated in favor of the + # socket_listener plugin + # see https://github.com/influxdata/telegraf/tree/master/plugins/inputs/socket_listener ` func (t *TcpListener) SampleConfig() string { @@ -98,6 +86,10 @@ func (t *TcpListener) Start(acc telegraf.Accumulator) error { t.Lock() defer t.Unlock() + log.Println("W! DEPRECATED: the TCP listener plugin has been deprecated " + + "in favor of the socket_listener plugin " + + "(https://github.com/influxdata/telegraf/tree/master/plugins/inputs/socket_listener)") + tags := map[string]string{ "address": t.ServiceAddress, } diff --git a/plugins/inputs/udp_listener/README.md b/plugins/inputs/udp_listener/README.md index ee675f535..6228090b6 100644 --- a/plugins/inputs/udp_listener/README.md +++ b/plugins/inputs/udp_listener/README.md @@ -1,86 +1,4 @@ # UDP listener service input plugin -The UDP listener is a service input plugin that listens for messages on a UDP -socket and adds those messages to InfluxDB. -The plugin expects messages in the -[Telegraf Input Data Formats](https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md). - -### Configuration: - -This is a sample configuration for the plugin. - -```toml -[[inputs.udp_listener]] - ## Address and port to host UDP listener on - service_address = ":8092" - - ## Number of UDP messages allowed to queue up. Once filled, the - ## UDP listener will start dropping packets. - allowed_pending_messages = 10000 - - ## Data format to consume. - ## Each data format has it's own unique set of configuration options, read - ## more about them here: - ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md - data_format = "influx" -``` - -## A Note on UDP OS Buffer Sizes - -Some OSes (most notably, Linux) place very restricive limits on the performance -of UDP protocols. It is _highly_ recommended that you increase these OS limits to -at least 8MB before trying to run large amounts of UDP traffic to your instance. -8MB is just a recommendation, and can be adjusted higher. - -### Linux -Check the current UDP/IP receive buffer limit & default by typing the following -commands: - -``` -sysctl net.core.rmem_max -sysctl net.core.rmem_default -``` - -If the values are less than 8388608 bytes you should add the following lines to -the /etc/sysctl.conf file: - -``` -net.core.rmem_max=8388608 -net.core.rmem_default=8388608 -``` - -Changes to /etc/sysctl.conf do not take effect until reboot. -To update the values immediately, type the following commands as root: - -``` -sysctl -w net.core.rmem_max=8388608 -sysctl -w net.core.rmem_default=8388608 -``` - -### BSD/Darwin - -On BSD/Darwin systems you need to add about a 15% padding to the kernel limit -socket buffer. Meaning if you want an 8MB buffer (8388608 bytes) you need to set -the kernel limit to `8388608*1.15 = 9646900`. This is not documented anywhere but -happens -[in the kernel here.](https://github.com/freebsd/freebsd/blob/master/sys/kern/uipc_sockbuf.c#L63-L64) - -Check the current UDP/IP buffer limit by typing the following command: - -``` -sysctl kern.ipc.maxsockbuf -``` - -If the value is less than 9646900 bytes you should add the following lines -to the /etc/sysctl.conf file (create it if necessary): - -``` -kern.ipc.maxsockbuf=9646900 -``` - -Changes to /etc/sysctl.conf do not take effect until reboot. -To update the values immediately, type the following commands as root: - -``` -sysctl -w kern.ipc.maxsockbuf=9646900 -``` +> DEPRECATED: As of version 1.3 the UDP listener plugin has been deprecated in favor of the +> [socket_listener plugin](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/socket_listener) diff --git a/plugins/inputs/udp_listener/udp_listener.go b/plugins/inputs/udp_listener/udp_listener.go index 518a3fe48..53c6a72f5 100644 --- a/plugins/inputs/udp_listener/udp_listener.go +++ b/plugins/inputs/udp_listener/udp_listener.go @@ -66,22 +66,9 @@ var malformedwarn = "E! udp_listener has received %d malformed packets" + " thus far." const sampleConfig = ` - ## Address and port to host UDP listener on - # service_address = ":8092" - - ## Number of UDP messages allowed to queue up. Once filled, the - ## UDP listener will start dropping packets. - # allowed_pending_messages = 10000 - - ## Set the buffer size of the UDP connection outside of OS default (in bytes) - ## If set to 0, take OS default - udp_buffer_size = 16777216 - - ## Data format to consume. - ## Each data format has it's own unique set of configuration options, read - ## more about them here: - ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md - data_format = "influx" + # DEPRECATED: the TCP listener plugin has been deprecated in favor of the + # socket_listener plugin + # see https://github.com/influxdata/telegraf/tree/master/plugins/inputs/socket_listener ` func (u *UdpListener) SampleConfig() string { @@ -106,6 +93,10 @@ func (u *UdpListener) Start(acc telegraf.Accumulator) error { u.Lock() defer u.Unlock() + log.Println("W! DEPRECATED: the UDP listener plugin has been deprecated " + + "in favor of the socket_listener plugin " + + "(https://github.com/influxdata/telegraf/tree/master/plugins/inputs/socket_listener)") + tags := map[string]string{ "address": u.ServiceAddress, }