2016-02-09 22:03:46 +00:00
|
|
|
# Kafka Consumer Input Plugin
|
2015-07-08 19:39:23 +00:00
|
|
|
|
|
|
|
The [Kafka](http://kafka.apache.org/) consumer plugin polls a specified Kafka
|
|
|
|
topic and adds messages to InfluxDB. The plugin assumes messages follow the
|
|
|
|
line protocol. [Consumer Group](http://godoc.org/github.com/wvanbergen/kafka/consumergroup)
|
|
|
|
is used to talk to the Kafka cluster so multiple instances of telegraf can read
|
|
|
|
from the same topic in parallel.
|
|
|
|
|
2016-07-27 17:53:04 +00:00
|
|
|
Now supports kafka new consumer (version 0.9+) with TLS
|
|
|
|
|
|
|
|
## Configuration[0.8]
|
2016-02-09 22:03:46 +00:00
|
|
|
|
|
|
|
```toml
|
|
|
|
# Read metrics from Kafka topic(s)
|
|
|
|
[[inputs.kafka_consumer]]
|
2016-07-27 17:53:04 +00:00
|
|
|
## is new consumer?
|
|
|
|
new_consumer = false
|
2016-02-18 21:26:51 +00:00
|
|
|
## topic(s) to consume
|
2016-02-09 22:03:46 +00:00
|
|
|
topics = ["telegraf"]
|
2016-02-18 21:26:51 +00:00
|
|
|
## an array of Zookeeper connection strings
|
2016-02-09 22:03:46 +00:00
|
|
|
zookeeper_peers = ["localhost:2181"]
|
2016-02-18 21:26:51 +00:00
|
|
|
## the name of the consumer group
|
2016-02-09 22:03:46 +00:00
|
|
|
consumer_group = "telegraf_metrics_consumers"
|
2016-02-18 21:26:51 +00:00
|
|
|
## Maximum number of metrics to buffer between collection intervals
|
2016-02-09 22:03:46 +00:00
|
|
|
metric_buffer = 100000
|
2016-02-18 21:26:51 +00:00
|
|
|
## Offset (must be either "oldest" or "newest")
|
2016-02-09 22:03:46 +00:00
|
|
|
offset = "oldest"
|
|
|
|
|
2016-06-22 16:23:53 +00:00
|
|
|
## Data format to consume.
|
2016-04-05 20:42:20 +00:00
|
|
|
|
2016-02-18 21:26:51 +00:00
|
|
|
## Each data format has it's own unique set of configuration options, read
|
|
|
|
## more about them here:
|
|
|
|
## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md
|
2016-02-09 22:03:46 +00:00
|
|
|
data_format = "influx"
|
|
|
|
```
|
|
|
|
|
2016-07-27 17:53:04 +00:00
|
|
|
|
|
|
|
|
|
|
|
## Configuration[0.9+]
|
|
|
|
|
|
|
|
```toml
|
|
|
|
# Read metrics from Kafka topic(s)
|
|
|
|
[[inputs.kafka_consumer]]
|
|
|
|
## is new consumer?
|
|
|
|
new_consumer = true
|
|
|
|
## topic(s) to consume
|
|
|
|
topics = ["telegraf"]
|
|
|
|
## an array of kafka 0.9+ brokers
|
|
|
|
broker_list = ["localhost:9092"]
|
|
|
|
## the name of the consumer group
|
|
|
|
consumer_group = "telegraf_kafka_consumer_group"
|
|
|
|
## Offset (must be either "oldest" or "newest")
|
|
|
|
offset = "oldest"
|
|
|
|
|
|
|
|
## Optional SSL Config
|
|
|
|
ssl_ca = "/etc/telegraf/ca.pem"
|
|
|
|
ssl_cert = "/etc/telegraf/cert.pem"
|
|
|
|
ssl_key = "/etc/telegraf/cert.key"
|
|
|
|
## Use SSL but skip chain & host verification
|
|
|
|
insecure_skip_verify = false
|
|
|
|
|
|
|
|
## Data format to consume.
|
|
|
|
|
|
|
|
## Each data format has it's own unique set of configuration options, read
|
|
|
|
## more about them here:
|
|
|
|
## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md
|
|
|
|
data_format = "influx"
|
|
|
|
```
|
|
|
|
|
|
|
|
|
|
|
|
|
2015-07-08 19:39:23 +00:00
|
|
|
## Testing
|
|
|
|
|
2016-06-22 16:23:53 +00:00
|
|
|
Running integration tests requires running Zookeeper & Kafka. See Makefile
|
|
|
|
for kafka container command.
|