From 7499c1f969608e30a94b4841bedb91339a186231 Mon Sep 17 00:00:00 2001 From: Cameron Sparr Date: Thu, 7 Jan 2016 15:21:10 -0700 Subject: [PATCH] 0.3.0: update README and documentation --- CHANGELOG.md | 7 ++- CONFIGURATION.md | 48 ++++++++++------ CONTRIBUTING.md | 68 ++++++++--------------- README.md | 115 ++++++++++++++++++++------------------- cmd/telegraf/telegraf.go | 62 ++++++++++++++------- 5 files changed, 161 insertions(+), 139 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d9856b5cf..310051ea1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,8 +11,11 @@ directory. - **breaking change** `jolokia` plugin: must use global tag/drop/pass parameters for configuration. - **breaking change** `twemproxy` plugin: `prefix` option removed. -- **breaking change** `procstat` cpu measurements are now prepended with `cpu_time_` instead of -only `cpu_` +- **breaking change** `procstat` cpu measurements are now prepended with `cpu_time_` +instead of only `cpu_` +- **breaking change** some command-line flags have been renamed to separate words. +`-configdirectory` -> `-config-directory`, `-filter` -> `-input-filter`, +`-outputfilter` -> `-output-filter` - The prometheus plugin schema has not been changed (measurements have not been aggregated). diff --git a/CONFIGURATION.md b/CONFIGURATION.md index fc822d461..ef91f0a06 100644 --- a/CONFIGURATION.md +++ b/CONFIGURATION.md @@ -1,6 +1,6 @@ # Telegraf Configuration -## Generating a config file +## Generating a Configuration File A default Telegraf config file can be generated using the `-sample-config` flag, like this: `telegraf -sample-config` @@ -9,7 +9,20 @@ To generate a file with specific inputs and outputs, you can use the `-input-filter` and `-output-filter` flags, like this: `telegraf -sample-config -input-filter cpu:mem:net:swap -output-filter influxdb:kafka` -## Plugin Configuration +## Telegraf Agent Configuration + +Telegraf has a few options you can configure under the `agent` section of the +config. + +* **hostname**: The hostname is passed as a tag. By default this will be +the value returned by `hostname` on the machine running Telegraf. +You can override that value here. +* **interval**: How often to gather metrics. Uses a simple number + +unit parser, e.g. "10s" for 10 seconds or "5m" for 5 minutes. +* **debug**: Set to true to gather and send metrics to STDOUT as well as +InfluxDB. + +## Input Configuration There are some configuration options that are configurable per plugin: @@ -22,7 +35,7 @@ There are some configuration options that are configurable per plugin: global interval, but if one particular plugin should be run less or more often, you can configure that here. -### Plugin Filters +### Input Filters There are also filters that can be configured per plugin: @@ -36,7 +49,7 @@ match against the tag name, and if it matches the measurement is emitted. * **tagdrop**: The inverse of tagpass. If a tag matches, the measurement is not emitted. This is tested on measurements that have passed the tagpass test. -### Plugin Configuration Examples +### Input Configuration Examples This is a full working config that will output CPU data to an InfluxDB instance at 192.168.59.103:8086, tagging measurements with dc="denver-1". It will output @@ -57,8 +70,8 @@ fields which begin with `time_`. database = "telegraf" # required. precision = "s" -# PLUGINS -[plugins] +# INPUTS +[inputs] [[inputs.cpu]] percpu = true totalcpu = false @@ -66,10 +79,10 @@ fields which begin with `time_`. drop = ["time_*"] ``` -### Plugin Config: tagpass and tagdrop +### Input Config: tagpass and tagdrop ```toml -[plugins] +[inputs] [[inputs.cpu]] percpu = true totalcpu = false @@ -88,7 +101,7 @@ fields which begin with `time_`. path = [ "/opt", "/home*" ] ``` -### Plugin Config: pass and drop +### Input Config: pass and drop ```toml # Drop all metrics for guest & steal CPU usage @@ -102,7 +115,7 @@ fields which begin with `time_`. pass = ["inodes*"] ``` -### Plugin config: prefix, suffix, and override +### Input config: prefix, suffix, and override This plugin will emit measurements with the name `cpu_total` @@ -122,7 +135,7 @@ This will emit measurements with the name `foobar` totalcpu = true ``` -### Plugin config: tags +### Input config: tags This plugin will emit measurements with two additional tags: `tag1=foo` and `tag2=bar` @@ -136,10 +149,12 @@ This plugin will emit measurements with two additional tags: `tag1=foo` and tag2 = "bar" ``` -### Multiple plugins of the same type +### Multiple inputs of the same type -Additional plugins (or outputs) of the same type can be specified, -just define more instances in the config file: +Additional inputs (or outputs) of the same type can be specified, +just define more instances in the config file. It is highly recommended that +you utilize `name_override`, `name_prefix`, or `name_suffix` config options +to avoid measurement collisions: ```toml [[inputs.cpu]] @@ -149,6 +164,7 @@ just define more instances in the config file: [[inputs.cpu]] percpu = true totalcpu = false + name_override = "percpu_usage" drop = ["cpu_time*"] ``` @@ -158,8 +174,8 @@ Telegraf also supports specifying multiple output sinks to send data to, configuring each output sink is different, but examples can be found by running `telegraf -sample-config`. -Outputs also support the same configurable options as plugins -(pass, drop, tagpass, tagdrop), added in 0.2.4 +Outputs also support the same configurable options as inputs +(pass, drop, tagpass, tagdrop) ```toml [[outputs.influxdb]] diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 12dcad1d0..a47ad2f17 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -3,7 +3,7 @@ Before we can merge a pull request, you will need to sign the CLA, which can be found [on our website](http://influxdb.com/community/cla.html) -## Plugins +## Input Plugins This section is for developers who want to create new collection inputs. Telegraf is entirely plugin driven. This interface allows for operators to @@ -13,23 +13,21 @@ to create new ways of generating metrics. Plugin authorship is kept as simple as possible to promote people to develop and submit new inputs. -### Plugin Guidelines +### Input Plugin Guidelines * A plugin must conform to the `inputs.Input` interface. -* Each generated metric automatically has the name of the plugin that generated -it prepended. This is to keep plugins honest. -* Plugins should call `inputs.Add` in their `init` function to register themselves. +* Input Plugins should call `inputs.Add` in their `init` function to register themselves. See below for a quick example. -* To be available within Telegraf itself, plugins must add themselves to the +* Input Plugins must be added to the `github.com/influxdb/telegraf/plugins/inputs/all/all.go` file. * The `SampleConfig` function should return valid toml that describes how the plugin can be configured. This is include in `telegraf -sample-config`. * The `Description` function should say in one line what this plugin does. -### Plugin interface +### Input interface ```go -type Plugin interface { +type Input interface { SampleConfig() string Description() string Gather(Accumulator) error @@ -52,45 +50,25 @@ type Accumulator interface { The way that a plugin emits metrics is by interacting with the Accumulator. The `Add` function takes 3 arguments: -* **measurement**: A string description of the metric. For instance `bytes_read` or `faults`. +* **measurement**: A string description of the metric. For instance `bytes_read` or ` +faults`. * **value**: A value for the metric. This accepts 5 different types of value: * **int**: The most common type. All int types are accepted but favor using `int64` Useful for counters, etc. * **float**: Favor `float64`, useful for gauges, percentages, etc. - * **bool**: `true` or `false`, useful to indicate the presence of a state. `light_on`, etc. - * **string**: Typically used to indicate a message, or some kind of freeform information. - * **time.Time**: Useful for indicating when a state last occurred, for instance `light_on_since`. + * **bool**: `true` or `false`, useful to indicate the presence of a state. `light_on`, + etc. + * **string**: Typically used to indicate a message, or some kind of freeform + information. + * **time.Time**: Useful for indicating when a state last occurred, for instance ` + light_on_since`. * **tags**: This is a map of strings to strings to describe the where or who about the metric. For instance, the `net` plugin adds a tag named `"interface"` set to the name of the network interface, like `"eth0"`. -The `AddFieldsWithTime` allows multiple values for a point to be passed. The values -used are the same type profile as **value** above. The **timestamp** argument -allows a point to be registered as having occurred at an arbitrary time. - Let's say you've written a plugin that emits metrics about processes on the current host. -```go - -type Process struct { - CPUTime float64 - MemoryBytes int64 - PID int -} - -func Gather(acc inputs.Accumulator) error { - for _, process := range system.Processes() { - tags := map[string]string { - "pid": fmt.Sprintf("%d", process.Pid), - } - - acc.Add("cpu", process.CPUTime, tags, time.Now()) - acc.Add("memory", process.MemoryBytes, tags, time.Now()) - } -} -``` - -### Plugin Example +### Input Plugin Example ```go package simple @@ -126,15 +104,15 @@ func init() { } ``` -## Service Plugins +## Service Input Plugins This section is for developers who want to create new "service" collection inputs. A service plugin differs from a regular plugin in that it operates a background service while Telegraf is running. One example would be the `statsd` plugin, which operates a statsd server. -Service Plugins are substantially more complicated than a regular plugin, as they -will require threads and locks to verify data integrity. Service Plugins should +Service Input Plugins are substantially more complicated than a regular plugin, as they +will require threads and locks to verify data integrity. Service Input Plugins should be avoided unless there is no way to create their behavior with a regular plugin. Their interface is quite similar to a regular plugin, with the addition of `Start()` @@ -157,13 +135,13 @@ type ServicePlugin interface { } ``` -## Outputs +## Output Plugins This section is for developers who want to create a new output sink. Outputs are created in a similar manner as collection plugins, and their interface has similar constructs. -### Output Guidelines +### Output Plugin Guidelines * An output must conform to the `outputs.Output` interface. * Outputs should call `outputs.Add` in their `init` function to register themselves. @@ -230,7 +208,7 @@ func init() { ``` -## Service Outputs +## Service Output Plugins This section is for developers who want to create new "service" output. A service output differs from a regular output in that it operates a background service @@ -243,7 +221,7 @@ and `Stop()` methods. ### Service Output Guidelines * Same as the `Output` guidelines, except that they must conform to the -`inputs.ServiceOutput` interface. +`output.ServiceOutput` interface. ### Service Output interface @@ -274,7 +252,7 @@ which would take some time to replicate. To overcome this situation we've decided to use docker containers to provide a fast and reproducible environment to test those services which require it. For other situations -(i.e: https://github.com/influxdb/telegraf/blob/master/plugins/redis/redis_test.go ) +(i.e: https://github.com/influxdb/telegraf/blob/master/plugins/redis/redis_test.go) a simple mock will suffice. To execute Telegraf tests follow these simple steps: diff --git a/README.md b/README.md index 46f865951..e8a7cc6a3 100644 --- a/README.md +++ b/README.md @@ -1,23 +1,35 @@ -# Telegraf - A native agent for InfluxDB [![Circle CI](https://circleci.com/gh/influxdata/telegraf.svg?style=svg)](https://circleci.com/gh/influxdata/telegraf) +# Telegraf [![Circle CI](https://circleci.com/gh/influxdata/telegraf.svg?style=svg)](https://circleci.com/gh/influxdata/telegraf) Telegraf is an agent written in Go for collecting metrics from the system it's -running on, or from other services, and writing them into InfluxDB. +running on, or from other services, and writing them into InfluxDB or other +[outputs](https://github.com/influxdata/telegraf#supported-output-plugins). Design goals are to have a minimal memory footprint with a plugin system so that developers in the community can easily add support for collecting metrics from well known services (like Hadoop, Postgres, or Redis) and third party APIs (like Mailchimp, AWS CloudWatch, or Google Analytics). -We'll eagerly accept pull requests for new plugins and will manage the set of -plugins that Telegraf supports. See the -[contributing guide](CONTRIBUTING.md) for instructions on -writing new inputs. +New input and output plugins are designed to be easy to contribute, +we'll eagerly accept pull +requests and will manage the set of plugins that Telegraf supports. +See the [contributing guide](CONTRIBUTING.md) for instructions on writing +new plugins. ## Installation: +NOTE: Telegraf 0.3.x is **not** backwards-compatible with previous versions of +telegraf, both in the database layout and the configuration file. 0.2.x will +continue to be supported, see below for download links. + +TODO: link to blog post about 0.3.x changes. + ### Linux deb and rpm packages: Latest: +* http://get.influxdb.org/telegraf/telegraf_0.3.0_amd64.deb +* http://get.influxdb.org/telegraf/telegraf-0.3.0-1.x86_64.rpm + +0.2.x: * http://get.influxdb.org/telegraf/telegraf_0.2.4_amd64.deb * http://get.influxdb.org/telegraf/telegraf-0.2.4-1.x86_64.rpm @@ -33,6 +45,11 @@ controlled via `systemctl [action] telegraf` ### Linux binaries: Latest: +* http://get.influxdb.org/telegraf/telegraf_linux_amd64_0.3.0.tar.gz +* http://get.influxdb.org/telegraf/telegraf_linux_386_0.3.0.tar.gz +* http://get.influxdb.org/telegraf/telegraf_linux_arm_0.3.0.tar.gz + +0.2.x: * http://get.influxdb.org/telegraf/telegraf_linux_amd64_0.2.4.tar.gz * http://get.influxdb.org/telegraf/telegraf_linux_386_0.2.4.tar.gz * http://get.influxdb.org/telegraf/telegraf_linux_arm_0.2.4.tar.gz @@ -51,32 +68,6 @@ brew update brew install telegraf ``` -### Version 0.3.0 Beta - -Version 0.3.0 will introduce many new breaking changes to Telegraf. For starters, -plugin measurements will be aggregated into fields. This means that there will no -longer be a `cpu_usage_idle` measurement, there will be a `cpu` measurement with -a `usage_idle` field. - -There will also be config file changes, meaning that your 0.2.x Telegraf config -files will no longer work properly. It is recommended that you use the -`-sample-config` flag to generate a new config file to see what the changes are. -You can also read the -[0.3.0 configuration guide](https://github.com/influxdb/telegraf/blob/0.3.0/CONFIGURATION.md) -to see some of the new features and options available. - -You can read more about the justifications for the aggregated measurements -[here](https://github.com/influxdb/telegraf/issues/152), and a more detailed -breakdown of the work [here](https://github.com/influxdb/telegraf/pull/437). -Once we're closer to a full release, there will be a detailed blog post -explaining all the changes. - -* http://get.influxdb.org/telegraf/telegraf_0.3.0-beta2_amd64.deb -* http://get.influxdb.org/telegraf/telegraf-0.3.0_beta2-1.x86_64.rpm -* http://get.influxdb.org/telegraf/telegraf_linux_amd64_0.3.0-beta2.tar.gz -* http://get.influxdb.org/telegraf/telegraf_linux_386_0.3.0-beta2.tar.gz -* http://get.influxdb.org/telegraf/telegraf_linux_arm_0.3.0-beta2.tar.gz - ### From Source: Telegraf manages dependencies via [gdm](https://github.com/sparrc/gdm), @@ -91,37 +82,49 @@ if you don't have it already. You also must build with golang version 1.4+. ### How to use it: -* Run `telegraf -sample-config > telegraf.conf` to create an initial configuration. -* Or run `telegraf -sample-config -input-filter cpu:mem -output-filter influxdb > telegraf.conf`. -to create a config file with only CPU and memory plugins defined, and InfluxDB -output defined. -* Edit the configuration to match your needs. -* Run `telegraf -config telegraf.conf -test` to output one full measurement -sample to STDOUT. NOTE: you may want to run as the telegraf user if you are using -the linux packages `sudo -u telegraf telegraf -config telegraf.conf -test` -* Run `telegraf -config telegraf.conf` to gather and send metrics to configured outputs. -* Run `telegraf -config telegraf.conf -input-filter system:swap`. -to run telegraf with only the system & swap plugins defined in the config. +```console +$ telegraf -help +Telegraf, The plugin-driven server agent for reporting metrics into InfluxDB -## Telegraf Options +Usage: -Telegraf has a few options you can configure under the `agent` section of the -config. + telegraf -* **hostname**: The hostname is passed as a tag. By default this will be -the value returned by `hostname` on the machine running Telegraf. -You can override that value here. -* **interval**: How often to gather metrics. Uses a simple number + -unit parser, e.g. "10s" for 10 seconds or "5m" for 5 minutes. -* **debug**: Set to true to gather and send metrics to STDOUT as well as -InfluxDB. +The flags are: + + -config configuration file to load + -test gather metrics once, print them to stdout, and exit + -sample-config print out full sample configuration to stdout + -config-directory directory containing additional *.conf files + -input-filter filter the input plugins to enable, separator is : + -output-filter filter the output plugins to enable, separator is : + -usage print usage for a plugin, ie, 'telegraf -usage mysql' + -version print the version to stdout + +Examples: + + # generate a telegraf config file: + telegraf -sample-config > telegraf.conf + + # generate config with only cpu input & influxdb output plugins defined + telegraf -sample-config -input-filter cpu -output-filter influxdb + + # run a single telegraf collection, outputing metrics to stdout + telegraf -config telegraf.conf -test + + # run telegraf with all plugins defined in config file + telegraf -config telegraf.conf + + # run telegraf, enabling the cpu & memory input, and influxdb output plugins + telegraf -config telegraf.conf -input-filter cpu:mem -output-filter influxdb +``` ## Configuration See the [configuration guide](CONFIGURATION.md) for a rundown of the more advanced configuration options. -## Supported Plugins +## Supported Input Plugins **You can view usage instructions for each plugin by running** `telegraf -usage `. @@ -166,7 +169,7 @@ Telegraf currently has support for collecting metrics from: * diskio * swap -## Supported Service Plugins +## Supported Input Service Plugins Telegraf can collect metrics via the following services: @@ -176,7 +179,7 @@ Telegraf can collect metrics via the following services: We'll be adding support for many more over the coming months. Read on if you want to add support for another service or third-party API. -## Supported Outputs +## Supported Output Plugins * influxdb * nsq diff --git a/cmd/telegraf/telegraf.go b/cmd/telegraf/telegraf.go index cc54dbc88..21e89ce04 100644 --- a/cmd/telegraf/telegraf.go +++ b/cmd/telegraf/telegraf.go @@ -31,43 +31,50 @@ var fOutputFilters = flag.String("output-filter", "", var fUsage = flag.String("usage", "", "print usage for a plugin, ie, 'telegraf -usage mysql'") +var fInputFiltersLegacy = flag.String("filter", "", + "filter the plugins to enable, separator is :") +var fOutputFiltersLegacy = flag.String("outputfilter", "", + "filter the outputs to enable, separator is :") +var fConfigDirectoryLegacy = flag.String("configdirectory", "", + "directory containing additional *.conf files") + // Telegraf version // -ldflags "-X main.Version=`git describe --always --tags`" var Version string -const usage = `Telegraf, The plugin-driven server agent for reporting metrics into InfluxDB +const usage = `Telegraf, The plugin-driven server agent for collecting and reporting metrics. Usage: - telegraf + telegraf The flags are: - -config configuration file to load - -test gather metrics once, print them to stdout, and exit - -sample-config print out full sample configuration to stdout - -config-directory directory containing additional *.conf files - -input-filter filter the input plugins to enable, separator is : - -output-filter filter the output plugins to enable, separator is : - -usage print usage for a plugin, ie, 'telegraf -usage mysql' - -version print the version to stdout + -config configuration file to load + -test gather metrics once, print them to stdout, and exit + -sample-config print out full sample configuration to stdout + -config-directory directory containing additional *.conf files + -input-filter filter the input plugins to enable, separator is : + -output-filter filter the output plugins to enable, separator is : + -usage print usage for a plugin, ie, 'telegraf -usage mysql' + -version print the version to stdout Examples: - # generate a telegraf config file: - telegraf -sample-config > telegraf.conf + # generate a telegraf config file: + telegraf -sample-config > telegraf.conf - # generate a telegraf config file with only cpu input and influxdb output enabled - telegraf -sample-config -input-filter cpu -output-filter influxdb + # generate config with only cpu input & influxdb output plugins defined + telegraf -sample-config -input-filter cpu -output-filter influxdb - # run a single telegraf collection, outputting metrics to stdout - telegraf -config telegraf.conf -test + # run a single telegraf collection, outputing metrics to stdout + telegraf -config telegraf.conf -test - # run telegraf with all plugins defined in config file - telegraf -config telegraf.conf + # run telegraf with all plugins defined in config file + telegraf -config telegraf.conf - # run telegraf, enabling only the cpu and memory inputs and influxdb output - telegraf -config telegraf.conf -input-filter cpu:mem -output-filter influxdb + # run telegraf, enabling the cpu & memory input, and influxdb output plugins + telegraf -config telegraf.conf -input-filter cpu:mem -output-filter influxdb ` func main() { @@ -79,12 +86,20 @@ func main() { } var inputFilters []string + if *fInputFiltersLegacy != "" { + inputFilter := strings.TrimSpace(*fInputFiltersLegacy) + inputFilters = strings.Split(":"+inputFilter+":", ":") + } if *fInputFilters != "" { inputFilter := strings.TrimSpace(*fInputFilters) inputFilters = strings.Split(":"+inputFilter+":", ":") } var outputFilters []string + if *fOutputFiltersLegacy != "" { + outputFilter := strings.TrimSpace(*fOutputFiltersLegacy) + outputFilters = strings.Split(":"+outputFilter+":", ":") + } if *fOutputFilters != "" { outputFilter := strings.TrimSpace(*fOutputFilters) outputFilters = strings.Split(":"+outputFilter+":", ":") @@ -129,6 +144,13 @@ func main() { return } + if *fConfigDirectoryLegacy != "" { + err = c.LoadDirectory(*fConfigDirectoryLegacy) + if err != nil { + log.Fatal(err) + } + } + if *fConfigDirectory != "" { err = c.LoadDirectory(*fConfigDirectory) if err != nil {