From bf9992b6136526d4f82b5851275be4ccd8c1cb34 Mon Sep 17 00:00:00 2001 From: Cameron Sparr Date: Tue, 8 Sep 2015 15:08:40 -0600 Subject: [PATCH 01/10] Update telegraf.service and packaging script for systemd Deals with most of #170 --- package.sh | 51 ++++++++++++++++++++++++++++------------ scripts/telegraf.service | 2 ++ 2 files changed, 38 insertions(+), 15 deletions(-) diff --git a/package.sh b/package.sh index cea7cdf26..fd7055be7 100755 --- a/package.sh +++ b/package.sh @@ -39,6 +39,7 @@ LOGROTATE_DIR=/etc/logrotate.d SAMPLE_CONFIGURATION=etc/config.sample.toml LOGROTATE_CONFIGURATION=etc/logrotate.d/telegraf INITD_SCRIPT=scripts/init.sh +SYSTEMD_SCRIPT=scripts/telegraf.service TMP_WORK_DIR=`mktemp -d` POST_INSTALL_PATH=`mktemp` @@ -156,27 +157,41 @@ generate_postinstall_script() { cat <$POST_INSTALL_PATH rm -f $INSTALL_ROOT_DIR/telegraf rm -f $INSTALL_ROOT_DIR/init.sh -ln -s $INSTALL_ROOT_DIR/versions/$version/telegraf $INSTALL_ROOT_DIR/telegraf -ln -s $INSTALL_ROOT_DIR/versions/$version/scripts/init.sh $INSTALL_ROOT_DIR/init.sh - -rm -f /etc/init.d/telegraf -ln -sfn $INSTALL_ROOT_DIR/init.sh /etc/init.d/telegraf -chmod +x /etc/init.d/telegraf -if which update-rc.d > /dev/null 2>&1 ; then - update-rc.d -f telegraf remove - update-rc.d telegraf defaults -else - chkconfig --add telegraf -fi +ln -sfn $INSTALL_ROOT_DIR/versions/$version/telegraf $INSTALL_ROOT_DIR/telegraf if ! id telegraf >/dev/null 2>&1; then useradd --system -U -M telegraf fi + +# Systemd +if which systemctl > /dev/null 2>&1 ; then + cp $INSTALL_ROOT_DIR/versions/$version/scripts/telegraf.service \ + /lib/systemd/system/telegraf.service + systemctl enable telegraf + +# Sysv +else + ln -sfn $INSTALL_ROOT_DIR/versions/$version/scripts/init.sh \ + $INSTALL_ROOT_DIR/init.sh + rm -f /etc/init.d/telegraf + ln -sfn $INSTALL_ROOT_DIR/init.sh /etc/init.d/telegraf + chmod +x /etc/init.d/telegraf + # update-rc.d sysv service: + if which update-rc.d > /dev/null 2>&1 ; then + update-rc.d -f telegraf remove + update-rc.d telegraf defaults + # CentOS-style sysv: + else + chkconfig --add telegraf + fi + + mkdir -p $TELEGRAF_LOG_DIR + chown -R -L telegraf:telegraf $TELEGRAF_LOG_DIR +fi + chown -R -L telegraf:telegraf $INSTALL_ROOT_DIR chmod -R a+rX $INSTALL_ROOT_DIR -mkdir -p $TELEGRAF_LOG_DIR -chown -R -L telegraf:telegraf $TELEGRAF_LOG_DIR EOF echo "Post-install script created successfully at $POST_INSTALL_PATH" } @@ -213,13 +228,19 @@ done echo "${BINS[*]} copied to $TMP_WORK_DIR/$INSTALL_ROOT_DIR/versions/$VERSION" cp $INITD_SCRIPT $TMP_WORK_DIR/$INSTALL_ROOT_DIR/versions/$VERSION/scripts - if [ $? -ne 0 ]; then echo "Failed to copy init.d script to packaging directory -- aborting." cleanup_exit 1 fi echo "$INITD_SCRIPT copied to $TMP_WORK_DIR/$INSTALL_ROOT_DIR/versions/$VERSION/scripts" +cp $SYSTEMD_SCRIPT $TMP_WORK_DIR/$INSTALL_ROOT_DIR/versions/$VERSION/scripts +if [ $? -ne 0 ]; then + echo "Failed to copy systemd file to packaging directory -- aborting." + cleanup_exit 1 +fi +echo "$SYSTEMD_SCRIPT copied to $TMP_WORK_DIR/$INSTALL_ROOT_DIR/versions/$VERSION/scripts" + cp $SAMPLE_CONFIGURATION $TMP_WORK_DIR/$CONFIG_ROOT_DIR/telegraf.conf if [ $? -ne 0 ]; then echo "Failed to copy $SAMPLE_CONFIGURATION to packaging directory -- aborting." diff --git a/scripts/telegraf.service b/scripts/telegraf.service index a5a764fef..87fcd0f22 100644 --- a/scripts/telegraf.service +++ b/scripts/telegraf.service @@ -8,6 +8,8 @@ EnvironmentFile=-/etc/default/telegraf User=telegraf ExecStart=/opt/telegraf/telegraf -config /etc/opt/telegraf/telegraf.conf $TELEGRAF_OPTS Restart=on-failure +KillMode=process [Install] WantedBy=multi-user.target +Alias=telegraf.service From 0780ad4ad9fcea2d90a8610e94ca941ba2249d50 Mon Sep 17 00:00:00 2001 From: Cameron Sparr Date: Wed, 9 Sep 2015 11:00:19 -0600 Subject: [PATCH 02/10] README updates for systemd and deb/rpm install --- CHANGELOG.md | 9 ++++- CONTRIBUTING.md | 31 ++++++++++++++++ README.md | 95 +++++++++++++++++++------------------------------ 3 files changed, 76 insertions(+), 59 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 26e307908..1222c9728 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,11 @@ -## v0.1.8 [unreleased] +## v0.1.9 [unreleased] + +### Features + +### Bugfixes +- [#170](https://github.com/influxdb/telegraf/issues/170): Systemd support + +## v0.1.8 [2015-09-04] ### Release Notes - Telegraf will now write data in UTC at second precision by default diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c28e52131..3e426594e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -128,3 +128,34 @@ func init() { ## Outputs TODO: this section will describe requirements for contributing an output + +## Unit Tests + +### Execute short tests + +execute `make test-short` + +### Execute long tests + +As Telegraf collects metrics from several third-party services it becomes a +difficult task to mock each service as some of them have complicated protocols +which would take some time to replicate. + +To overcome this situation we've decided to use docker containers to provide a +fast and reproducible environment to test those services which require it. +For other situations +(i.e: https://github.com/influxdb/telegraf/blob/master/plugins/redis/redis_test.go ) +a simple mock will suffice. + +To execute Telegraf tests follow these simple steps: + +- Install docker compose following [these](https://docs.docker.com/compose/install/) +instructions + - mac users should be able to simply do `brew install boot2docker` + and `brew install docker-compose` +- execute `make test` + +### Unit test troubleshooting + +Try cleaning up your test environment by executing `make test-cleanup` and +re-running diff --git a/README.md b/README.md index a5c7c921d..748fd4890 100644 --- a/README.md +++ b/README.md @@ -1,46 +1,58 @@ # Telegraf - A native agent for InfluxDB [![Circle CI](https://circleci.com/gh/influxdb/telegraf.svg?style=svg)](https://circleci.com/gh/influxdb/telegraf) Telegraf is an agent written in Go for collecting metrics from the system it's -running on or from other services and writing them into InfluxDB. +running on, or from other services, and writing them into InfluxDB. Design goals are to have a minimal memory footprint with a plugin system so that developers in the community can easily add support for collecting metrics -from well known services (like Hadoop, or Postgres, or Redis) and third party +from well known services (like Hadoop, Postgres, or Redis) and third party APIs (like Mailchimp, AWS CloudWatch, or Google Analytics). We'll eagerly accept pull requests for new plugins and will manage the set of -plugins that Telegraf supports. See the bottom of this doc for instructions on +plugins that Telegraf supports. See the +[contributing guide](CONTRIBUTING.md) for instructions on writing new plugins. -## Quickstart +## Installation: -* Build from source or download telegraf: - -### Linux packages for Debian/Ubuntu and RHEL/CentOS: - -NOTE: version 0.1.4+ has introduced some breaking changes! A 0.1.4+ telegraf -agent is NOT backwards-compatible with a config file from 0.1.3 and below. -That being said, the difference is not huge, see below for an example on -how to setup the new config file. - -As well, due to a breaking change to the InfluxDB integer line-protocol, there +Due to a breaking change to the InfluxDB integer line-protocol, there are some InfluxDB compatibility requirements: -* InfluxDB 0.9.3+ (including nightly builds) requires Telegraf 0.1.5+ +* InfluxDB 0.9.3+ requires Telegraf 0.1.5+ * InfluxDB 0.9.2 and prior requires Telegraf 0.1.4 +### Linux deb and rpm packages: + Latest: * http://get.influxdb.org/telegraf/telegraf_0.1.8_amd64.deb * http://get.influxdb.org/telegraf/telegraf-0.1.8-1.x86_64.rpm -Binaries: +0.1.4: +* http://get.influxdb.org/telegraf/telegraf_0.1.4_amd64.deb +* http://get.influxdb.org/telegraf/telegraf-0.1.4-1.x86_64.rpm + +##### Package instructions: + +* Telegraf binary is installed in `/opt/telegraf/telegraf` +* Telegraf daemon configuration file is in `/etc/opt/telegraf/telegraf.conf` +* On sysv systems, the telegraf daemon can be controlled via +`service telegraf [action]` +* On systemd systems (such as Ubuntu 15+), the telegraf daemon can be +controlled via `systemctl [action] telegraf` + +### Linux binaries: + +Latest: * http://get.influxdb.org/telegraf/telegraf_linux_amd64_0.1.8.tar.gz * http://get.influxdb.org/telegraf/telegraf_linux_386_0.1.8.tar.gz * http://get.influxdb.org/telegraf/telegraf_linux_arm_0.1.8.tar.gz -0.1.4: -* http://get.influxdb.org/telegraf/telegraf_0.1.4_amd64.deb -* http://get.influxdb.org/telegraf/telegraf-0.1.4-1.x86_64.rpm +##### Binary instructions: + +These are standalone binaries that can be unpacked and executed on any linux +system. They can be unpacked and renamed in a location such as +`/usr/local/bin` for convenience. A config file will need to be generated, +see "How to use it" below. ### OSX via Homebrew: @@ -62,19 +74,17 @@ if you don't have it already. You also must build with golang version 1.4+ ### How to use it: -* Run `telegraf -sample-config > telegraf.toml` to create an initial configuration +* Run `telegraf -sample-config > telegraf.conf` to create an initial configuration * Edit the configuration to match your needs -* Run `telegraf -config telegraf.toml -test` to output one full measurement sample to STDOUT -* Run `telegraf -config telegraf.toml` to gather and send metrics to configured outputs. -* Run `telegraf -config telegraf.toml -filter system:swap` +* Run `telegraf -config telegraf.conf -test` to output one full measurement sample to STDOUT +* Run `telegraf -config telegraf.conf` to gather and send metrics to configured outputs. +* Run `telegraf -config telegraf.conf -filter system:swap` to enable only the system & swap plugins defined in the config. ## Telegraf Options Telegraf has a few options you can configure under the `agent` section of the -config. If you don't see an `agent` section run -`telegraf -sample-config > telegraf.toml` to create a valid initial -configuration: +config. * **hostname**: The hostname is passed as a tag. By default this will be the value retured by `hostname` on the machine running Telegraf. @@ -194,36 +204,5 @@ found by running `telegraf -sample-config` ## Contributing Please see the -[contributing guide](https://github.com/influxdb/telegraf/blob/master/CONTRIBUTING.md) +[contributing guide](CONTRIBUTING.md) for details on contributing a plugin or output to Telegraf - -## Testing - -### Execute short tests - -execute `make test-short` - -### Execute long tests - -As Telegraf collects metrics from several third-party services it becomes a -difficult task to mock each service as some of them have complicated protocols -which would take some time to replicate. - -To overcome this situation we've decided to use docker containers to provide a -fast and reproducible environment to test those services which require it. -For other situations -(i.e: https://github.com/influxdb/telegraf/blob/master/plugins/redis/redis_test.go ) -a simple mock will suffice. - -To execute Telegraf tests follow these simple steps: - -- Install docker compose following [these](https://docs.docker.com/compose/install/) -instructions - - mac users should be able to simply do `brew install boot2docker` - and `brew install docker-compose` -- execute `make test` - -### Unit test troubleshooting - -Try cleaning up your test environment by executing `make test-cleanup` and -re-running From a9b97c7a2bf30ac717663c3484e089761749bf70 Mon Sep 17 00:00:00 2001 From: Cameron Sparr Date: Wed, 9 Sep 2015 12:07:58 -0600 Subject: [PATCH 03/10] Bump go version number to 1.5 --- package.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.sh b/package.sh index fd7055be7..406e7fce3 100755 --- a/package.sh +++ b/package.sh @@ -50,7 +50,7 @@ MAINTAINER=support@influxdb.com VENDOR=InfluxDB DESCRIPTION="InfluxDB Telegraf agent" PKG_DEPS=(coreutils) -GO_VERSION="go1.4.2" +GO_VERSION="go1.5" GOPATH_INSTALL= BINS=( telegraf From a7ed46160ad68937a3c1385d28d736352e1fe883 Mon Sep 17 00:00:00 2001 From: Cameron Sparr Date: Wed, 9 Sep 2015 12:19:07 -0600 Subject: [PATCH 04/10] Re-arrange repo files for root dir cleanup --- CONTRIBUTING.md | 6 ++++-- Makefile | 7 ++++--- circle.yml | 2 +- Vagrantfile => scripts/Vagrantfile | 2 +- circle-test.sh => scripts/circle-test.sh | 0 docker-compose.yml => scripts/docker-compose.yml | 0 package.sh => scripts/package.sh | 2 +- 7 files changed, 11 insertions(+), 8 deletions(-) rename Vagrantfile => scripts/Vagrantfile (94%) rename circle-test.sh => scripts/circle-test.sh (100%) rename docker-compose.yml => scripts/docker-compose.yml (100%) rename package.sh => scripts/package.sh (99%) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3e426594e..c08c50b5b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -151,10 +151,12 @@ To execute Telegraf tests follow these simple steps: - Install docker compose following [these](https://docs.docker.com/compose/install/) instructions - - mac users should be able to simply do `brew install boot2docker` - and `brew install docker-compose` - execute `make test` +**OSX users**: you will need to install `boot2docker` or `docker-machine`. +The Makefile will assume that you have a `docker-machine` box called `default` to +get the IP address. + ### Unit test troubleshooting Try cleaning up your test environment by executing `make test-cleanup` and diff --git a/Makefile b/Makefile index 89db1d5b0..90443b2f9 100644 --- a/Makefile +++ b/Makefile @@ -22,10 +22,11 @@ prepare: docker-compose: ifeq ($(UNAME), Darwin) - ADVERTISED_HOST=$(shell sh -c 'boot2docker ip') docker-compose up -d + ADVERTISED_HOST=$(shell sh -c 'boot2docker ip || docker-machine ip default') \ + docker-compose --file scripts/docker-compose.yml up -d endif ifeq ($(UNAME), Linux) - ADVERTISED_HOST=localhost docker-compose up -d + ADVERTISED_HOST=localhost docker-compose --file scripts/docker-compose.yml up -d endif test: prepare docker-compose @@ -35,6 +36,6 @@ test-short: prepare $(GOPATH)/bin/godep go test -short ./... test-cleanup: - docker-compose kill + docker-compose --file scripts/docker-compose.yml kill .PHONY: test diff --git a/circle.yml b/circle.yml index 2c82bcfa7..0dcd6fc9f 100644 --- a/circle.yml +++ b/circle.yml @@ -4,5 +4,5 @@ dependencies: test: override: - - bash circle-test.sh + - bash scripts/circle-test.sh diff --git a/Vagrantfile b/scripts/Vagrantfile similarity index 94% rename from Vagrantfile rename to scripts/Vagrantfile index 72124a8ac..3c0199bdb 100644 --- a/Vagrantfile +++ b/scripts/Vagrantfile @@ -7,7 +7,7 @@ VAGRANTFILE_API_VERSION = "2" Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| config.vm.box = "ubuntu/trusty64" - config.vm.synced_folder ".", "/home/vagrant/go/src/github.com/influxdb/telegraf", + config.vm.synced_folder "..", "/home/vagrant/go/src/github.com/influxdb/telegraf", type: "rsync", rsync__args: ["--verbose", "--archive", "--delete", "-z", "--safe-links"], rsync__exclude: ["./telegraf", ".vagrant/"] diff --git a/circle-test.sh b/scripts/circle-test.sh similarity index 100% rename from circle-test.sh rename to scripts/circle-test.sh diff --git a/docker-compose.yml b/scripts/docker-compose.yml similarity index 100% rename from docker-compose.yml rename to scripts/docker-compose.yml diff --git a/package.sh b/scripts/package.sh similarity index 99% rename from package.sh rename to scripts/package.sh index 406e7fce3..7f5196e33 100755 --- a/package.sh +++ b/scripts/package.sh @@ -204,7 +204,7 @@ if [ "$1" == "-h" ]; then fi VERSION=`git describe --always --tags | tr -d v` - +cd `git rev-parse --show-toplevel` echo -e "\nStarting package process, version: $VERSION\n" if [ "$CIRCLE_BRANCH" == "" ]; then From 3c7c8926fb436e683c6a74201b1c90f390ef419f Mon Sep 17 00:00:00 2001 From: Cameron Sparr Date: Wed, 9 Sep 2015 15:56:10 -0600 Subject: [PATCH 05/10] Support InfluxDB clusters Closes #143 --- CHANGELOG.md | 5 ++ agent.go | 3 ++ etc/config.sample.toml | 2 +- outputs/influxdb/influxdb.go | 95 ++++++++++++++++++++++++++---------- 4 files changed, 78 insertions(+), 27 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1222c9728..77c8dac8f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,11 @@ ## v0.1.9 [unreleased] +### Release Notes +- InfluxDB output config change: `url` is now `urls`, and is a list. Config files +will still be backwards compatible if only `url` is specified. + ### Features +- [#143](https://github.com/influxdb/telegraf/issues/143): InfluxDB clustering support ### Bugfixes - [#170](https://github.com/influxdb/telegraf/issues/170): Systemd support diff --git a/agent.go b/agent.go index 2bfa0cf8f..1f40eb04b 100644 --- a/agent.go +++ b/agent.go @@ -84,6 +84,9 @@ func NewAgent(config *Config) (*Agent, error) { // Connect connects to all configured outputs func (a *Agent) Connect() error { for _, o := range a.outputs { + if a.Debug { + log.Printf("Attempting connection to output: %s\n", o.name) + } err := o.output.Connect() if err != nil { return err diff --git a/etc/config.sample.toml b/etc/config.sample.toml index 5307b6df8..58c851bec 100644 --- a/etc/config.sample.toml +++ b/etc/config.sample.toml @@ -37,7 +37,7 @@ [outputs] [outputs.influxdb] # The full HTTP endpoint URL for your InfluxDB instance - url = "http://localhost:8086" # required. + urls = ["http://localhost:8086"] # required. # The target database for metrics. This database must already exist database = "telegraf" # required. diff --git a/outputs/influxdb/influxdb.go b/outputs/influxdb/influxdb.go index 5bb74b4e3..c47b6cd8c 100644 --- a/outputs/influxdb/influxdb.go +++ b/outputs/influxdb/influxdb.go @@ -1,8 +1,10 @@ package influxdb import ( + "errors" "fmt" "log" + "math/rand" "net/url" "strings" @@ -12,19 +14,23 @@ import ( ) type InfluxDB struct { + // URL is only for backwards compatability URL string + URLs []string `toml:"urls"` Username string Password string Database string UserAgent string Timeout t.Duration - conn *client.Client + conns []*client.Client } var sampleConfig = ` # The full HTTP endpoint URL for your InfluxDB instance - url = "http://localhost:8086" # required. + # Multiple urls can be specified for InfluxDB cluster support. Server to + # write to will be randomly chosen each interval. + urls = ["http://localhost:8086"] # required. # The target database for metrics. This database must already exist database = "telegraf" # required. @@ -42,33 +48,58 @@ var sampleConfig = ` ` func (i *InfluxDB) Connect() error { - u, err := url.Parse(i.URL) - if err != nil { - return err + var urls []*url.URL + for _, URL := range i.URLs { + u, err := url.Parse(URL) + if err != nil { + return err + } + urls = append(urls, u) } - c, err := client.NewClient(client.Config{ - URL: *u, - Username: i.Username, - Password: i.Password, - UserAgent: i.UserAgent, - Timeout: i.Timeout.Duration, - }) - - if err != nil { - return err + // Backward-compatability with single Influx URL config files + // This could eventually be removed in favor of specifying the urls as a list + if i.URL != "" { + u, err := url.Parse(i.URL) + if err != nil { + return err + } + urls = append(urls, u) } - _, err = c.Query(client.Query{ - Command: fmt.Sprintf("CREATE DATABASE %s", i.Database), - }) - - if err != nil && !strings.Contains(err.Error(), "database already exists") { - log.Fatal(err) + var conns []*client.Client + for _, parsed_url := range urls { + c, err := client.NewClient(client.Config{ + URL: *parsed_url, + Username: i.Username, + Password: i.Password, + UserAgent: i.UserAgent, + Timeout: i.Timeout.Duration, + }) + if err != nil { + return err + } + conns = append(conns, c) } - i.conn = c - return nil + // This will get set to nil if a successful connection is made + err := errors.New("Could not create database on any server") + + for _, conn := range conns { + _, e := conn.Query(client.Query{ + Command: fmt.Sprintf("CREATE DATABASE %s", i.Database), + }) + + if e != nil && !strings.Contains(e.Error(), "database already exists") { + log.Println("ERROR: " + e.Error()) + } else { + err = nil + break + } + } + + i.conns = conns + return err } func (i *InfluxDB) Close() error { @@ -84,12 +115,24 @@ func (i *InfluxDB) Description() string { return "Configuration for influxdb server to send metrics to" } +// Choose a random server in the cluster to write to until a successful write +// occurs, logging each unsuccessful. If all servers fail, return error. func (i *InfluxDB) Write(bp client.BatchPoints) error { bp.Database = i.Database - if _, err := i.conn.Write(bp); err != nil { - return err + + // This will get set to nil if a successful write occurs + err := errors.New("Could not write to any InfluxDB server in cluster") + + p := rand.Perm(len(i.conns)) + for _, n := range p { + if _, e := i.conns[n].Write(bp); e != nil { + log.Println("ERROR: " + e.Error()) + } else { + err = nil + break + } } - return nil + return err } func init() { From 81f4aa9a5dcd8746e370f746ec66954099badcaf Mon Sep 17 00:00:00 2001 From: Cameron Sparr Date: Wed, 9 Sep 2015 21:27:58 -0600 Subject: [PATCH 06/10] Fix bug in setting the precision before gathering metrics Closes #175 --- CHANGELOG.md | 1 + accumulator.go | 6 +++++- agent.go | 17 +++++++++-------- 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 77c8dac8f..8477ba494 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ will still be backwards compatible if only `url` is specified. ### Bugfixes - [#170](https://github.com/influxdb/telegraf/issues/170): Systemd support +- [#175](https://github.com/influxdb/telegraf/issues/175): Set write precision before gathering metrics ## v0.1.8 [2015-09-04] diff --git a/accumulator.go b/accumulator.go index b756cc16e..ea90b6620 100644 --- a/accumulator.go +++ b/accumulator.go @@ -25,7 +25,11 @@ type BatchPoints struct { } // Add adds a measurement -func (bp *BatchPoints) Add(measurement string, val interface{}, tags map[string]string) { +func (bp *BatchPoints) Add( + measurement string, + val interface{}, + tags map[string]string, +) { bp.mu.Lock() defer bp.mu.Unlock() diff --git a/agent.go b/agent.go index 1f40eb04b..12533d286 100644 --- a/agent.go +++ b/agent.go @@ -196,16 +196,17 @@ func (a *Agent) crankParallel() error { go func(plugin *runningPlugin) { defer wg.Done() - var acc BatchPoints - acc.Debug = a.Debug - acc.Prefix = plugin.name + "_" - acc.Config = plugin.config + var bp BatchPoints + bp.Debug = a.Debug + bp.Prefix = plugin.name + "_" + bp.Config = plugin.config + bp.Precision = a.Precision - if err := plugin.plugin.Gather(&acc); err != nil { + if err := plugin.plugin.Gather(&bp); err != nil { log.Printf("Error in plugin [%s]: %s", plugin.name, err) } - points <- &acc + points <- &bp }(plugin) } @@ -233,6 +234,7 @@ func (a *Agent) crank() error { var bp BatchPoints bp.Debug = a.Debug + bp.Precision = a.Precision for _, plugin := range a.plugins { bp.Prefix = plugin.name + "_" @@ -248,7 +250,6 @@ func (a *Agent) crank() error { if a.UTC { bp.Time = bp.Time.UTC() } - bp.Precision = a.Precision return a.flush(&bp) } @@ -266,6 +267,7 @@ func (a *Agent) crankSeparate(shutdown chan struct{}, plugin *runningPlugin) err bp.Prefix = plugin.name + "_" bp.Config = plugin.config + bp.Precision = a.Precision if err := plugin.plugin.Gather(&bp); err != nil { log.Printf("Error in plugin [%s]: %s", plugin.name, err) @@ -277,7 +279,6 @@ func (a *Agent) crankSeparate(shutdown chan struct{}, plugin *runningPlugin) err if a.UTC { bp.Time = bp.Time.UTC() } - bp.Precision = a.Precision if err := a.flush(&bp); err != nil { outerr = errors.New("Error encountered processing plugins & outputs") From a55f6498c86838412769f1991cf9ca5fabc80ca2 Mon Sep 17 00:00:00 2001 From: Vye Wilson Date: Thu, 10 Sep 2015 10:01:08 -0700 Subject: [PATCH 07/10] Makefile will now honor GOBIN, if set Closes #181 --- CHANGELOG.md | 1 + Makefile | 15 +++++++++------ 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8477ba494..5cea99e8d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ will still be backwards compatible if only `url` is specified. ### Features - [#143](https://github.com/influxdb/telegraf/issues/143): InfluxDB clustering support +- [#181](https://github.com/influxdb/telegraf/issues/181): Makefile GOBIN support. Thanks @Vye! ### Bugfixes - [#170](https://github.com/influxdb/telegraf/issues/170): Systemd support diff --git a/Makefile b/Makefile index 90443b2f9..a907f468d 100644 --- a/Makefile +++ b/Makefile @@ -1,19 +1,22 @@ UNAME := $(shell sh -c 'uname') VERSION := $(shell sh -c 'git describe --always --tags') +ifndef GOBIN + GOBIN = $(GOPATH)/bin +endif build: prepare - $(GOPATH)/bin/godep go build -o telegraf -ldflags \ + $(GOBIN)/godep go build -o telegraf -ldflags \ "-X main.Version $(VERSION)" \ ./cmd/telegraf/telegraf.go build-linux-bins: prepare - GOARCH=amd64 GOOS=linux $(GOPATH)/bin/godep go build -o telegraf_linux_amd64 \ + GOARCH=amd64 GOOS=linux $(GOBIN)/godep go build -o telegraf_linux_amd64 \ -ldflags "-X main.Version $(VERSION)" \ ./cmd/telegraf/telegraf.go - GOARCH=386 GOOS=linux $(GOPATH)/bin/godep go build -o telegraf_linux_386 \ + GOARCH=386 GOOS=linux $(GOBIN)/godep go build -o telegraf_linux_386 \ -ldflags "-X main.Version $(VERSION)" \ ./cmd/telegraf/telegraf.go - GOARCH=arm GOOS=linux $(GOPATH)/bin/godep go build -o telegraf_linux_arm \ + GOARCH=arm GOOS=linux $(GOBIN)/godep go build -o telegraf_linux_arm \ -ldflags "-X main.Version $(VERSION)" \ ./cmd/telegraf/telegraf.go @@ -30,10 +33,10 @@ ifeq ($(UNAME), Linux) endif test: prepare docker-compose - $(GOPATH)/bin/godep go test -v ./... + $(GOBIN)/godep go test -v ./... test-short: prepare - $(GOPATH)/bin/godep go test -short ./... + $(GOBIN)/godep go test -short ./... test-cleanup: docker-compose --file scripts/docker-compose.yml kill From f7a43179904510e8aae8fb8a296d6e7b1011e5a1 Mon Sep 17 00:00:00 2001 From: Cameron Sparr Date: Thu, 10 Sep 2015 11:21:40 -0600 Subject: [PATCH 08/10] Fix multiple redis server bug, do not cache the TCP connections Fixes #178 --- CHANGELOG.md | 1 + plugins/redis/redis.go | 63 +++++++++++++++++++----------------------- 2 files changed, 29 insertions(+), 35 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5cea99e8d..e0888ec75 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ will still be backwards compatible if only `url` is specified. ### Bugfixes - [#170](https://github.com/influxdb/telegraf/issues/170): Systemd support - [#175](https://github.com/influxdb/telegraf/issues/175): Set write precision before gathering metrics +- [#178](https://github.com/influxdb/telegraf/issues/178): redis plugin, multiple server thread hang bug ## v0.1.8 [2015-09-04] diff --git a/plugins/redis/redis.go b/plugins/redis/redis.go index 4a95772aa..30077bc4f 100644 --- a/plugins/redis/redis.go +++ b/plugins/redis/redis.go @@ -15,9 +15,6 @@ import ( type Redis struct { Servers []string - - c net.Conn - buf []byte } var sampleConfig = ` @@ -112,41 +109,37 @@ func (r *Redis) Gather(acc plugins.Accumulator) error { const defaultPort = "6379" func (r *Redis) gatherServer(addr *url.URL, acc plugins.Accumulator) error { - if r.c == nil { - - _, _, err := net.SplitHostPort(addr.Host) - if err != nil { - addr.Host = addr.Host + ":" + defaultPort - } - - c, err := net.Dial("tcp", addr.Host) - if err != nil { - return fmt.Errorf("Unable to connect to redis server '%s': %s", addr.Host, err) - } - - if addr.User != nil { - pwd, set := addr.User.Password() - if set && pwd != "" { - c.Write([]byte(fmt.Sprintf("AUTH %s\r\n", pwd))) - - rdr := bufio.NewReader(c) - - line, err := rdr.ReadString('\n') - if err != nil { - return err - } - if line[0] != '+' { - return fmt.Errorf("%s", strings.TrimSpace(line)[1:]) - } - } - } - - r.c = c + _, _, err := net.SplitHostPort(addr.Host) + if err != nil { + addr.Host = addr.Host + ":" + defaultPort } - r.c.Write([]byte("info\r\n")) + c, err := net.Dial("tcp", addr.Host) + if err != nil { + return fmt.Errorf("Unable to connect to redis server '%s': %s", addr.Host, err) + } + defer c.Close() - rdr := bufio.NewReader(r.c) + if addr.User != nil { + pwd, set := addr.User.Password() + if set && pwd != "" { + c.Write([]byte(fmt.Sprintf("AUTH %s\r\n", pwd))) + + rdr := bufio.NewReader(c) + + line, err := rdr.ReadString('\n') + if err != nil { + return err + } + if line[0] != '+' { + return fmt.Errorf("%s", strings.TrimSpace(line)[1:]) + } + } + } + + c.Write([]byte("info\r\n")) + + rdr := bufio.NewReader(c) line, err := rdr.ReadString('\n') if err != nil { From d8482cc286ad4afc78cc707702b5494450479d4a Mon Sep 17 00:00:00 2001 From: Cameron Sparr Date: Thu, 10 Sep 2015 13:57:19 -0600 Subject: [PATCH 09/10] darwin net plugin fix, really need to godep vendor gopsutil --- CHANGELOG.md | 1 + plugins/system/ps/net/net_darwin.go | 20 +++++++++++++------- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e0888ec75..4973337c9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ will still be backwards compatible if only `url` is specified. - [#170](https://github.com/influxdb/telegraf/issues/170): Systemd support - [#175](https://github.com/influxdb/telegraf/issues/175): Set write precision before gathering metrics - [#178](https://github.com/influxdb/telegraf/issues/178): redis plugin, multiple server thread hang bug +- Fix net plugin on darwin ## v0.1.8 [2015-09-04] diff --git a/plugins/system/ps/net/net_darwin.go b/plugins/system/ps/net/net_darwin.go index a9843fea3..7fb21713c 100644 --- a/plugins/system/ps/net/net_darwin.go +++ b/plugins/system/ps/net/net_darwin.go @@ -7,7 +7,7 @@ import ( "strconv" "strings" - "github.com/influxdb/telegraf/plugins/system/ps/common" + "github.com/shirou/gopsutil/common" ) func NetIOCounters(pernic bool) ([]NetIOCountersStat, error) { @@ -26,7 +26,7 @@ func NetIOCounters(pernic bool) ([]NetIOCountersStat, error) { // skip first line continue } - if common.StringContains(exists, values[0]) { + if common.StringsHas(exists, values[0]) { // skip if already get continue } @@ -38,11 +38,14 @@ func NetIOCounters(pernic bool) ([]NetIOCountersStat, error) { base = 0 } - parsed := make([]uint64, 0, 3) + parsed := make([]uint64, 0, 6) vv := []string{ - values[base+3], // PacketsRecv - values[base+4], // Errin - values[base+5], // Dropin + values[base+3], // Ipkts == PacketsRecv + values[base+4], // Ierrs == Errin + values[base+5], // Ibytes == BytesRecv + values[base+6], // Opkts == PacketsSent + values[base+7], // Oerrs == Errout + values[base+8], // Obytes == BytesSent } for _, target := range vv { if target == "-" { @@ -61,7 +64,10 @@ func NetIOCounters(pernic bool) ([]NetIOCountersStat, error) { Name: values[0], PacketsRecv: parsed[0], Errin: parsed[1], - Dropin: parsed[2], + BytesRecv: parsed[2], + PacketsSent: parsed[3], + Errout: parsed[4], + BytesSent: parsed[5], } ret = append(ret, n) } From bd00f46d8b239d24d30e96ed1c69c50d2f0e0334 Mon Sep 17 00:00:00 2001 From: Ruslan Islamgaliev Date: Thu, 10 Sep 2015 14:27:50 +0300 Subject: [PATCH 10/10] Fix docker stats to make it work on centos 7. issue #58 issue #84 --- CHANGELOG.md | 1 + plugins/system/ps/docker/docker_linux.go | 18 ++++++++++++++---- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4973337c9..2cfaefda5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ will still be backwards compatible if only `url` is specified. - [#175](https://github.com/influxdb/telegraf/issues/175): Set write precision before gathering metrics - [#178](https://github.com/influxdb/telegraf/issues/178): redis plugin, multiple server thread hang bug - Fix net plugin on darwin +- [#84](https://github.com/influxdb/telegraf/issues/84): Fix docker plugin on CentOS. Thanks @neezgee! ## v0.1.8 [2015-09-04] diff --git a/plugins/system/ps/docker/docker_linux.go b/plugins/system/ps/docker/docker_linux.go index 51f3bc9ca..4c274af28 100644 --- a/plugins/system/ps/docker/docker_linux.go +++ b/plugins/system/ps/docker/docker_linux.go @@ -4,6 +4,7 @@ package docker import ( "encoding/json" + "os" "os/exec" "path" "strconv" @@ -48,9 +49,13 @@ func CgroupCPU(containerid string, base string) (*cpu.CPUTimesStat, error) { if len(base) == 0 { base = "/sys/fs/cgroup/cpuacct/docker" } - path := path.Join(base, containerid, "cpuacct.stat") + statfile := path.Join(base, containerid, "cpuacct.stat") - lines, err := common.ReadLines(path) + if _, err := os.Stat(statfile); os.IsNotExist(err) { + statfile = path.Join("/sys/fs/cgroup/cpuacct/system.slice", "docker-" + containerid + ".scope", "cpuacct.stat") + } + + lines, err := common.ReadLines(statfile) if err != nil { return nil, err } @@ -86,12 +91,17 @@ func CgroupMem(containerid string, base string) (*CgroupMemStat, error) { if len(base) == 0 { base = "/sys/fs/cgroup/memory/docker" } - path := path.Join(base, containerid, "memory.stat") + statfile := path.Join(base, containerid, "memory.stat") + + if _, err := os.Stat(statfile); os.IsNotExist(err) { + statfile = path.Join("/sys/fs/cgroup/memory/system.slice", "docker-" + containerid + ".scope", "memory.stat") + } + // empty containerid means all cgroup if len(containerid) == 0 { containerid = "all" } - lines, err := common.ReadLines(path) + lines, err := common.ReadLines(statfile) if err != nil { return nil, err }