Adding Kafka docker container and utilizing it in unit tests

Makefile

@@ -2,9 +2,12 @@ prepare:
 	go get -d -v -t ./...
 	docker-compose up -d --no-recreate
 
-test: prepare
+test-short: prepare
 	go test -short ./...
 
+test: prepare
+	go test ./...
+
 update:
 	go get -u -v -d -t ./...
 

README.md

@@ -185,10 +185,26 @@ As Telegraf collects metrics from several third-party services it becomes a diff
 some of them have complicated protocols which would take some time to replicate. 
 
 To overcome this situation we've decided to use docker containers to provide a fast and reproducible environment
-to test those services which require it. For other situations (i.e: https://github.com/influxdb/telegraf/blob/master/plugins/redis/redis_test.go ) a simple mock will suffice. 
+to test those services which require it. For other situations (i.e: https://github.com/influxdb/telegraf/blob/master/plugins/redis/redis_test.go ) a simple mock will suffice.
 
-To execute Telegraf tests follow this simple steps:
+To execute Telegraf tests follow these simple steps:
 
 - Install docker compose following [these](https://docs.docker.com/compose/install/) instructions
-- execute `make test`
+    - NOTE: mac users should be able to simply do `brew install boot2docker` and `brew install docker-compose`
 
+### Execute short tests:
+
+execute `make short-test`
+
+### Execute long tests:
+These tests requre additional docker containers, such as for kafka
+
+Mac:
+execute ``ADVERTISED_HOST=`boot2docker ip` make test``
+
+Linux:
+execute `ADVERTISED_HOST=localhost make test`
+
+### Unit test troubleshooting:
+
+Try killing your docker containers via `docker-compose kill` and then re-running

circle.yml

@@ -11,9 +11,13 @@ test:
     - go vet ./...
     # Verify that all files are properly go formatted
     - "[ `git ls-files | grep '.go$' | xargs gofmt -l 2>&1 | wc -l` -eq 0 ]"
+    # Only docker-compose up kafka, the other services are already running
+    # see: https://circleci.com/docs/environment#databases
+    # - docker-compose up -d kafka
   override:
     # Enforce that testutil, cmd, and main directory are fully linted
     - golint .
     - golint testutil/...
     - golint cmd/...
     # TODO run unit tests
+    # - go test -short ./...

docker-compose.yml

@@ -14,3 +14,14 @@ postgres:
     image: postgres
     ports:
         - "5432:5432"
+
+# advertised host env variable must be set at runtime, ie,
+#   ADVERTISED_HOST=`boot2docker ip` docker-compose up -d
+kafka:
+    image: spotify/kafka
+    ports:
+        - "2181:2181"
+        - "9092:9092"
+    environment:
+        ADVERTISED_HOST:
+        ADVERTISED_PORT: 9092

plugins/kafka_consumer/kafka_consumer_integration_test.go

@@ -2,8 +2,6 @@ package kafka_consumer
 
 import (
 	"fmt"
-	"os"
-	"strings"
 	"testing"
 	"time"
 
@@ -19,17 +17,8 @@ func TestReadsMetricsFromKafka(t *testing.T) {
 	}
 	var zkPeers, brokerPeers []string
 
-	if len(os.Getenv("ZOOKEEPER_PEERS")) == 0 {
+	zkPeers = []string{testutil.GetLocalHost() + ":2181"}
-		zkPeers = []string{"localhost:2181"}
+	brokerPeers = []string{testutil.GetLocalHost() + ":9092"}
-	} else {
-		zkPeers = strings.Split(os.Getenv("ZOOKEEPER_PEERS"), ",")
-	}
-
-	if len(os.Getenv("KAFKA_PEERS")) == 0 {
-		brokerPeers = []string{"localhost:9092"}
-	} else {
-		brokerPeers = strings.Split(os.Getenv("KAFKA_PEERS"), ",")
-	}
 
 	k := &Kafka{
 		ConsumerGroupName: "telegraf_test_consumers",