Space Igloo

CHANGELOG.md

@@ -1,4 +1,4 @@
-## v0.13 [2016-05-09]
+## v0.13 [unreleased]
 
 ### Release Notes
 

Godeps

@@ -25,7 +25,7 @@ github.com/gorilla/mux c9e326e2bdec29039a3761c07bece13133863e1e
 github.com/hailocab/go-hostpool e80d13ce29ede4452c43dea11e79b9bc8a15b478
 github.com/hpcloud/tail b2940955ab8b26e19d43a43c4da0475dd81bdb56
 github.com/influxdata/config b79f6829346b8d6e78ba73544b1e1038f1f1c9da
-github.com/influxdata/influxdb e094138084855d444195b252314dfee9eae34cab
+github.com/influxdata/influxdb 21db76b3374c733f37ed16ad93f3484020034351
 github.com/influxdata/toml af4df43894b16e3fd2b788d01bd27ad0776ef2d0
 github.com/klauspost/crc32 19b0b332c9e4516a6370a0456e6182c3b5036720
 github.com/lib/pq e182dc4027e2ded4b19396d638610f2653295f36
@@ -42,7 +42,7 @@ github.com/prometheus/client_model fa8ad6fec33561be4280a8f0514318c79d7f6cb6
 github.com/prometheus/common e8eabff8812b05acf522b45fdcd725a785188e37
 github.com/prometheus/procfs 406e5b7bfd8201a36e2bb5f7bdae0b03380c2ce8
 github.com/samuel/go-zookeeper 218e9c81c0dd8b3b18172b2bbfad92cc7d6db55f
-github.com/shirou/gopsutil 37d89088411de59a4ef9fc340afa0e89dfcb4ea9
+github.com/shirou/gopsutil 1f32ce1bb380845be7f5d174ac641a2c592c0c42
 github.com/soniah/gosnmp b1b4f885b12c5dcbd021c5cee1c904110de6db7d
 github.com/streadway/amqp b4f3ceab0337f013208d31348b578d83c0064744
 github.com/stretchr/testify 1f4a1643a57e798696635ea4c126e9127adb7d3c

Makefile

@@ -14,21 +14,21 @@ windows: prepare-windows build-windows
 
 # Only run the build (no dependency grabbing)
 build:
-	go install -ldflags "-X main.version=$(VERSION)" ./...
+	go install -ldflags "-X main.Version=$(VERSION)" ./...
 
 build-windows:
 	go build -o telegraf.exe -ldflags \
-		"-X main.version=$(VERSION)" \
+		"-X main.Version=$(VERSION)" \
 		./cmd/telegraf/telegraf.go
 
 build-for-docker:
 	CGO_ENABLED=0 GOOS=linux go build -installsuffix cgo -o telegraf -ldflags \
-					"-s -X main.version=$(VERSION)" \
+					"-s -X main.Version=$(VERSION)" \
 					./cmd/telegraf/telegraf.go
 
 # Build with race detector
 dev: prepare
-	go build -race -ldflags "-X main.version=$(VERSION)" ./...
+	go build -race -ldflags "-X main.Version=$(VERSION)" ./...
 
 # run package script
 package:

README.md

@@ -20,12 +20,12 @@ new plugins.
 ### Linux deb and rpm Packages:
 
 Latest:
-* https://dl.influxdata.com/telegraf/releases/telegraf_0.13.0-1_amd64.deb
-* https://dl.influxdata.com/telegraf/releases/telegraf-0.13.0-1.x86_64.rpm
+* http://get.influxdb.org/telegraf/telegraf_0.12.1-1_amd64.deb
+* http://get.influxdb.org/telegraf/telegraf-0.12.1-1.x86_64.rpm
 
 Latest (arm):
-* https://dl.influxdata.com/telegraf/releases/telegraf_0.13.0-1_armhf.deb
-* https://dl.influxdata.com/telegraf/releases/telegraf-0.13.0-1.armhf.rpm
+* http://get.influxdb.org/telegraf/telegraf_0.12.1-1_armhf.deb
+* http://get.influxdb.org/telegraf/telegraf-0.12.1-1.armhf.rpm
 
 ##### Package Instructions:
 
@@ -46,28 +46,28 @@ to use this repo to install & update telegraf.
 ### Linux tarballs:
 
 Latest:
-* https://dl.influxdata.com/telegraf/releases/telegraf-0.13.0-1_linux_amd64.tar.gz
-* https://dl.influxdata.com/telegraf/releases/telegraf-0.13.0-1_linux_i386.tar.gz
-* https://dl.influxdata.com/telegraf/releases/telegraf-0.13.0-1_linux_armhf.tar.gz
+* http://get.influxdb.org/telegraf/telegraf-0.12.1-1_linux_amd64.tar.gz
+* http://get.influxdb.org/telegraf/telegraf-0.12.1-1_linux_i386.tar.gz
+* http://get.influxdb.org/telegraf/telegraf-0.12.1-1_linux_armhf.tar.gz
 
 ##### tarball Instructions:
 
 To install the full directory structure with config file, run:
 
 ```
-sudo tar -C / -zxvf ./telegraf-0.13.0-1_linux_amd64.tar.gz
+sudo tar -C / -zxvf ./telegraf-0.12.1-1_linux_amd64.tar.gz
 ```
 
 To extract only the binary, run:
 
 ```
-tar -zxvf telegraf-0.13.0-1_linux_amd64.tar.gz --strip-components=3 ./usr/bin/telegraf
+tar -zxvf telegraf-0.12.1-1_linux_amd64.tar.gz --strip-components=3 ./usr/bin/telegraf
 ```
 
 ### FreeBSD tarball:
 
 Latest:
-* https://dl.influxdata.com/telegraf/releases/telegraf-0.13.0-1_freebsd_amd64.tar.gz
+* http://get.influxdb.org/telegraf/telegraf-0.12.1-1_freebsd_amd64.tar.gz
 
 ##### tarball Instructions:
 
@@ -87,8 +87,8 @@ brew install telegraf
 ### Windows Binaries (EXPERIMENTAL)
 
 Latest:
-* https://dl.influxdata.com/telegraf/releases/telegraf-0.13.0-1_windows_amd64.zip
-* https://dl.influxdata.com/telegraf/releases/telegraf-0.13.0-1_windows_i386.zip
+* http://get.influxdb.org/telegraf/telegraf-0.12.1-1_windows_amd64.zip
+* http://get.influxdb.org/telegraf/telegraf-0.12.1-1_windows_i386.zip
 
 ### From Source:
 

cmd/telegraf/telegraf.go

@@ -46,13 +46,9 @@ var fOutputFiltersLegacy = flag.String("outputfilter", "",
 var fConfigDirectoryLegacy = flag.String("configdirectory", "",
 	"directory containing additional *.conf files")
 
-// Telegraf version, populated linker.
-//   ie, -ldflags "-X main.version=`git describe --always --tags`"
-var (
-	version string
-	commit  string
-	branch  string
-)
+// Telegraf version
+//	-ldflags "-X main.Version=`git describe --always --tags`"
+var Version string
 
 const usage = `Telegraf, The plugin-driven server agent for collecting and reporting metrics.
 
@@ -136,7 +132,7 @@ func main() {
 		if len(args) > 0 {
 			switch args[0] {
 			case "version":
-				v := fmt.Sprintf("Telegraf - version %s", version)
+				v := fmt.Sprintf("Telegraf - Version %s", Version)
 				fmt.Println(v)
 				return
 			case "config":
@@ -162,7 +158,7 @@ func main() {
 		}
 
 		if *fVersion {
-			v := fmt.Sprintf("Telegraf - version %s", version)
+			v := fmt.Sprintf("Telegraf - Version %s", Version)
 			fmt.Println(v)
 			return
 		}
@@ -255,7 +251,7 @@ func main() {
 			}
 		}()
 
-		log.Printf("Starting Telegraf (version %s)\n", version)
+		log.Printf("Starting Telegraf (version %s)\n", Version)
 		log.Printf("Loaded outputs: %s", strings.Join(c.OutputNames(), " "))
 		log.Printf("Loaded inputs: %s", strings.Join(c.InputNames(), " "))
 		log.Printf("Tags enabled: %s", c.ListTags())

plugins/inputs/all/all.go

@@ -19,6 +19,7 @@ import (
 	_ "github.com/influxdata/telegraf/plugins/inputs/haproxy"
 	_ "github.com/influxdata/telegraf/plugins/inputs/http_response"
 	_ "github.com/influxdata/telegraf/plugins/inputs/httpjson"
+	_ "github.com/influxdata/telegraf/plugins/inputs/igloo"
 	_ "github.com/influxdata/telegraf/plugins/inputs/influxdb"
 	_ "github.com/influxdata/telegraf/plugins/inputs/ipmi_sensor"
 	_ "github.com/influxdata/telegraf/plugins/inputs/jolokia"

plugins/inputs/igloo/README.md

@@ -0,0 +1,23 @@
+# igloo Input Plugin
+
+The igloo plugin "tails" a logfile and parses each log message.
+
+By default, the igloo plugin acts like the following unix tail command:
+
+```
+tail -F --lines=0 myfile.log
+```
+
+- `-F` means that it will follow the _name_ of the given file, so
+that it will be compatible with log-rotated files, and that it will retry on
+inaccessible files.
+- `--lines=0` means that it will start at the end of the file (unless
+the `from_beginning` option is set).
+
+see http://man7.org/linux/man-pages/man1/tail.1.html for more details.
+
+### Configuration:
+
+```toml
+```
+

plugins/inputs/igloo/igloo.go

@@ -0,0 +1,331 @@
+package igloo
+
+import (
+	"fmt"
+	"log"
+	"regexp"
+	"sort"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/hpcloud/tail"
+
+	"github.com/influxdata/telegraf"
+	"github.com/influxdata/telegraf/internal/globpath"
+	"github.com/influxdata/telegraf/plugins/inputs"
+)
+
+// format of timestamps
+const (
+	rfcFormat string = "%s-%s-%sT%s:%s:%s.%sZ"
+)
+
+var (
+	// regex for finding timestamps
+	tRe = regexp.MustCompile(`Timestamp=((\d{4})-(\d{2})-(\d{2}) (\d{2}):(\d{2}):(\d{2}),(\d+))`)
+)
+
+type Tail struct {
+	Files         []string
+	FromBeginning bool
+	TagKeys       []string
+	Counters      []string
+	NumFields     []string
+	StrFields     []string
+
+	numfieldsRe map[string]*regexp.Regexp
+	strfieldsRe map[string]*regexp.Regexp
+	countersRe  map[string]*regexp.Regexp
+	tagsRe      map[string]*regexp.Regexp
+
+	counters map[string]map[string]int64
+
+	tailers []*tail.Tail
+	wg      sync.WaitGroup
+	acc     telegraf.Accumulator
+
+	sync.Mutex
+}
+
+func NewTail() *Tail {
+	return &Tail{
+		FromBeginning: false,
+	}
+}
+
+const sampleConfig = `
+  ## logfiles to parse.
+  ##
+  ## These accept standard unix glob matching rules, but with the addition of
+  ## ** as a "super asterisk". ie:
+  ##   "/var/log/**.log"  -> recursively find all .log files in /var/log
+  ##   "/var/log/*/*.log" -> find all .log files with a parent dir in /var/log
+  ##   "/var/log/apache.log" -> just tail the apache log file
+  ##
+  ## See https://github.com/gobwas/glob for more examples
+  ##
+  files = ["$HOME/sample.log"]
+  ## Read file from beginning.
+  from_beginning = false
+
+  ## Each log message is searched for these tag keys in TagKey=Value format.
+  ## Any that are found will be tagged on the resulting influx measurements.
+  tag_keys = [
+    "HostLocal",
+    "ProductName",
+    "OperationName",
+  ]
+
+  ## counters are keys which are treated as counters.
+  ##   so if counters = ["Result"], then this means that the following ocurrence
+  ##   on a log line:
+  ##     Result=Success
+  ##   would be treated as a counter: Result_Success, and it will be incremented
+  ##   for every occurrence, until Telegraf is restarted.
+  counters   = ["Result"]
+  ## num_fields are log line occurrences that are translated into numerical
+  ## fields. ie:
+  ##   Duration=1
+  num_fields = ["Duration", "Attempt"]
+  ## str_fields are log line occurences that are translated into string fields,
+  ## ie:
+  ##   ActivityGUID=0bb03bf4-ae1d-4487-bb6f-311653b35760
+  str_fields = ["ActivityGUID"]
+`
+
+func (t *Tail) SampleConfig() string {
+	return sampleConfig
+}
+
+func (t *Tail) Description() string {
+	return "Stream an igloo file, like the tail -f command"
+}
+
+func (t *Tail) Gather(acc telegraf.Accumulator) error {
+	return nil
+}
+
+func (t *Tail) buildRegexes() error {
+	t.numfieldsRe = make(map[string]*regexp.Regexp)
+	t.strfieldsRe = make(map[string]*regexp.Regexp)
+	t.tagsRe = make(map[string]*regexp.Regexp)
+	t.countersRe = make(map[string]*regexp.Regexp)
+	t.counters = make(map[string]map[string]int64)
+
+	for _, field := range t.NumFields {
+		re, err := regexp.Compile(field + `=([0-9\.]+)`)
+		if err != nil {
+			return err
+		}
+		t.numfieldsRe[field] = re
+	}
+
+	for _, field := range t.StrFields {
+		re, err := regexp.Compile(field + `=([0-9a-zA-Z\.\-]+)`)
+		if err != nil {
+			return err
+		}
+		t.strfieldsRe[field] = re
+	}
+
+	for _, field := range t.TagKeys {
+		re, err := regexp.Compile(field + `=([0-9a-zA-Z\.\-]+)`)
+		if err != nil {
+			return err
+		}
+		t.tagsRe[field] = re
+	}
+
+	for _, field := range t.Counters {
+		re, err := regexp.Compile("(" + field + ")" + `=([0-9a-zA-Z\.\-]+)`)
+		if err != nil {
+			return err
+		}
+		t.countersRe[field] = re
+	}
+
+	return nil
+}
+
+func (t *Tail) Start(acc telegraf.Accumulator) error {
+	t.Lock()
+	defer t.Unlock()
+
+	t.acc = acc
+	if err := t.buildRegexes(); err != nil {
+		return err
+	}
+
+	var seek tail.SeekInfo
+	if !t.FromBeginning {
+		seek.Whence = 2
+		seek.Offset = 0
+	}
+
+	var errS string
+	// Create a "tailer" for each file
+	for _, filepath := range t.Files {
+		g, err := globpath.Compile(filepath)
+		if err != nil {
+			log.Printf("ERROR Glob %s failed to compile, %s", filepath, err)
+		}
+		for file, _ := range g.Match() {
+			tailer, err := tail.TailFile(file,
+				tail.Config{
+					ReOpen:   true,
+					Follow:   true,
+					Location: &seek,
+				})
+			if err != nil {
+				errS += err.Error() + " "
+				continue
+			}
+			// create a goroutine for each "tailer"
+			go t.receiver(tailer)
+			t.tailers = append(t.tailers, tailer)
+		}
+	}
+
+	if errS != "" {
+		return fmt.Errorf(errS)
+	}
+	return nil
+}
+
+// this is launched as a goroutine to continuously watch a tailed logfile
+// for changes, parse any incoming msgs, and add to the accumulator.
+func (t *Tail) receiver(tailer *tail.Tail) {
+	t.wg.Add(1)
+	defer t.wg.Done()
+
+	var err error
+	var line *tail.Line
+	for line = range tailer.Lines {
+		if line.Err != nil {
+			log.Printf("ERROR tailing file %s, Error: %s\n",
+				tailer.Filename, err)
+			continue
+		}
+		err = t.Parse(line.Text)
+		if err != nil {
+			log.Printf("ERROR: %s", err)
+		}
+	}
+}
+
+func (t *Tail) Parse(line string) error {
+	// find the timestamp:
+	match := tRe.FindAllStringSubmatch(line, -1)
+	if len(match) < 1 {
+		return nil
+	}
+	if len(match[0]) < 9 {
+		return nil
+	}
+	// make an rfc3339 timestamp and parse it:
+	ts, err := time.Parse(time.RFC3339Nano,
+		fmt.Sprintf(rfcFormat, match[0][2], match[0][3], match[0][4], match[0][5], match[0][6], match[0][7], match[0][8]))
+	if err != nil {
+		return nil
+	}
+
+	fields := make(map[string]interface{})
+	tags := make(map[string]string)
+
+	// parse numerical fields:
+	for name, re := range t.numfieldsRe {
+		match := re.FindAllStringSubmatch(line, -1)
+		if len(match) < 1 {
+			continue
+		}
+		if len(match[0]) < 2 {
+			continue
+		}
+		num, err := strconv.ParseFloat(match[0][1], 64)
+		if err == nil {
+			fields[name] = num
+		}
+	}
+
+	// parse string fields:
+	for name, re := range t.strfieldsRe {
+		match := re.FindAllStringSubmatch(line, -1)
+		if len(match) < 1 {
+			continue
+		}
+		if len(match[0]) < 2 {
+			continue
+		}
+		fields[name] = match[0][1]
+	}
+
+	// parse tags:
+	for name, re := range t.tagsRe {
+		match := re.FindAllStringSubmatch(line, -1)
+		if len(match) < 1 {
+			continue
+		}
+		if len(match[0]) < 2 {
+			continue
+		}
+		tags[name] = match[0][1]
+	}
+
+	if len(t.countersRe) > 0 {
+		// Make a unique key for the measurement name/tags
+		var tg []string
+		for k, v := range tags {
+			tg = append(tg, fmt.Sprintf("%s=%s", k, v))
+		}
+		sort.Strings(tg)
+		hash := fmt.Sprintf("%s%s", strings.Join(tg, ""), "igloo")
+
+		// check if this hash already has a counter map
+		_, ok := t.counters[hash]
+		if !ok {
+			// doesnt have counter map, so make one
+			t.counters[hash] = make(map[string]int64)
+		}
+
+		// search for counter matches:
+		for _, re := range t.countersRe {
+			match := re.FindAllStringSubmatch(line, -1)
+			if len(match) < 1 {
+				continue
+			}
+			if len(match[0]) < 3 {
+				continue
+			}
+			counterName := match[0][1] + "_" + match[0][2]
+			// increment this counter
+			t.counters[hash][counterName] += 1
+			// add this counter to the output fields
+			fields[counterName] = t.counters[hash][counterName]
+		}
+	}
+
+	t.acc.AddFields("igloo", fields, tags, ts)
+	return nil
+}
+
+func (t *Tail) Stop() {
+	t.Lock()
+	defer t.Unlock()
+
+	for _, t := range t.tailers {
+		err := t.Stop()
+		if err != nil {
+			log.Printf("ERROR stopping tail on file %s\n", t.Filename)
+		}
+		t.Cleanup()
+	}
+	t.wg.Wait()
+}
+
+func init() {
+	inputs.Add("igloo", func() telegraf.Input {
+		return NewTail()
+	})
+}

scripts/build.py

@@ -568,7 +568,7 @@ def package(build_output, version, nightly=False, rc=None, iteration=1, static=F
                     # For windows and static builds, just copy
                     # binaries to root of package (no other scripts or
                     # directories)
-                    package_scripts(build_root, windows=True)
+                    package_scripts(build_root, config_only=True)
                 else:
                     create_package_fs(build_root)
                     package_scripts(build_root)