Resume from last known offset when reloading in tail input (#6074)

plugins/inputs/tail/tail.go

@@ -9,6 +9,7 @@ import (
 	"sync"
 
 	"github.com/influxdata/tail"
+
 	"github.com/influxdata/telegraf"
 	"github.com/influxdata/telegraf/internal/globpath"
 	"github.com/influxdata/telegraf/plugins/inputs"
@@ -19,6 +20,11 @@ const (
 	defaultWatchMethod = "inotify"
 )
 
+var (
+	offsets      = make(map[string]int64)
+	offsetsMutex = new(sync.Mutex)
+)
+
 type Tail struct {
 	Files         []string
 	FromBeginning bool
@@ -26,6 +32,7 @@ type Tail struct {
 	WatchMethod   string
 
 	tailers    map[string]*tail.Tail
+	offsets    map[string]int64
 	parserFunc parsers.ParserFunc
 	wg         sync.WaitGroup
 	acc        telegraf.Accumulator
@@ -34,8 +41,16 @@ type Tail struct {
 }
 
 func NewTail() *Tail {
+	offsetsMutex.Lock()
+	offsetsCopy := make(map[string]int64, len(offsets))
+	for k, v := range offsets {
+		offsetsCopy[k] = v
+	}
+	offsetsMutex.Unlock()
+
 	return &Tail{
 		FromBeginning: false,
+		offsets:       offsetsCopy,
 	}
 }
 
@@ -87,18 +102,19 @@ func (t *Tail) Start(acc telegraf.Accumulator) error {
 	t.acc = acc
 	t.tailers = make(map[string]*tail.Tail)
 
-	return t.tailNewFiles(t.FromBeginning)
+	err := t.tailNewFiles(t.FromBeginning)
+
+	// clear offsets
+	t.offsets = make(map[string]int64)
+	// assumption that once Start is called, all parallel plugins have already been initialized
+	offsetsMutex.Lock()
+	offsets = make(map[string]int64)
+	offsetsMutex.Unlock()
+
+	return err
 }
 
 func (t *Tail) tailNewFiles(fromBeginning bool) error {
-	var seek *tail.SeekInfo
-	if !t.Pipe && !fromBeginning {
-		seek = &tail.SeekInfo{
-			Whence: 2,
-			Offset: 0,
-		}
-	}
-
 	var poll bool
 	if t.WatchMethod == "poll" {
 		poll = true
@@ -108,7 +124,7 @@ func (t *Tail) tailNewFiles(fromBeginning bool) error {
 	for _, filepath := range t.Files {
 		g, err := globpath.Compile(filepath)
 		if err != nil {
-			t.acc.AddError(fmt.Errorf("E! Error Glob %s failed to compile, %s", filepath, err))
+			t.acc.AddError(fmt.Errorf("glob %s failed to compile, %s", filepath, err))
 		}
 		for _, file := range g.Match() {
 			if _, ok := t.tailers[file]; ok {
@@ -116,6 +132,22 @@ func (t *Tail) tailNewFiles(fromBeginning bool) error {
 				continue
 			}
 
+			var seek *tail.SeekInfo
+			if !t.Pipe && !fromBeginning {
+				if offset, ok := t.offsets[file]; ok {
+					log.Printf("D! [inputs.tail] using offset %d for file: %v", offset, file)
+					seek = &tail.SeekInfo{
+						Whence: 0,
+						Offset: offset,
+					}
+				} else {
+					seek = &tail.SeekInfo{
+						Whence: 2,
+						Offset: 0,
+					}
+				}
+			}
+
 			tailer, err := tail.TailFile(file,
 				tail.Config{
 					ReOpen:    true,
@@ -159,8 +191,7 @@ func (t *Tail) receiver(parser parsers.Parser, tailer *tail.Tail) {
 	var line *tail.Line
 	for line = range tailer.Lines {
 		if line.Err != nil {
-			t.acc.AddError(fmt.Errorf("E! Error tailing file %s, Error: %s\n",
-				tailer.Filename, err))
+			t.acc.AddError(fmt.Errorf("error tailing file %s, Error: %s", tailer.Filename, err))
 			continue
 		}
 		// Fix up files with Windows line endings.
@@ -188,7 +219,7 @@ func (t *Tail) receiver(parser parsers.Parser, tailer *tail.Tail) {
 				t.acc.AddFields(m.Name(), m.Fields(), tags, m.Time())
 			}
 		} else {
-			t.acc.AddError(fmt.Errorf("E! Malformed log line in %s: [%s], Error: %s\n",
+			t.acc.AddError(fmt.Errorf("malformed log line in %s: [%s], Error: %s",
 				tailer.Filename, line.Text, err))
 		}
 	}
@@ -196,8 +227,7 @@ func (t *Tail) receiver(parser parsers.Parser, tailer *tail.Tail) {
 	log.Printf("D! [inputs.tail] tail removed for file: %v", tailer.Filename)
 
 	if err := tailer.Err(); err != nil {
-		t.acc.AddError(fmt.Errorf("E! Error tailing file %s, Error: %s\n",
-			tailer.Filename, err))
+		t.acc.AddError(fmt.Errorf("error tailing file %s, Error: %s", tailer.Filename, err))
 	}
 }
 
@@ -206,13 +236,29 @@ func (t *Tail) Stop() {
 	defer t.Unlock()
 
 	for _, tailer := range t.tailers {
+		if !t.Pipe && !t.FromBeginning {
+			// store offset for resume
+			offset, err := tailer.Tell()
+			if err == nil {
+				log.Printf("D! [inputs.tail] recording offset %d for file: %v", offset, tailer.Filename)
+			} else {
+				t.acc.AddError(fmt.Errorf("error recording offset for file %s", tailer.Filename))
+			}
+		}
 		err := tailer.Stop()
 		if err != nil {
-			t.acc.AddError(fmt.Errorf("E! Error stopping tail on file %s\n", tailer.Filename))
+			t.acc.AddError(fmt.Errorf("error stopping tail on file %s", tailer.Filename))
 		}
 	}
 
 	t.wg.Wait()
+
+	// persist offsets
+	offsetsMutex.Lock()
+	for k, v := range t.offsets {
+		offsets[k] = v
+	}
+	offsetsMutex.Unlock()
 }
 
 func (t *Tail) SetParserFunc(fn parsers.ParserFunc) {

plugins/inputs/tail/tail_test.go

@@ -108,7 +108,7 @@ func TestTailBadLine(t *testing.T) {
 	require.NoError(t, err)
 
 	acc.WaitError(1)
-	assert.Contains(t, acc.Errors[0].Error(), "E! Malformed log line")
+	assert.Contains(t, acc.Errors[0].Error(), "malformed log line")
 }
 
 func TestTailDosLineendings(t *testing.T) {