telegraf/metric/reader_test.go

package metric

import (
	"io"
	"io/ioutil"
	"regexp"
	"testing"
	"time"

	"github.com/influxdata/telegraf"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
)

func BenchmarkMetricReader(b *testing.B) {
	metrics := make([]telegraf.Metric, 10)
	for i := 0; i < 10; i++ {
		metrics[i], _ = New("foo", map[string]string{},
			map[string]interface{}{"value": int64(1)}, time.Now())
	}
	for n := 0; n < b.N; n++ {
		r := NewReader(metrics)
		io.Copy(ioutil.Discard, r)
	}
}

func TestMetricReader(t *testing.T) {
	ts := time.Unix(1481032190, 0)
	metrics := make([]telegraf.Metric, 10)
	for i := 0; i < 10; i++ {
		metrics[i], _ = New("foo", map[string]string{},
			map[string]interface{}{"value": int64(1)}, ts)
	}

	r := NewReader(metrics)

	buf := make([]byte, 35)
	for i := 0; i < 10; i++ {
		n, err := r.Read(buf)
		if err != nil {
			assert.True(t, err == io.EOF, err.Error())
		}
		assert.Equal(t, 33, n)
		assert.Equal(t, "foo value=1i 1481032190000000000\n", string(buf[0:n]))
	}

	// reader should now be done, and always return 0, io.EOF
	for i := 0; i < 10; i++ {
		n, err := r.Read(buf)
		assert.True(t, err == io.EOF, err.Error())
		assert.Equal(t, 0, n)
	}
}

func TestMetricReader_OverflowMetric(t *testing.T) {
	ts := time.Unix(1481032190, 0)
	m, _ := New("foo", map[string]string{},
		map[string]interface{}{"value": int64(10)}, ts)
	metrics := []telegraf.Metric{m}

	r := NewReader(metrics)
	buf := make([]byte, 5)

	tests := []struct {
		exp string
		err error
		n   int
	}{
		{
			"foo v",
			nil,
			5,
		},
		{
			"alue=",
			nil,
			5,
		},
		{
			"10i 1",
			nil,
			5,
		},
		{
			"48103",
			nil,
			5,
		},
		{
			"21900",
			nil,
			5,
		},
		{
			"00000",
			nil,
			5,
		},
		{
			"000\n",
			io.EOF,
			4,
		},
		{
			"",
			io.EOF,
			0,
		},
	}

	for _, test := range tests {
		n, err := r.Read(buf)
		assert.Equal(t, test.n, n)
		assert.Equal(t, test.exp, string(buf[0:n]))
		assert.Equal(t, test.err, err)
	}
}

// Regression test for when a metric is the same size as the buffer.
//
// Previously EOF would not be set until the next call to Read.
func TestMetricReader_MetricSizeEqualsBufferSize(t *testing.T) {
	ts := time.Unix(1481032190, 0)
	m1, _ := New("foo", map[string]string{},
		map[string]interface{}{"a": int64(1)}, ts)
	metrics := []telegraf.Metric{m1}

	r := NewReader(metrics)
	buf := make([]byte, m1.Len())

	for {
		n, err := r.Read(buf)
		// Should never read 0 bytes unless at EOF, unless input buffer is 0 length
		if n == 0 {
			require.Equal(t, io.EOF, err)
			break
		}
		// Lines should be terminated with a LF
		if err == io.EOF {
			require.Equal(t, uint8('\n'), buf[n-1])
			break
		}
		require.NoError(t, err)
	}
}

// Regression test for when a metric requires to be split and one of the
// split metrics is exactly the size of the buffer.
//
// Previously an empty string would be returned on the next Read without error,
// and then next Read call would panic.
func TestMetricReader_SplitWithExactLengthSplit(t *testing.T) {
	ts := time.Unix(1481032190, 0)
	m1, _ := New("foo", map[string]string{},
		map[string]interface{}{"a": int64(1), "bb": int64(2)}, ts)
	metrics := []telegraf.Metric{m1}

	r := NewReader(metrics)
	buf := make([]byte, 30)

	//  foo a=1i,bb=2i 1481032190000000000\n // len 35
	//
	// Requires this specific split order:
	//  foo a=1i 1481032190000000000\n  // len 29
	//  foo bb=2i 1481032190000000000\n // len 30

	for {
		n, err := r.Read(buf)
		// Should never read 0 bytes unless at EOF, unless input buffer is 0 length
		if n == 0 {
			require.Equal(t, io.EOF, err)
			break
		}
		// Lines should be terminated with a LF
		if err == io.EOF {
			require.Equal(t, uint8('\n'), buf[n-1])
			break
		}
		require.NoError(t, err)
	}
}

// Regresssion test for when a metric requires to be split and one of the
// split metrics is larger than the buffer.
//
// Previously the metric index would be set incorrectly causing a panic.
func TestMetricReader_SplitOverflowOversized(t *testing.T) {
	ts := time.Unix(1481032190, 0)
	m1, _ := New("foo", map[string]string{},
		map[string]interface{}{
			"a":   int64(1),
			"bbb": int64(2),
		}, ts)
	metrics := []telegraf.Metric{m1}

	r := NewReader(metrics)
	buf := make([]byte, 30)

	// foo a=1i,bbb=2i 1481032190000000000\n // len 36
	//
	// foo a=1i 1481032190000000000\n  // len 29
	// foo bbb=2i 1481032190000000000\n // len 31

	for {
		n, err := r.Read(buf)
		// Should never read 0 bytes unless at EOF, unless input buffer is 0 length
		if n == 0 {
			require.Equal(t, io.EOF, err)
			break
		}
		// Lines should be terminated with a LF
		if err == io.EOF {
			require.Equal(t, uint8('\n'), buf[n-1])
			break
		}
		require.NoError(t, err)
	}
}

// Regresssion test for when a split metric exactly fits in the buffer.
//
// Previously the metric would be overflow split when not required.
func TestMetricReader_SplitOverflowUneeded(t *testing.T) {
	ts := time.Unix(1481032190, 0)
	m1, _ := New("foo", map[string]string{},
		map[string]interface{}{"a": int64(1), "b": int64(2)}, ts)
	metrics := []telegraf.Metric{m1}

	r := NewReader(metrics)
	buf := make([]byte, 29)

	// foo a=1i,b=2i 1481032190000000000\n // len 34
	//
	// foo a=1i 1481032190000000000\n  // len 29
	// foo b=2i 1481032190000000000\n // len 29

	for {
		n, err := r.Read(buf)
		// Should never read 0 bytes unless at EOF, unless input buffer is 0 length
		if n == 0 {
			require.Equal(t, io.EOF, err)
			break
		}
		// Lines should be terminated with a LF
		if err == io.EOF {
			require.Equal(t, uint8('\n'), buf[n-1])
			break
		}
		require.NoError(t, err)
	}
}

func TestMetricReader_OverflowMultipleMetrics(t *testing.T) {
	ts := time.Unix(1481032190, 0)
	m, _ := New("foo", map[string]string{},
		map[string]interface{}{"value": int64(10)}, ts)
	metrics := []telegraf.Metric{m, m.Copy()}

	r := NewReader(metrics)
	buf := make([]byte, 10)

	tests := []struct {
		exp string
		err error
		n   int
	}{
		{
			"foo value=",
			nil,
			10,
		},
		{
			"10i 148103",
			nil,
			10,
		},
		{
			"2190000000",
			nil,
			10,
		},
		{
			"000\n",
			nil,
			4,
		},
		{
			"foo value=",
			nil,
			10,
		},
		{
			"10i 148103",
			nil,
			10,
		},
		{
			"2190000000",
			nil,
			10,
		},
		{
			"000\n",
			io.EOF,
			4,
		},
		{
			"",
			io.EOF,
			0,
		},
	}

	for _, test := range tests {
		n, err := r.Read(buf)
		assert.Equal(t, test.n, n)
		assert.Equal(t, test.exp, string(buf[0:n]))
		assert.Equal(t, test.err, err)
	}
}

// test splitting a metric
func TestMetricReader_SplitMetric(t *testing.T) {
	ts := time.Unix(1481032190, 0)
	m1, _ := New("foo", map[string]string{},
		map[string]interface{}{
			"value1": int64(10),
			"value2": int64(10),
			"value3": int64(10),
			"value4": int64(10),
			"value5": int64(10),
			"value6": int64(10),
		},
		ts,
	)
	metrics := []telegraf.Metric{m1}

	r := NewReader(metrics)
	buf := make([]byte, 60)

	tests := []struct {
		expRegex string
		err      error
		n        int
	}{
		{
			`foo value\d=10i,value\d=10i,value\d=10i 1481032190000000000\n`,
			nil,
			57,
		},
		{
			`foo value\d=10i,value\d=10i,value\d=10i 1481032190000000000\n`,
			io.EOF,
			57,
		},
		{
			"",
			io.EOF,
			0,
		},
	}

	for _, test := range tests {
		n, err := r.Read(buf)
		assert.Equal(t, test.n, n)
		re := regexp.MustCompile(test.expRegex)
		assert.True(t, re.MatchString(string(buf[0:n])), string(buf[0:n]))
		assert.Equal(t, test.err, err)
	}
}

// test an array with one split metric and one unsplit
func TestMetricReader_SplitMetric2(t *testing.T) {
	ts := time.Unix(1481032190, 0)
	m1, _ := New("foo", map[string]string{},
		map[string]interface{}{
			"value1": int64(10),
			"value2": int64(10),
			"value3": int64(10),
			"value4": int64(10),
			"value5": int64(10),
			"value6": int64(10),
		},
		ts,
	)
	m2, _ := New("foo", map[string]string{},
		map[string]interface{}{
			"value1": int64(10),
		},
		ts,
	)
	metrics := []telegraf.Metric{m1, m2}

	r := NewReader(metrics)
	buf := make([]byte, 60)

	tests := []struct {
		expRegex string
		err      error
		n        int
	}{
		{
			`foo value\d=10i,value\d=10i,value\d=10i 1481032190000000000\n`,
			nil,
			57,
		},
		{
			`foo value\d=10i,value\d=10i,value\d=10i 1481032190000000000\n`,
			nil,
			57,
		},
		{
			`foo value1=10i 1481032190000000000\n`,
			io.EOF,
			35,
		},
		{
			"",
			io.EOF,
			0,
		},
	}

	for _, test := range tests {
		n, err := r.Read(buf)
		assert.Equal(t, test.n, n)
		re := regexp.MustCompile(test.expRegex)
		assert.True(t, re.MatchString(string(buf[0:n])), string(buf[0:n]))
		assert.Equal(t, test.err, err)
	}
}

// test split that results in metrics that are still too long, which results in
// the reader falling back to regular overflow.
func TestMetricReader_SplitMetricTooLong(t *testing.T) {
	ts := time.Unix(1481032190, 0)
	m1, _ := New("foo", map[string]string{},
		map[string]interface{}{
			"value1": int64(10),
			"value2": int64(10),
		},
		ts,
	)
	metrics := []telegraf.Metric{m1}

	r := NewReader(metrics)
	buf := make([]byte, 30)

	tests := []struct {
		expRegex string
		err      error
		n        int
	}{
		{
			`foo value\d=10i,value\d=10i 1481`,
			nil,
			30,
		},
		{
			`032190000000000\n`,
			io.EOF,
			16,
		},
		{
			"",
			io.EOF,
			0,
		},
	}

	for _, test := range tests {
		n, err := r.Read(buf)
		assert.Equal(t, test.n, n)
		re := regexp.MustCompile(test.expRegex)
		assert.True(t, re.MatchString(string(buf[0:n])), string(buf[0:n]))
		assert.Equal(t, test.err, err)
	}
}

// test split with a changing buffer size in the middle of subsequent calls
// to Read
func TestMetricReader_SplitMetricChangingBuffer(t *testing.T) {
	ts := time.Unix(1481032190, 0)
	m1, _ := New("foo", map[string]string{},
		map[string]interface{}{
			"value1": int64(10),
			"value2": int64(10),
			"value3": int64(10),
		},
		ts,
	)
	m2, _ := New("foo", map[string]string{},
		map[string]interface{}{
			"value1": int64(10),
		},
		ts,
	)
	metrics := []telegraf.Metric{m1, m2}

	r := NewReader(metrics)

	tests := []struct {
		expRegex string
		err      error
		n        int
		buf      []byte
	}{
		{
			`foo value\d=10i 1481032190000000000\n`,
			nil,
			35,
			make([]byte, 36),
		},
		{
			`foo value\d=10i 148103219000000`,
			nil,
			30,
			make([]byte, 30),
		},
		{
			`0000\n`,
			nil,
			5,
			make([]byte, 30),
		},
		{
			`foo value\d=10i 1481032190000000000\n`,
			nil,
			35,
			make([]byte, 36),
		},
		{
			`foo value1=10i 1481032190000000000\n`,
			io.EOF,
			35,
			make([]byte, 36),
		},
		{
			"",
			io.EOF,
			0,
			make([]byte, 36),
		},
	}

	for _, test := range tests {
		n, err := r.Read(test.buf)
		assert.Equal(t, test.n, n, test.expRegex)
		re := regexp.MustCompile(test.expRegex)
		assert.True(t, re.MatchString(string(test.buf[0:n])), string(test.buf[0:n]))
		assert.Equal(t, test.err, err, test.expRegex)
	}
}

// test split with a changing buffer size in the middle of subsequent calls
// to Read
func TestMetricReader_SplitMetricChangingBuffer2(t *testing.T) {
	ts := time.Unix(1481032190, 0)
	m1, _ := New("foo", map[string]string{},
		map[string]interface{}{
			"value1": int64(10),
			"value2": int64(10),
		},
		ts,
	)
	m2, _ := New("foo", map[string]string{},
		map[string]interface{}{
			"value1": int64(10),
		},
		ts,
	)
	metrics := []telegraf.Metric{m1, m2}

	r := NewReader(metrics)

	tests := []struct {
		expRegex string
		err      error
		n        int
		buf      []byte
	}{
		{
			`foo value\d=10i 1481032190000000000\n`,
			nil,
			35,
			make([]byte, 36),
		},
		{
			`foo value\d=10i 148103219000000`,
			nil,
			30,
			make([]byte, 30),
		},
		{
			`0000\n`,
			nil,
			5,
			make([]byte, 30),
		},
		{
			`foo value1=10i 1481032190000000000\n`,
			io.EOF,
			35,
			make([]byte, 36),
		},
		{
			"",
			io.EOF,
			0,
			make([]byte, 36),
		},
	}

	for _, test := range tests {
		n, err := r.Read(test.buf)
		assert.Equal(t, test.n, n, test.expRegex)
		re := regexp.MustCompile(test.expRegex)
		assert.True(t, re.MatchString(string(test.buf[0:n])), string(test.buf[0:n]))
		assert.Equal(t, test.err, err, test.expRegex)
	}
}

func TestMetricRoundtrip(t *testing.T) {
	const lp = `nstat,bu=linux,cls=server,dc=cer,env=production,host=hostname,name=netstat,sr=database IpExtInBcastOctets=12570626154i,IpExtInBcastPkts=95541226i,IpExtInCEPkts=0i,IpExtInCsumErrors=0i,IpExtInECT0Pkts=55674i,IpExtInECT1Pkts=0i,IpExtInMcastOctets=5928296i,IpExtInMcastPkts=174365i,IpExtInNoECTPkts=17965863529i,IpExtInNoRoutes=20i,IpExtInOctets=3334866321815i,IpExtInTruncatedPkts=0i,IpExtOutBcastOctets=0i,IpExtOutBcastPkts=0i,IpExtOutMcastOctets=0i,IpExtOutMcastPkts=0i,IpExtOutOctets=31397892391399i,TcpExtArpFilter=0i,TcpExtBusyPollRxPackets=0i,TcpExtDelayedACKLocked=14094i,TcpExtDelayedACKLost=302083i,TcpExtDelayedACKs=55486507i,TcpExtEmbryonicRsts=11879i,TcpExtIPReversePathFilter=0i,TcpExtListenDrops=1736i,TcpExtListenOverflows=0i,TcpExtLockDroppedIcmps=0i,TcpExtOfoPruned=0i,TcpExtOutOfWindowIcmps=8i,TcpExtPAWSActive=0i,TcpExtPAWSEstab=974i,TcpExtPAWSPassive=0i,TcpExtPruneCalled=0i,TcpExtRcvPruned=0i,TcpExtSyncookiesFailed=12593i,TcpExtSyncookiesRecv=0i,TcpExtSyncookiesSent=0i,TcpExtTCPACKSkippedChallenge=0i,TcpExtTCPACKSkippedFinWait2=0i,TcpExtTCPACKSkippedPAWS=806i,TcpExtTCPACKSkippedSeq=519i,TcpExtTCPACKSkippedSynRecv=0i,TcpExtTCPACKSkippedTimeWait=0i,TcpExtTCPAbortFailed=0i,TcpExtTCPAbortOnClose=22i,TcpExtTCPAbortOnData=36593i,TcpExtTCPAbortOnLinger=0i,TcpExtTCPAbortOnMemory=0i,TcpExtTCPAbortOnTimeout=674i,TcpExtTCPAutoCorking=494253233i,TcpExtTCPBacklogDrop=0i,TcpExtTCPChallengeACK=281i,TcpExtTCPDSACKIgnoredNoUndo=93354i,TcpExtTCPDSACKIgnoredOld=336i,TcpExtTCPDSACKOfoRecv=0i,TcpExtTCPDSACKOfoSent=7i,TcpExtTCPDSACKOldSent=302073i,TcpExtTCPDSACKRecv=215884i,TcpExtTCPDSACKUndo=7633i,TcpExtTCPDeferAcceptDrop=0i,TcpExtTCPDirectCopyFromBacklog=0i,TcpExtTCPDirectCopyFromPrequeue=0i,TcpExtTCPFACKReorder=1320i,TcpExtTCPFastOpenActive=0i,TcpExtTCPFastOpenActiveFail=0i,TcpExtTCPFastOpenCookieReqd=0i,TcpExtTCPFastOpenListenOverflow=0i,TcpExtTCPFastOpenPassive=0i,TcpExtTCPFastOpenPassiveFail=0i,TcpExtTCPFastRetrans=350681i,TcpExtTCPForwardRetrans=142168i,TcpExtTCPFromZeroWindowAdv=4317i,TcpExtTCPFullUndo=29502i,TcpExtTCPHPAcks=10267073000i,TcpExtTCPHPHits=5629837098i,TcpExtTCPHPHitsToUser=0i,TcpExtTCPHystartDelayCwnd=285127i,TcpExtTCPHystartDelayDetect=12318i,TcpExtTCPHystartTrainCwnd=69160570i,TcpExtTCPHystartTrainDetect=3315799i,TcpExtTCPLossFailures=109i,TcpExtTCPLossProbeRecovery=110819i,TcpExtTCPLossProbes=233995i,TcpExtTCPLossUndo=5276i,TcpExtTCPLostRetransmit=397i,TcpExtTCPMD5NotFound=0i,TcpExtTCPMD5Unexpected=0i,TcpExtTCPMemoryPressures=0i,TcpExtTCPMinTTLDrop=0i,TcpExtTCPOFODrop=0i,TcpExtTCPOFOMerge=7i,TcpExtTCPOFOQueue=15196i,TcpExtTCPOrigDataSent=29055119435i,TcpExtTCPPartialUndo=21320i,TcpExtTCPPrequeueDropped=0i,TcpExtTCPPrequeued=0i,TcpExtTCPPureAcks=1236441827i,TcpExtTCPRcvCoalesce=225590473i,TcpExtTCPRcvCollapsed=0i,TcpExtTCPRenoFailures=0i,TcpExtTCPRenoRecovery=0i,TcpExtTCPRenoRecoveryFail=0i,TcpExtTCPRenoReorder=0i,TcpExtTCPReqQFullDoCookies=0i,TcpExtTCPReqQFullDrop=0i,TcpExtTCPRetransFail=41i,TcpExtTCPSACKDiscard=0i,TcpExtTCPSACKReneging=0i,TcpExtTCPSACKReorder=4307i,TcpExtTCPSYNChallenge=244i,TcpExtTCPSackFailures=1698i,TcpExtTCPSackMerged=184668i,TcpExtTCPSackRecovery=97369i,TcpExtTCPSackRecoveryFail=381i,TcpExtTCPSackShiftFallback=2697079i,TcpExtTCPSackShifted=760299i,TcpExtTCPSchedulerFailed=0i,TcpExtTCPSlowStartRetrans=9276i,TcpExtTCPSpuriousRTOs=959i,TcpExtTCPSpuriousRtxHostQueues=2973i,TcpExtTCPSynRetrans=200970i,TcpExtTCPTSReorder=15221i,TcpExtTCPTimeWaitOverflow=0i,TcpExtTCPTimeouts=70127i,TcpExtTCPToZeroWindowAdv=4317i,TcpExtTCPWantZeroWindowAdv=2133i,TcpExtTW=24809813i,TcpExtTWKilled=0i,TcpExtTWRecycled=0i 1496460785000000000
nstat,bu=linux,cls=server,dc=cer,env=production,host=hostname,name=snmp,sr=database IcmpInAddrMaskReps=0i,IcmpInAddrMasks=90i,IcmpInCsumErrors=0i,IcmpInDestUnreachs=284401i,IcmpInEchoReps=9i,IcmpInEchos=1761912i,IcmpInErrors=407i,IcmpInMsgs=2047767i,IcmpInParmProbs=0i,IcmpInRedirects=0i,IcmpInSrcQuenchs=0i,IcmpInTimeExcds=46i,IcmpInTimestampReps=0i,IcmpInTimestamps=1309i,IcmpMsgInType0=9i,IcmpMsgInType11=46i,IcmpMsgInType13=1309i,IcmpMsgInType17=90i,IcmpMsgInType3=284401i,IcmpMsgInType8=1761912i,IcmpMsgOutType0=1761912i,IcmpMsgOutType14=1248i,IcmpMsgOutType3=108709i,IcmpMsgOutType8=9i,IcmpOutAddrMaskReps=0i,IcmpOutAddrMasks=0i,IcmpOutDestUnreachs=108709i,IcmpOutEchoReps=1761912i,IcmpOutEchos=9i,IcmpOutErrors=0i,IcmpOutMsgs=1871878i,IcmpOutParmProbs=0i,IcmpOutRedirects=0i,IcmpOutSrcQuenchs=0i,IcmpOutTimeExcds=0i,IcmpOutTimestampReps=1248i,IcmpOutTimestamps=0i,IpDefaultTTL=64i,IpForwDatagrams=0i,IpForwarding=2i,IpFragCreates=0i,IpFragFails=0i,IpFragOKs=0i,IpInAddrErrors=0i,IpInDelivers=17658795773i,IpInDiscards=0i,IpInHdrErrors=0i,IpInReceives=17659269339i,IpInUnknownProtos=0i,IpOutDiscards=236976i,IpOutNoRoutes=1009i,IpOutRequests=23466783734i,IpReasmFails=0i,IpReasmOKs=0i,IpReasmReqds=0i,IpReasmTimeout=0i,TcpActiveOpens=23308977i,TcpAttemptFails=3757543i,TcpCurrEstab=280i,TcpEstabResets=184792i,TcpInCsumErrors=0i,TcpInErrs=232i,TcpInSegs=17536573089i,TcpMaxConn=-1i,TcpOutRsts=4051451i,TcpOutSegs=29836254873i,TcpPassiveOpens=176546974i,TcpRetransSegs=878085i,TcpRtoAlgorithm=1i,TcpRtoMax=120000i,TcpRtoMin=200i,UdpInCsumErrors=0i,UdpInDatagrams=24441661i,UdpInErrors=0i,UdpLiteInCsumErrors=0i,UdpLiteInDatagrams=0i,UdpLiteInErrors=0i,UdpLiteNoPorts=0i,UdpLiteOutDatagrams=0i,UdpLiteRcvbufErrors=0i,UdpLiteSndbufErrors=0i,UdpNoPorts=17660i,UdpOutDatagrams=51807896i,UdpRcvbufErrors=0i,UdpSndbufErrors=236922i 1496460785000000000
`
	metrics, err := Parse([]byte(lp))
	require.NoError(t, err)
	r := NewReader(metrics)
	buf := make([]byte, 128)
	_, err = r.Read(buf)
	require.NoError(t, err)
	metrics, err = Parse(buf)
	require.NoError(t, err)
}