From 6caa896cb103a4066e46ae47dc8e8e476bebb6d5 Mon Sep 17 00:00:00 2001 From: Daniel Nelson Date: Wed, 23 Aug 2017 15:17:26 -0700 Subject: [PATCH] Escape backslash within string fields (#3161) --- metric/escape.go | 10 ++++-- metric/metric.go | 8 +---- metric/metric_test.go | 7 +--- metric/reader_test.go | 78 +++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 88 insertions(+), 15 deletions(-) diff --git a/metric/escape.go b/metric/escape.go index a717f514d..cb040b7f2 100644 --- a/metric/escape.go +++ b/metric/escape.go @@ -20,8 +20,14 @@ var ( // stringFieldEscaper is for escaping string field values only. // see https://docs.influxdata.com/influxdb/v1.0/write_protocols/line_protocol_tutorial/#special-characters-and-keywords - stringFieldEscaper = strings.NewReplacer(`"`, `\"`) - stringFieldUnEscaper = strings.NewReplacer(`\"`, `"`) + stringFieldEscaper = strings.NewReplacer( + `"`, `\"`, + `\`, `\\`, + ) + stringFieldUnEscaper = strings.NewReplacer( + `\"`, `"`, + `\\`, `\`, + ) ) func escape(s string, t string) string { diff --git a/metric/metric.go b/metric/metric.go index dd2d6cbc0..63e609290 100644 --- a/metric/metric.go +++ b/metric/metric.go @@ -77,16 +77,10 @@ func New( // pre-allocate capacity of the fields slice fieldlen := 0 - for k, v := range fields { + for k, _ := range fields { if strings.HasSuffix(k, `\`) { return nil, fmt.Errorf("Metric cannot have field key ending with a backslash") } - switch val := v.(type) { - case string: - if strings.HasSuffix(val, `\`) { - return nil, fmt.Errorf("Metric cannot have field value ending with a backslash") - } - } // 10 bytes is completely arbitrary, but will at least prevent some // amount of allocations. There's a small possibility this will create diff --git a/metric/metric_test.go b/metric/metric_test.go index e875a213f..b1a23e6df 100644 --- a/metric/metric_test.go +++ b/metric/metric_test.go @@ -257,6 +257,7 @@ func TestNewMetric_Fields(t *testing.T) { "string": "test", "quote_string": `x"y`, "backslash_quote_string": `x\"y`, + "backslash": `x\y`, } m, err := New("cpu", tags, fields, now) assert.NoError(t, err) @@ -708,12 +709,6 @@ func TestNewMetric_TrailingSlash(t *testing.T) { `value\`: "x", }, }, - { - name: "cpu", - fields: map[string]interface{}{ - "value": `x\`, - }, - }, { name: "cpu", tags: map[string]string{ diff --git a/metric/reader_test.go b/metric/reader_test.go index 1537fc960..645b618eb 100644 --- a/metric/reader_test.go +++ b/metric/reader_test.go @@ -4,6 +4,7 @@ import ( "io" "io/ioutil" "regexp" + "strings" "testing" "time" @@ -620,6 +621,83 @@ func TestMetricReader_SplitMetricChangingBuffer2(t *testing.T) { } } +func TestReader_Read(t *testing.T) { + epoch := time.Unix(0, 0) + + type args struct { + name string + tags map[string]string + fields map[string]interface{} + t time.Time + mType []telegraf.ValueType + } + tests := []struct { + name string + args args + expected []byte + }{ + { + name: "escape backslashes in string field", + args: args{ + name: "cpu", + tags: map[string]string{}, + fields: map[string]interface{}{"value": `test\`}, + t: epoch, + }, + expected: []byte(`cpu value="test\\" 0`), + }, + { + name: "escape quote in string field", + args: args{ + name: "cpu", + tags: map[string]string{}, + fields: map[string]interface{}{"value": `test"`}, + t: epoch, + }, + expected: []byte(`cpu value="test\"" 0`), + }, + { + name: "escape quote and backslash in string field", + args: args{ + name: "cpu", + tags: map[string]string{}, + fields: map[string]interface{}{"value": `test\"`}, + t: epoch, + }, + expected: []byte(`cpu value="test\\\"" 0`), + }, + { + name: "escape multiple backslash in string field", + args: args{ + name: "cpu", + tags: map[string]string{}, + fields: map[string]interface{}{"value": `test\\`}, + t: epoch, + }, + expected: []byte(`cpu value="test\\\\" 0`), + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + buf := make([]byte, 512) + m, err := New(tt.args.name, tt.args.tags, tt.args.fields, tt.args.t, tt.args.mType...) + require.NoError(t, err) + + r := NewReader([]telegraf.Metric{m}) + num, err := r.Read(buf) + if err != io.EOF { + require.NoError(t, err) + } + line := string(buf[:num]) + // This is done so that we can use raw strings in the test spec + noeol := strings.TrimRight(line, "\n") + require.Equal(t, string(tt.expected), noeol) + require.Equal(t, len(tt.expected)+1, num) + }) + } +} + func TestMetricRoundtrip(t *testing.T) { const lp = `nstat,bu=linux,cls=server,dc=cer,env=production,host=hostname,name=netstat,sr=database IpExtInBcastOctets=12570626154i,IpExtInBcastPkts=95541226i,IpExtInCEPkts=0i,IpExtInCsumErrors=0i,IpExtInECT0Pkts=55674i,IpExtInECT1Pkts=0i,IpExtInMcastOctets=5928296i,IpExtInMcastPkts=174365i,IpExtInNoECTPkts=17965863529i,IpExtInNoRoutes=20i,IpExtInOctets=3334866321815i,IpExtInTruncatedPkts=0i,IpExtOutBcastOctets=0i,IpExtOutBcastPkts=0i,IpExtOutMcastOctets=0i,IpExtOutMcastPkts=0i,IpExtOutOctets=31397892391399i,TcpExtArpFilter=0i,TcpExtBusyPollRxPackets=0i,TcpExtDelayedACKLocked=14094i,TcpExtDelayedACKLost=302083i,TcpExtDelayedACKs=55486507i,TcpExtEmbryonicRsts=11879i,TcpExtIPReversePathFilter=0i,TcpExtListenDrops=1736i,TcpExtListenOverflows=0i,TcpExtLockDroppedIcmps=0i,TcpExtOfoPruned=0i,TcpExtOutOfWindowIcmps=8i,TcpExtPAWSActive=0i,TcpExtPAWSEstab=974i,TcpExtPAWSPassive=0i,TcpExtPruneCalled=0i,TcpExtRcvPruned=0i,TcpExtSyncookiesFailed=12593i,TcpExtSyncookiesRecv=0i,TcpExtSyncookiesSent=0i,TcpExtTCPACKSkippedChallenge=0i,TcpExtTCPACKSkippedFinWait2=0i,TcpExtTCPACKSkippedPAWS=806i,TcpExtTCPACKSkippedSeq=519i,TcpExtTCPACKSkippedSynRecv=0i,TcpExtTCPACKSkippedTimeWait=0i,TcpExtTCPAbortFailed=0i,TcpExtTCPAbortOnClose=22i,TcpExtTCPAbortOnData=36593i,TcpExtTCPAbortOnLinger=0i,TcpExtTCPAbortOnMemory=0i,TcpExtTCPAbortOnTimeout=674i,TcpExtTCPAutoCorking=494253233i,TcpExtTCPBacklogDrop=0i,TcpExtTCPChallengeACK=281i,TcpExtTCPDSACKIgnoredNoUndo=93354i,TcpExtTCPDSACKIgnoredOld=336i,TcpExtTCPDSACKOfoRecv=0i,TcpExtTCPDSACKOfoSent=7i,TcpExtTCPDSACKOldSent=302073i,TcpExtTCPDSACKRecv=215884i,TcpExtTCPDSACKUndo=7633i,TcpExtTCPDeferAcceptDrop=0i,TcpExtTCPDirectCopyFromBacklog=0i,TcpExtTCPDirectCopyFromPrequeue=0i,TcpExtTCPFACKReorder=1320i,TcpExtTCPFastOpenActive=0i,TcpExtTCPFastOpenActiveFail=0i,TcpExtTCPFastOpenCookieReqd=0i,TcpExtTCPFastOpenListenOverflow=0i,TcpExtTCPFastOpenPassive=0i,TcpExtTCPFastOpenPassiveFail=0i,TcpExtTCPFastRetrans=350681i,TcpExtTCPForwardRetrans=142168i,TcpExtTCPFromZeroWindowAdv=4317i,TcpExtTCPFullUndo=29502i,TcpExtTCPHPAcks=10267073000i,TcpExtTCPHPHits=5629837098i,TcpExtTCPHPHitsToUser=0i,TcpExtTCPHystartDelayCwnd=285127i,TcpExtTCPHystartDelayDetect=12318i,TcpExtTCPHystartTrainCwnd=69160570i,TcpExtTCPHystartTrainDetect=3315799i,TcpExtTCPLossFailures=109i,TcpExtTCPLossProbeRecovery=110819i,TcpExtTCPLossProbes=233995i,TcpExtTCPLossUndo=5276i,TcpExtTCPLostRetransmit=397i,TcpExtTCPMD5NotFound=0i,TcpExtTCPMD5Unexpected=0i,TcpExtTCPMemoryPressures=0i,TcpExtTCPMinTTLDrop=0i,TcpExtTCPOFODrop=0i,TcpExtTCPOFOMerge=7i,TcpExtTCPOFOQueue=15196i,TcpExtTCPOrigDataSent=29055119435i,TcpExtTCPPartialUndo=21320i,TcpExtTCPPrequeueDropped=0i,TcpExtTCPPrequeued=0i,TcpExtTCPPureAcks=1236441827i,TcpExtTCPRcvCoalesce=225590473i,TcpExtTCPRcvCollapsed=0i,TcpExtTCPRenoFailures=0i,TcpExtTCPRenoRecovery=0i,TcpExtTCPRenoRecoveryFail=0i,TcpExtTCPRenoReorder=0i,TcpExtTCPReqQFullDoCookies=0i,TcpExtTCPReqQFullDrop=0i,TcpExtTCPRetransFail=41i,TcpExtTCPSACKDiscard=0i,TcpExtTCPSACKReneging=0i,TcpExtTCPSACKReorder=4307i,TcpExtTCPSYNChallenge=244i,TcpExtTCPSackFailures=1698i,TcpExtTCPSackMerged=184668i,TcpExtTCPSackRecovery=97369i,TcpExtTCPSackRecoveryFail=381i,TcpExtTCPSackShiftFallback=2697079i,TcpExtTCPSackShifted=760299i,TcpExtTCPSchedulerFailed=0i,TcpExtTCPSlowStartRetrans=9276i,TcpExtTCPSpuriousRTOs=959i,TcpExtTCPSpuriousRtxHostQueues=2973i,TcpExtTCPSynRetrans=200970i,TcpExtTCPTSReorder=15221i,TcpExtTCPTimeWaitOverflow=0i,TcpExtTCPTimeouts=70127i,TcpExtTCPToZeroWindowAdv=4317i,TcpExtTCPWantZeroWindowAdv=2133i,TcpExtTW=24809813i,TcpExtTWKilled=0i,TcpExtTWRecycled=0i 1496460785000000000 nstat,bu=linux,cls=server,dc=cer,env=production,host=hostname,name=snmp,sr=database IcmpInAddrMaskReps=0i,IcmpInAddrMasks=90i,IcmpInCsumErrors=0i,IcmpInDestUnreachs=284401i,IcmpInEchoReps=9i,IcmpInEchos=1761912i,IcmpInErrors=407i,IcmpInMsgs=2047767i,IcmpInParmProbs=0i,IcmpInRedirects=0i,IcmpInSrcQuenchs=0i,IcmpInTimeExcds=46i,IcmpInTimestampReps=0i,IcmpInTimestamps=1309i,IcmpMsgInType0=9i,IcmpMsgInType11=46i,IcmpMsgInType13=1309i,IcmpMsgInType17=90i,IcmpMsgInType3=284401i,IcmpMsgInType8=1761912i,IcmpMsgOutType0=1761912i,IcmpMsgOutType14=1248i,IcmpMsgOutType3=108709i,IcmpMsgOutType8=9i,IcmpOutAddrMaskReps=0i,IcmpOutAddrMasks=0i,IcmpOutDestUnreachs=108709i,IcmpOutEchoReps=1761912i,IcmpOutEchos=9i,IcmpOutErrors=0i,IcmpOutMsgs=1871878i,IcmpOutParmProbs=0i,IcmpOutRedirects=0i,IcmpOutSrcQuenchs=0i,IcmpOutTimeExcds=0i,IcmpOutTimestampReps=1248i,IcmpOutTimestamps=0i,IpDefaultTTL=64i,IpForwDatagrams=0i,IpForwarding=2i,IpFragCreates=0i,IpFragFails=0i,IpFragOKs=0i,IpInAddrErrors=0i,IpInDelivers=17658795773i,IpInDiscards=0i,IpInHdrErrors=0i,IpInReceives=17659269339i,IpInUnknownProtos=0i,IpOutDiscards=236976i,IpOutNoRoutes=1009i,IpOutRequests=23466783734i,IpReasmFails=0i,IpReasmOKs=0i,IpReasmReqds=0i,IpReasmTimeout=0i,TcpActiveOpens=23308977i,TcpAttemptFails=3757543i,TcpCurrEstab=280i,TcpEstabResets=184792i,TcpInCsumErrors=0i,TcpInErrs=232i,TcpInSegs=17536573089i,TcpMaxConn=-1i,TcpOutRsts=4051451i,TcpOutSegs=29836254873i,TcpPassiveOpens=176546974i,TcpRetransSegs=878085i,TcpRtoAlgorithm=1i,TcpRtoMax=120000i,TcpRtoMin=200i,UdpInCsumErrors=0i,UdpInDatagrams=24441661i,UdpInErrors=0i,UdpLiteInCsumErrors=0i,UdpLiteInDatagrams=0i,UdpLiteInErrors=0i,UdpLiteNoPorts=0i,UdpLiteOutDatagrams=0i,UdpLiteRcvbufErrors=0i,UdpLiteSndbufErrors=0i,UdpNoPorts=17660i,UdpOutDatagrams=51807896i,UdpRcvbufErrors=0i,UdpSndbufErrors=236922i 1496460785000000000