dom/telegraf
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Add new line protocol parser and serializer, influxdb output (#3924)

Browse Source
This commit is contained in:
Daniel Nelson
2018-03-27 17:30:51 -07:00
committed by GitHub
parent 503881d4d7
commit 1c0f63a90d
70 changed files with 26827 additions and 6533 deletions
Download Patch File Download Diff File Expand all files Collapse all files

77
plugins/inputs/exec/exec_test.go
View File

@@ -144,83 +144,6 @@ func TestCommandError(t *testing.T) {
assert.Equal(t, acc.NFields(), 0, "No new points should have been added")
}
func TestLineProtocolParse(t *testing.T) {
parser, _ := parsers.NewInfluxParser()
e := &Exec{
runner: newRunnerMock([]byte(lineProtocol), nil),
Commands: []string{"line-protocol"},
parser: parser,
}
var acc testutil.Accumulator
require.NoError(t, acc.GatherError(e.Gather))
fields := map[string]interface{}{
"usage_idle": float64(99),
"usage_busy": float64(1),
}
tags := map[string]string{
"host": "foo",
"datacenter": "us-east",
}
acc.AssertContainsTaggedFields(t, "cpu", fields, tags)
}
func TestLineProtocolEmptyParse(t *testing.T) {
parser, _ := parsers.NewInfluxParser()
e := &Exec{
runner: newRunnerMock([]byte(lineProtocolEmpty), nil),
Commands: []string{"line-protocol"},
parser: parser,
}
var acc testutil.Accumulator
err := e.Gather(&acc)
require.NoError(t, err)
}
func TestLineProtocolShortParse(t *testing.T) {
parser, _ := parsers.NewInfluxParser()
e := &Exec{
runner: newRunnerMock([]byte(lineProtocolShort), nil),
Commands: []string{"line-protocol"},
parser: parser,
}
var acc testutil.Accumulator
err := acc.GatherError(e.Gather)
require.Error(t, err)
assert.Contains(t, err.Error(), "buffer too short", "A buffer too short error was expected")
}
func TestLineProtocolParseMultiple(t *testing.T) {
parser, _ := parsers.NewInfluxParser()
e := &Exec{
runner: newRunnerMock([]byte(lineProtocolMulti), nil),
Commands: []string{"line-protocol"},
parser: parser,
}
var acc testutil.Accumulator
err := acc.GatherError(e.Gather)
require.NoError(t, err)
fields := map[string]interface{}{
"usage_idle": float64(99),
"usage_busy": float64(1),
}
tags := map[string]string{
"host": "foo",
"datacenter": "us-east",
}
cpuTags := []string{"cpu0", "cpu1", "cpu2", "cpu3", "cpu4", "cpu5", "cpu6"}
for _, cpu := range cpuTags {
tags["cpu"] = cpu
acc.AssertContainsTaggedFields(t, "cpu", fields, tags)
}
}
func TestExecCommandWithGlob(t *testing.T) {
parser, _ := parsers.NewValueParser("metric", "string", nil)
e := NewExec()

33
plugins/inputs/http_listener/http_listener.go
View File

@@ -53,9 +53,10 @@ type HTTPListener struct {
listener net.Listener
parser influx.InfluxParser
acc telegraf.Accumulator
pool *pool
handler *influx.MetricHandler
parser *influx.Parser
acc telegraf.Accumulator
pool *pool
BytesRecv selfstat.Stat
RequestsServed selfstat.Stat
@@ -176,6 +177,9 @@ func (h *HTTPListener) Start(acc telegraf.Accumulator) error {
h.listener = listener
h.Port = listener.Addr().(*net.TCPAddr).Port
h.handler = influx.NewMetricHandler()
h.parser = influx.NewParser(h.handler)
h.wg.Add(1)
go func() {
defer h.wg.Done()
@@ -336,7 +340,11 @@ func (h *HTTPListener) serveWrite(res http.ResponseWriter, req *http.Request) {
}
func (h *HTTPListener) parse(b []byte, t time.Time, precision string) error {
metrics, err := h.parser.ParseWithDefaultTimePrecision(b, t, precision)
h.handler.SetPrecision(getPrecisionMultiplier(precision))
metrics, err := h.parser.Parse(b)
if err != nil {
return err
}
for _, m := range metrics {
h.acc.AddFields(m.Name(), m.Fields(), m.Tags(), m.Time())
@@ -408,6 +416,23 @@ func (h *HTTPListener) AuthenticateIfSet(handler http.HandlerFunc, res http.Resp
}
}
func getPrecisionMultiplier(precision string) time.Duration {
d := time.Nanosecond
switch precision {
case "u":
d = time.Microsecond
case "ms":
d = time.Millisecond
case "s":
d = time.Second
case "m":
d = time.Minute
case "h":
d = time.Hour
}
return d
}
func init() {
inputs.Add("http_listener", func() telegraf.Input {
return &HTTPListener{

4
plugins/inputs/logparser/grok/grok.go
View File

@@ -326,6 +326,10 @@ func (p *Parser) ParseLine(line string) (telegraf.Metric, error) {
}
}
if len(fields) == 0 {
return nil, fmt.Errorf("logparser_grok: must have one or more fields")
}
return metric.New(p.Measurement, tags, fields, p.tsModder.tsMod(timestamp))
}

20
plugins/inputs/logparser/grok/grok_test.go
View File

@@ -799,7 +799,7 @@ func TestTimezoneEmptyCompileFileAndParse(t *testing.T) {
},
metricA.Fields())
assert.Equal(t, map[string]string{"response_code": "200"}, metricA.Tags())
assert.Equal(t, int64(1465040505000000000), metricA.UnixNano())
assert.Equal(t, int64(1465040505000000000), metricA.Time().UnixNano())
metricB, err := p.ParseLine(`[04/06/2016--12:41:45] 1.25 mystring dropme nomodifier`)
require.NotNil(t, metricB)
@@ -812,7 +812,7 @@ func TestTimezoneEmptyCompileFileAndParse(t *testing.T) {
},
metricB.Fields())
assert.Equal(t, map[string]string{}, metricB.Tags())
assert.Equal(t, int64(1465044105000000000), metricB.UnixNano())
assert.Equal(t, int64(1465044105000000000), metricB.Time().UnixNano())
}
func TestTimezoneMalformedCompileFileAndParse(t *testing.T) {
@@ -835,7 +835,7 @@ func TestTimezoneMalformedCompileFileAndParse(t *testing.T) {
},
metricA.Fields())
assert.Equal(t, map[string]string{"response_code": "200"}, metricA.Tags())
assert.Equal(t, int64(1465040505000000000), metricA.UnixNano())
assert.Equal(t, int64(1465040505000000000), metricA.Time().UnixNano())
metricB, err := p.ParseLine(`[04/06/2016--12:41:45] 1.25 mystring dropme nomodifier`)
require.NotNil(t, metricB)
@@ -848,7 +848,7 @@ func TestTimezoneMalformedCompileFileAndParse(t *testing.T) {
},
metricB.Fields())
assert.Equal(t, map[string]string{}, metricB.Tags())
assert.Equal(t, int64(1465044105000000000), metricB.UnixNano())
assert.Equal(t, int64(1465044105000000000), metricB.Time().UnixNano())
}
func TestTimezoneEuropeCompileFileAndParse(t *testing.T) {
@@ -871,7 +871,7 @@ func TestTimezoneEuropeCompileFileAndParse(t *testing.T) {
},
metricA.Fields())
assert.Equal(t, map[string]string{"response_code": "200"}, metricA.Tags())
assert.Equal(t, int64(1465040505000000000), metricA.UnixNano())
assert.Equal(t, int64(1465040505000000000), metricA.Time().UnixNano())
metricB, err := p.ParseLine(`[04/06/2016--12:41:45] 1.25 mystring dropme nomodifier`)
require.NotNil(t, metricB)
@@ -884,7 +884,7 @@ func TestTimezoneEuropeCompileFileAndParse(t *testing.T) {
},
metricB.Fields())
assert.Equal(t, map[string]string{}, metricB.Tags())
assert.Equal(t, int64(1465036905000000000), metricB.UnixNano())
assert.Equal(t, int64(1465036905000000000), metricB.Time().UnixNano())
}
func TestTimezoneAmericasCompileFileAndParse(t *testing.T) {
@@ -907,7 +907,7 @@ func TestTimezoneAmericasCompileFileAndParse(t *testing.T) {
},
metricA.Fields())
assert.Equal(t, map[string]string{"response_code": "200"}, metricA.Tags())
assert.Equal(t, int64(1465040505000000000), metricA.UnixNano())
assert.Equal(t, int64(1465040505000000000), metricA.Time().UnixNano())
metricB, err := p.ParseLine(`[04/06/2016--12:41:45] 1.25 mystring dropme nomodifier`)
require.NotNil(t, metricB)
@@ -920,7 +920,7 @@ func TestTimezoneAmericasCompileFileAndParse(t *testing.T) {
},
metricB.Fields())
assert.Equal(t, map[string]string{}, metricB.Tags())
assert.Equal(t, int64(1465058505000000000), metricB.UnixNano())
assert.Equal(t, int64(1465058505000000000), metricB.Time().UnixNano())
}
func TestTimezoneLocalCompileFileAndParse(t *testing.T) {
@@ -943,7 +943,7 @@ func TestTimezoneLocalCompileFileAndParse(t *testing.T) {
},
metricA.Fields())
assert.Equal(t, map[string]string{"response_code": "200"}, metricA.Tags())
assert.Equal(t, int64(1465040505000000000), metricA.UnixNano())
assert.Equal(t, int64(1465040505000000000), metricA.Time().UnixNano())
metricB, err := p.ParseLine(`[04/06/2016--12:41:45] 1.25 mystring dropme nomodifier`)
require.NotNil(t, metricB)
@@ -956,5 +956,5 @@ func TestTimezoneLocalCompileFileAndParse(t *testing.T) {
},
metricB.Fields())
assert.Equal(t, map[string]string{}, metricB.Tags())
assert.Equal(t, time.Date(2016, time.June, 4, 12, 41, 45, 0, time.Local).UnixNano(), metricB.UnixNano())
assert.Equal(t, time.Date(2016, time.June, 4, 12, 41, 45, 0, time.Local).UnixNano(), metricB.Time().UnixNano())
}

2
plugins/inputs/nats_consumer/nats_consumer_test.go
View File

@@ -59,7 +59,7 @@ func TestRunParserInvalidMsg(t *testing.T) {
in <- natsMsg(invalidMsg)
acc.WaitError(1)
assert.Contains(t, acc.Errors[0].Error(), "E! subject: telegraf, error: metric parsing error")
assert.Contains(t, acc.Errors[0].Error(), "E! subject: telegraf, error: metric parse error")
assert.EqualValues(t, 0, acc.NMetrics())
}

8
plugins/inputs/prometheus/parser_test.go
View File

@@ -111,9 +111,11 @@ func TestParseValidPrometheus(t *testing.T) {
"gauge": float64(1),
}, metrics[0].Fields())
assert.Equal(t, map[string]string{
"osVersion": "CentOS Linux 7 (Core)",
"dockerVersion": "1.8.2",
"kernelVersion": "3.10.0-229.20.1.el7.x86_64",
"osVersion": "CentOS Linux 7 (Core)",
"cadvisorRevision": "",
"cadvisorVersion": "",
"dockerVersion": "1.8.2",
"kernelVersion": "3.10.0-229.20.1.el7.x86_64",
}, metrics[0].Tags())
// Counter value

6
plugins/inputs/udp_listener/udp_listener_test.go
View File

@@ -47,11 +47,13 @@ func TestHighTrafficUDP(t *testing.T) {
ServiceAddress: ":8126",
AllowedPendingMessages: 100000,
}
listener.parser, _ = parsers.NewInfluxParser()
var err error
listener.parser, err = parsers.NewInfluxParser()
require.NoError(t, err)
acc := &testutil.Accumulator{}
// send multiple messages to socket
err := listener.Start(acc)
err = listener.Start(acc)
require.NoError(t, err)
conn, err := net.Dial("udp", "127.0.0.1:8126")