dom
/
telegraf
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Refactor InfluxDB listener (#6974) 

Use streaming parser in InfluxDB listener
This commit is contained in:
reimda 2020-03-04 11:13:44 -07:00 committed by GitHub
parent ab8438dcc6
commit a0276385b1
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
21 changed files with 19898 additions and 18303 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
internal/http.go
View File

@ -6,27 +6,27 @@ import (
"net/http" "net/http"
) )
// ErrorFunc is a callback for writing an error response. type BasicAuthErrorFunc func(rw http.ResponseWriter)
type ErrorFunc func(rw http.ResponseWriter, code int)
// AuthHandler returns a http handler that requires HTTP basic auth // AuthHandler returns a http handler that requires HTTP basic auth
// credentials to match the given username and password. // credentials to match the given username and password.
func AuthHandler(username, password string, onError ErrorFunc) func(h http.Handler) http.Handler { func AuthHandler(username, password, realm string, onError BasicAuthErrorFunc) func(h http.Handler) http.Handler {
return func(h http.Handler) http.Handler { return func(h http.Handler) http.Handler {
return &basicAuthHandler{ return &basicAuthHandler{
username: username, username: username,
password: password, password: password,
realm: realm,
onError: onError, onError: onError,
next: h, next: h,
} }
} }
} }
type basicAuthHandler struct { type basicAuthHandler struct {
username string username string
password string password string
onError ErrorFunc realm string
onError BasicAuthErrorFunc
next http.Handler next http.Handler
} }
@ -37,7 +37,9 @@ func (h *basicAuthHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request)
subtle.ConstantTimeCompare([]byte(reqUsername), []byte(h.username)) != 1 || subtle.ConstantTimeCompare([]byte(reqUsername), []byte(h.username)) != 1 ||
subtle.ConstantTimeCompare([]byte(reqPassword), []byte(h.password)) != 1 { subtle.ConstantTimeCompare([]byte(reqPassword), []byte(h.password)) != 1 {
h.onError(rw, http.StatusUnauthorized) rw.Header().Set("WWW-Authenticate", "Basic realm=\""+h.realm+"\"")
h.onError(rw)
http.Error(rw, http.StatusText(http.StatusUnauthorized), http.StatusUnauthorized)
return return
} }
} }
@ -45,6 +47,9 @@ func (h *basicAuthHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request)
h.next.ServeHTTP(rw, req) h.next.ServeHTTP(rw, req)
} }
// ErrorFunc is a callback for writing an error response.
type ErrorFunc func(rw http.ResponseWriter, code int)
// IPRangeHandler returns a http handler that requires the remote address to be // IPRangeHandler returns a http handler that requires the remote address to be
// in the specified network. // in the specified network.
func IPRangeHandler(network []*net.IPNet, onError ErrorFunc) func(h http.Handler) http.Handler { func IPRangeHandler(network []*net.IPNet, onError ErrorFunc) func(h http.Handler) http.Handler {

55
metric/builder.go
View File

@ -1,55 +0,0 @@
package metric
import (
"time"
"github.com/influxdata/telegraf"
)
type TimeFunc func() time.Time
type Builder struct {
TimeFunc
TimePrecision time.Duration
*metric
}
func NewBuilder() *Builder {
b := &Builder{
TimeFunc: time.Now,
TimePrecision: 1 * time.Nanosecond,
}
b.Reset()
return b
}
func (b *Builder) SetName(name string) {
b.name = name
}
func (b *Builder) AddTag(key string, value string) {
b.metric.AddTag(key, value)
}
func (b *Builder) AddField(key string, value interface{}) {
b.metric.AddField(key, value)
}
func (b *Builder) SetTime(tm time.Time) {
b.tm = tm
}
func (b *Builder) Reset() {
b.metric = &metric{
tp: telegraf.Untyped,
}
}
func (b *Builder) Metric() (telegraf.Metric, error) {
if b.tm.IsZero() {
b.tm = b.TimeFunc().Truncate(b.TimePrecision)
}
return b.metric, nil
}

14
metric/metric.go
View File

@ -50,13 +50,15 @@ func New(
sort.Slice(m.tags, func(i, j int) bool { return m.tags[i].Key < m.tags[j].Key }) sort.Slice(m.tags, func(i, j int) bool { return m.tags[i].Key < m.tags[j].Key })
} }
m.fields = make([]*telegraf.Field, 0, len(fields)) if len(fields) > 0 {
for k, v := range fields { m.fields = make([]*telegraf.Field, 0, len(fields))
v := convertField(v) for k, v := range fields {
if v == nil { v := convertField(v)
continue if v == nil {
continue
}
m.AddField(k, v)
} }
m.AddField(k, v)
} }
return m, nil return m, nil

8
plugins/inputs/influxdb_listener/README.md
View File

@ -30,13 +30,13 @@ submits data to InfluxDB determines the destination database.
## maximum duration before timing out write of the response ## maximum duration before timing out write of the response
write_timeout = "10s" write_timeout = "10s"
## Maximum allowed http request body size in bytes. ## Maximum allowed HTTP request body size in bytes.
## 0 means to use the default of 536,870,912 bytes (500 mebibytes) ## 0 means to use the default of 32MiB.
max_body_size = 0 max_body_size = 0
## Maximum line size allowed to be sent in bytes. ## Maximum line size allowed to be sent in bytes.
## 0 means to use the default of 65536 bytes (64 kibibytes) ## deprecated in 1.14; parser now handles lines of unlimited length and option is ignored
max_line_size = 0 # max_line_size = 0
## Set one or more allowed client CA certificate file names to ## Set one or more allowed client CA certificate file names to
## enable mutually authenticated TLS connections ## enable mutually authenticated TLS connections

43
plugins/inputs/influxdb_listener/bufferpool.go
View File

@ -1,43 +0,0 @@
package http_listener
import (
"sync/atomic"
)
type pool struct {
buffers chan []byte
size int
created int64
}
// NewPool returns a new pool object.
// n is the number of buffers
// bufSize is the size (in bytes) of each buffer
func NewPool(n, bufSize int) *pool {
return &pool{
buffers: make(chan []byte, n),
size: bufSize,
}
}
func (p *pool) get() []byte {
select {
case b := <-p.buffers:
return b
default:
atomic.AddInt64(&p.created, 1)
return make([]byte, p.size)
}
}
func (p *pool) put(b []byte) {
select {
case p.buffers <- b:
default:
}
}
func (p *pool) ncreated() int64 {
return atomic.LoadInt64(&p.created)
}

464
plugins/inputs/influxdb_listener/http_listener.go
View File

@ -1,464 +0,0 @@
package http_listener
import (
"bytes"
"compress/gzip"
"crypto/subtle"
"crypto/tls"
"encoding/json"
"fmt"
"io"
"net"
"net/http"
"sync"
"time"
"github.com/influxdata/telegraf"
"github.com/influxdata/telegraf/internal"
tlsint "github.com/influxdata/telegraf/internal/tls"
"github.com/influxdata/telegraf/plugins/inputs"
"github.com/influxdata/telegraf/plugins/parsers/influx"
"github.com/influxdata/telegraf/selfstat"
)
const (
// DEFAULT_MAX_BODY_SIZE is the default maximum request body size, in bytes.
// if the request body is over this size, we will return an HTTP 413 error.
// 500 MB
DEFAULT_MAX_BODY_SIZE = 500 * 1024 * 1024
// MAX_LINE_SIZE is the maximum size, in bytes, that can be allocated for
// a single InfluxDB point.
// 64 KB
DEFAULT_MAX_LINE_SIZE = 64 * 1024
)
type TimeFunc func() time.Time
type HTTPListener struct {
ServiceAddress string `toml:"service_address"`
// Port gets pulled out of ServiceAddress
Port int
tlsint.ServerConfig
ReadTimeout internal.Duration `toml:"read_timeout"`
WriteTimeout internal.Duration `toml:"write_timeout"`
MaxBodySize internal.Size `toml:"max_body_size"`
MaxLineSize internal.Size `toml:"max_line_size"`
BasicUsername string `toml:"basic_username"`
BasicPassword string `toml:"basic_password"`
DatabaseTag string `toml:"database_tag"`
TimeFunc
mu sync.Mutex
wg sync.WaitGroup
listener net.Listener
handler *influx.MetricHandler
parser *influx.Parser
acc telegraf.Accumulator
pool *pool
BytesRecv selfstat.Stat
RequestsServed selfstat.Stat
WritesServed selfstat.Stat
QueriesServed selfstat.Stat
PingsServed selfstat.Stat
RequestsRecv selfstat.Stat
WritesRecv selfstat.Stat
QueriesRecv selfstat.Stat
PingsRecv selfstat.Stat
NotFoundsServed selfstat.Stat
BuffersCreated selfstat.Stat
AuthFailures selfstat.Stat
Log telegraf.Logger
longLines selfstat.Stat
}
const sampleConfig = `
## Address and port to host HTTP listener on
service_address = ":8186"
## maximum duration before timing out read of the request
read_timeout = "10s"
## maximum duration before timing out write of the response
write_timeout = "10s"
## Maximum allowed http request body size in bytes.
## 0 means to use the default of 524,288,000 bytes (500 mebibytes)
max_body_size = "500MiB"
## Maximum line size allowed to be sent in bytes.
## 0 means to use the default of 65536 bytes (64 kibibytes)
max_line_size = "64KiB"
## Optional tag name used to store the database.
## If the write has a database in the query string then it will be kept in this tag name.
## This tag can be used in downstream outputs.
## The default value of nothing means it will be off and the database will not be recorded.
# database_tag = ""
## Set one or more allowed client CA certificate file names to
## enable mutually authenticated TLS connections
tls_allowed_cacerts = ["/etc/telegraf/clientca.pem"]
## Add service certificate and key
tls_cert = "/etc/telegraf/cert.pem"
tls_key = "/etc/telegraf/key.pem"
## Optional username and password to accept for HTTP basic authentication.
## You probably want to make sure you have TLS configured above for this.
# basic_username = "foobar"
# basic_password = "barfoo"
`
func (h *HTTPListener) SampleConfig() string {
return sampleConfig
}
func (h *HTTPListener) Description() string {
return "Influx HTTP write listener"
}
func (h *HTTPListener) Gather(_ telegraf.Accumulator) error {
h.BuffersCreated.Set(h.pool.ncreated())
return nil
}
// Start starts the http listener service.
func (h *HTTPListener) Start(acc telegraf.Accumulator) error {
h.mu.Lock()
defer h.mu.Unlock()
tags := map[string]string{
"address": h.ServiceAddress,
}
h.BytesRecv = selfstat.Register("http_listener", "bytes_received", tags)
h.RequestsServed = selfstat.Register("http_listener", "requests_served", tags)
h.WritesServed = selfstat.Register("http_listener", "writes_served", tags)
h.QueriesServed = selfstat.Register("http_listener", "queries_served", tags)
h.PingsServed = selfstat.Register("http_listener", "pings_served", tags)
h.RequestsRecv = selfstat.Register("http_listener", "requests_received", tags)
h.WritesRecv = selfstat.Register("http_listener", "writes_received", tags)
h.QueriesRecv = selfstat.Register("http_listener", "queries_received", tags)
h.PingsRecv = selfstat.Register("http_listener", "pings_received", tags)
h.NotFoundsServed = selfstat.Register("http_listener", "not_founds_served", tags)
h.BuffersCreated = selfstat.Register("http_listener", "buffers_created", tags)
h.AuthFailures = selfstat.Register("http_listener", "auth_failures", tags)
h.longLines = selfstat.Register("http_listener", "long_lines", tags)
if h.MaxBodySize.Size == 0 {
h.MaxBodySize.Size = DEFAULT_MAX_BODY_SIZE
}
if h.MaxLineSize.Size == 0 {
h.MaxLineSize.Size = DEFAULT_MAX_LINE_SIZE
}
if h.ReadTimeout.Duration < time.Second {
h.ReadTimeout.Duration = time.Second * 10
}
if h.WriteTimeout.Duration < time.Second {
h.WriteTimeout.Duration = time.Second * 10
}
h.acc = acc
h.pool = NewPool(200, int(h.MaxLineSize.Size))
tlsConf, err := h.ServerConfig.TLSConfig()
if err != nil {
return err
}
server := &http.Server{
Addr: h.ServiceAddress,
Handler: h,
ReadTimeout: h.ReadTimeout.Duration,
WriteTimeout: h.WriteTimeout.Duration,
TLSConfig: tlsConf,
}
var listener net.Listener
if tlsConf != nil {
listener, err = tls.Listen("tcp", h.ServiceAddress, tlsConf)
} else {
listener, err = net.Listen("tcp", h.ServiceAddress)
}
if err != nil {
return err
}
h.listener = listener
h.Port = listener.Addr().(*net.TCPAddr).Port
h.handler = influx.NewMetricHandler()
h.parser = influx.NewParser(h.handler)
h.wg.Add(1)
go func() {
defer h.wg.Done()
server.Serve(h.listener)
}()
h.Log.Infof("Started HTTP listener service on %s", h.ServiceAddress)
return nil
}
// Stop cleans up all resources
func (h *HTTPListener) Stop() {
h.mu.Lock()
defer h.mu.Unlock()
h.listener.Close()
h.wg.Wait()
h.Log.Infof("Stopped HTTP listener service on %s", h.ServiceAddress)
}
func (h *HTTPListener) ServeHTTP(res http.ResponseWriter, req *http.Request) {
h.RequestsRecv.Incr(1)
defer h.RequestsServed.Incr(1)
switch req.URL.Path {
case "/write":
h.WritesRecv.Incr(1)
defer h.WritesServed.Incr(1)
h.AuthenticateIfSet(h.serveWrite, res, req)
case "/query":
h.QueriesRecv.Incr(1)
defer h.QueriesServed.Incr(1)
// Deliver a dummy response to the query endpoint, as some InfluxDB
// clients test endpoint availability with a query
h.AuthenticateIfSet(func(res http.ResponseWriter, req *http.Request) {
res.Header().Set("Content-Type", "application/json")
res.Header().Set("X-Influxdb-Version", "1.0")
res.WriteHeader(http.StatusOK)
res.Write([]byte("{\"results\":[]}"))
}, res, req)
case "/ping":
h.PingsRecv.Incr(1)
defer h.PingsServed.Incr(1)
verbose := req.URL.Query().Get("verbose")
// respond to ping requests
if verbose != "" && verbose != "0" && verbose != "false" {
res.WriteHeader(http.StatusOK)
b, _ := json.Marshal(map[string]string{"version": "1.0"}) // based on header set above
res.Write(b)
} else {
res.WriteHeader(http.StatusNoContent)
}
default:
defer h.NotFoundsServed.Incr(1)
// Don't know how to respond to calls to other endpoints
h.AuthenticateIfSet(http.NotFound, res, req)
}
}
func (h *HTTPListener) serveWrite(res http.ResponseWriter, req *http.Request) {
// Check that the content length is not too large for us to handle.
if req.ContentLength > h.MaxBodySize.Size {
tooLarge(res)
return
}
now := h.TimeFunc()
precision := req.URL.Query().Get("precision")
db := req.URL.Query().Get("db")
// Handle gzip request bodies
body := req.Body
if req.Header.Get("Content-Encoding") == "gzip" {
var err error
body, err = gzip.NewReader(req.Body)
if err != nil {
h.Log.Debug(err.Error())
badRequest(res, err.Error())
return
}
defer body.Close()
}
body = http.MaxBytesReader(res, body, h.MaxBodySize.Size)
var return400 bool
var hangingBytes bool
buf := h.pool.get()
defer h.pool.put(buf)
bufStart := 0
for {
n, err := io.ReadFull(body, buf[bufStart:])
if err != nil && err != io.ErrUnexpectedEOF && err != io.EOF {
h.Log.Debug(err.Error())
// problem reading the request body
badRequest(res, err.Error())
return
}
h.BytesRecv.Incr(int64(n))
if err == io.EOF {
if return400 {
badRequest(res, "")
} else {
res.WriteHeader(http.StatusNoContent)
}
return
}
if hangingBytes {
i := bytes.IndexByte(buf, '\n')
if i == -1 {
// still didn't find a newline, keep scanning
continue
}
// rotate the bit remaining after the first newline to the front of the buffer
i++ // start copying after the newline
bufStart = len(buf) - i
if bufStart > 0 {
copy(buf, buf[i:])
}
hangingBytes = false
continue
}
if err == io.ErrUnexpectedEOF {
// finished reading the request body
err = h.parse(buf[:n+bufStart], now, precision, db)
if err != nil {
h.Log.Debugf("%s: %s", err.Error(), bufStart+n)
return400 = true
}
if return400 {
if err != nil {
badRequest(res, err.Error())
} else {
badRequest(res, "")
}
} else {
res.WriteHeader(http.StatusNoContent)
}
return
}
// if we got down here it means that we filled our buffer, and there
// are still bytes remaining to be read. So we will parse up until the
// final newline, then push the rest of the bytes into the next buffer.
i := bytes.LastIndexByte(buf, '\n')
if i == -1 {
h.longLines.Incr(1)
// drop any line longer than the max buffer size
h.Log.Debugf("Http_listener received a single line longer than the maximum of %d bytes",
len(buf))
hangingBytes = true
return400 = true
bufStart = 0
continue
}
if err := h.parse(buf[:i+1], now, precision, db); err != nil {
h.Log.Debug(err.Error())
return400 = true
}
// rotate the bit remaining after the last newline to the front of the buffer
i++ // start copying after the newline
bufStart = len(buf) - i
if bufStart > 0 {
copy(buf, buf[i:])
}
}
}
func (h *HTTPListener) parse(b []byte, t time.Time, precision, db string) error {
h.mu.Lock()
defer h.mu.Unlock()
h.handler.SetTimePrecision(getPrecisionMultiplier(precision))
h.handler.SetTimeFunc(func() time.Time { return t })
metrics, err := h.parser.Parse(b)
if err != nil {
return fmt.Errorf("unable to parse: %s", err.Error())
}
for _, m := range metrics {
// Do we need to keep the database name in the query string.
// If a tag has been supplied to put the db in and we actually got a db query,
// then we write it in. This overwrites the database tag if one was sent.
// This makes it behave like the influx endpoint.
if h.DatabaseTag != "" && db != "" {
m.AddTag(h.DatabaseTag, db)
}
h.acc.AddFields(m.Name(), m.Fields(), m.Tags(), m.Time())
}
return nil
}
func tooLarge(res http.ResponseWriter) {
res.Header().Set("Content-Type", "application/json")
res.Header().Set("X-Influxdb-Version", "1.0")
res.Header().Set("X-Influxdb-Error", "http: request body too large")
res.WriteHeader(http.StatusRequestEntityTooLarge)
res.Write([]byte(`{"error":"http: request body too large"}`))
}
func badRequest(res http.ResponseWriter, errString string) {
res.Header().Set("Content-Type", "application/json")
res.Header().Set("X-Influxdb-Version", "1.0")
if errString == "" {
errString = "http: bad request"
}
res.Header().Set("X-Influxdb-Error", errString)
res.WriteHeader(http.StatusBadRequest)
res.Write([]byte(fmt.Sprintf(`{"error":%q}`, errString)))
}
func (h *HTTPListener) AuthenticateIfSet(handler http.HandlerFunc, res http.ResponseWriter, req *http.Request) {
if h.BasicUsername != "" && h.BasicPassword != "" {
reqUsername, reqPassword, ok := req.BasicAuth()
if !ok ||
subtle.ConstantTimeCompare([]byte(reqUsername), []byte(h.BasicUsername)) != 1 ||
subtle.ConstantTimeCompare([]byte(reqPassword), []byte(h.BasicPassword)) != 1 {
h.AuthFailures.Incr(1)
http.Error(res, "Unauthorized.", http.StatusUnauthorized)
return
}
handler(res, req)
} else {
handler(res, req)
}
}
func getPrecisionMultiplier(precision string) time.Duration {
d := time.Nanosecond
switch precision {
case "u":
d = time.Microsecond
case "ms":
d = time.Millisecond
case "s":
d = time.Second
case "m":
d = time.Minute
case "h":
d = time.Hour
}
return d
}
func init() {
// http_listener deprecated in 1.9
inputs.Add("http_listener", func() telegraf.Input {
return &HTTPListener{
ServiceAddress: ":8186",
TimeFunc: time.Now,
}
})
inputs.Add("influxdb_listener", func() telegraf.Input {
return &HTTPListener{
ServiceAddress: ":8186",
TimeFunc: time.Now,
}
})
}

484
plugins/inputs/influxdb_listener/http_listener_test.go
View File

File diff suppressed because one or more lines are too long

406
plugins/inputs/influxdb_listener/influxdb_listener.go Normal file
View File

@ -0,0 +1,406 @@
package influxdb_listener
import (
"compress/gzip"
"context"
"crypto/tls"
"encoding/json"
"fmt"
"net"
"net/http"
"time"
"github.com/influxdata/telegraf"
"github.com/influxdata/telegraf/internal"
tlsint "github.com/influxdata/telegraf/internal/tls"
"github.com/influxdata/telegraf/plugins/inputs"
"github.com/influxdata/telegraf/plugins/parsers/influx"
"github.com/influxdata/telegraf/selfstat"
)
const (
// defaultMaxBodySize is the default maximum request body size, in bytes.
// if the request body is over this size, we will return an HTTP 413 error.
defaultMaxBodySize = 32 * 1024 * 1024
)
type InfluxDBListener struct {
ServiceAddress string `toml:"service_address"`
port int
tlsint.ServerConfig
ReadTimeout internal.Duration `toml:"read_timeout"`
WriteTimeout internal.Duration `toml:"write_timeout"`
MaxBodySize internal.Size `toml:"max_body_size"`
MaxLineSize internal.Size `toml:"max_line_size"` // deprecated in 1.14; ignored
BasicUsername string `toml:"basic_username"`
BasicPassword string `toml:"basic_password"`
DatabaseTag string `toml:"database_tag"`
timeFunc influx.TimeFunc
listener net.Listener
server http.Server
acc telegraf.Accumulator
bytesRecv selfstat.Stat
requestsServed selfstat.Stat
writesServed selfstat.Stat
queriesServed selfstat.Stat
pingsServed selfstat.Stat
requestsRecv selfstat.Stat
notFoundsServed selfstat.Stat
buffersCreated selfstat.Stat
authFailures selfstat.Stat
Log telegraf.Logger `toml:"-"`
mux http.ServeMux
}
const sampleConfig = `
## Address and port to host InfluxDB listener on
service_address = ":8186"
## maximum duration before timing out read of the request
read_timeout = "10s"
## maximum duration before timing out write of the response
write_timeout = "10s"
## Maximum allowed HTTP request body size in bytes.
## 0 means to use the default of 32MiB.
max_body_size = "32MiB"
## Optional tag name used to store the database.
## If the write has a database in the query string then it will be kept in this tag name.
## This tag can be used in downstream outputs.
## The default value of nothing means it will be off and the database will not be recorded.
# database_tag = ""
## Set one or more allowed client CA certificate file names to
## enable mutually authenticated TLS connections
tls_allowed_cacerts = ["/etc/telegraf/clientca.pem"]
## Add service certificate and key
tls_cert = "/etc/telegraf/cert.pem"
tls_key = "/etc/telegraf/key.pem"
## Optional username and password to accept for HTTP basic authentication.
## You probably want to make sure you have TLS configured above for this.
# basic_username = "foobar"
# basic_password = "barfoo"
`
func (h *InfluxDBListener) SampleConfig() string {
return sampleConfig
}
func (h *InfluxDBListener) Description() string {
return "Accept metrics over InfluxDB 1.x HTTP API"
}
func (h *InfluxDBListener) Gather(_ telegraf.Accumulator) error {
return nil
}
func (h *InfluxDBListener) routes() {
authHandler := internal.AuthHandler(h.BasicUsername, h.BasicPassword, "influxdb",
func(_ http.ResponseWriter) {
h.authFailures.Incr(1)
},
)
h.mux.Handle("/write", authHandler(h.handleWrite()))
h.mux.Handle("/query", authHandler(h.handleQuery()))
h.mux.Handle("/ping", h.handlePing())
h.mux.Handle("/", authHandler(h.handleDefault()))
}
func (h *InfluxDBListener) Init() error {
tags := map[string]string{
"address": h.ServiceAddress,
}
h.bytesRecv = selfstat.Register("influxdb_listener", "bytes_received", tags)
h.requestsServed = selfstat.Register("influxdb_listener", "requests_served", tags)
h.writesServed = selfstat.Register("influxdb_listener", "writes_served", tags)
h.queriesServed = selfstat.Register("influxdb_listener", "queries_served", tags)
h.pingsServed = selfstat.Register("influxdb_listener", "pings_served", tags)
h.requestsRecv = selfstat.Register("influxdb_listener", "requests_received", tags)
h.notFoundsServed = selfstat.Register("influxdb_listener", "not_founds_served", tags)
h.buffersCreated = selfstat.Register("influxdb_listener", "buffers_created", tags)
h.authFailures = selfstat.Register("influxdb_listener", "auth_failures", tags)
h.routes()
if h.MaxBodySize.Size == 0 {
h.MaxBodySize.Size = defaultMaxBodySize
}
if h.MaxLineSize.Size != 0 {
h.Log.Warnf("Use of deprecated configuration: 'max_line_size'; parser now handles lines of unlimited length and option is ignored")
}
if h.ReadTimeout.Duration < time.Second {
h.ReadTimeout.Duration = time.Second * 10
}
if h.WriteTimeout.Duration < time.Second {
h.WriteTimeout.Duration = time.Second * 10
}
return nil
}
// Start starts the InfluxDB listener service.
func (h *InfluxDBListener) Start(acc telegraf.Accumulator) error {
h.acc = acc
tlsConf, err := h.ServerConfig.TLSConfig()
if err != nil {
return err
}
h.server = http.Server{
Addr: h.ServiceAddress,
Handler: h,
ReadTimeout: h.ReadTimeout.Duration,
WriteTimeout: h.WriteTimeout.Duration,
TLSConfig: tlsConf,
}
var listener net.Listener
if tlsConf != nil {
listener, err = tls.Listen("tcp", h.ServiceAddress, tlsConf)
if err != nil {
return err
}
} else {
listener, err = net.Listen("tcp", h.ServiceAddress)
if err != nil {
return err
}
}
h.listener = listener
h.port = listener.Addr().(*net.TCPAddr).Port
go func() {
err = h.server.Serve(h.listener)
if err != http.ErrServerClosed {
h.Log.Infof("Error serving HTTP on %s", h.ServiceAddress)
}
}()
h.Log.Infof("Started HTTP listener service on %s", h.ServiceAddress)
return nil
}
// Stop cleans up all resources
func (h *InfluxDBListener) Stop() {
err := h.server.Shutdown(context.Background())
if err != nil {
h.Log.Infof("Error shutting down HTTP server: %v", err.Error())
}
}
func (h *InfluxDBListener) ServeHTTP(res http.ResponseWriter, req *http.Request) {
h.requestsRecv.Incr(1)
h.mux.ServeHTTP(res, req)
h.requestsServed.Incr(1)
}
func (h *InfluxDBListener) handleQuery() http.HandlerFunc {
return func(res http.ResponseWriter, req *http.Request) {
defer h.queriesServed.Incr(1)
// Deliver a dummy response to the query endpoint, as some InfluxDB
// clients test endpoint availability with a query
res.Header().Set("Content-Type", "application/json")
res.Header().Set("X-Influxdb-Version", "1.0")
res.WriteHeader(http.StatusOK)
res.Write([]byte("{\"results\":[]}"))
}
}
func (h *InfluxDBListener) handlePing() http.HandlerFunc {
return func(res http.ResponseWriter, req *http.Request) {
defer h.pingsServed.Incr(1)
verbose := req.URL.Query().Get("verbose")
// respond to ping requests
if verbose != "" && verbose != "0" && verbose != "false" {
res.WriteHeader(http.StatusOK)
b, _ := json.Marshal(map[string]string{"version": "1.0"}) // based on header set above
res.Write(b)
} else {
res.WriteHeader(http.StatusNoContent)
}
}
}
func (h *InfluxDBListener) handleDefault() http.HandlerFunc {
return func(res http.ResponseWriter, req *http.Request) {
defer h.notFoundsServed.Incr(1)
http.NotFound(res, req)
}
}
func (h *InfluxDBListener) handleWrite() http.HandlerFunc {
return func(res http.ResponseWriter, req *http.Request) {
defer h.writesServed.Incr(1)
// Check that the content length is not too large for us to handle.
if req.ContentLength > h.MaxBodySize.Size {
tooLarge(res)
return
}
db := req.URL.Query().Get("db")
body := req.Body
body = http.MaxBytesReader(res, body, h.MaxBodySize.Size)
// Handle gzip request bodies
if req.Header.Get("Content-Encoding") == "gzip" {
var err error
body, err = gzip.NewReader(body)
if err != nil {
h.Log.Debugf("Error decompressing request body: %v", err.Error())
badRequest(res, err.Error())
return
}
defer body.Close()
}
parser := influx.NewStreamParser(body)
parser.SetTimeFunc(h.timeFunc)
precisionStr := req.URL.Query().Get("precision")
if precisionStr != "" {
precision := getPrecisionMultiplier(precisionStr)
parser.SetTimePrecision(precision)
}
var m telegraf.Metric
var err error
var parseErrorCount int
var lastPos int = 0
var firstParseErrorStr string
for {
select {
case <-req.Context().Done():
// Shutting down before parsing is finished.
res.WriteHeader(http.StatusServiceUnavailable)
return
default:
}
m, err = parser.Next()
pos := parser.Position()
h.bytesRecv.Incr(int64(pos - lastPos))
lastPos = pos
// Continue parsing metrics even if some are malformed
if parseErr, ok := err.(*influx.ParseError); ok {
parseErrorCount += 1
errStr := parseErr.Error()
if firstParseErrorStr == "" {
firstParseErrorStr = errStr
}
continue
} else if err != nil {
// Either we're exiting cleanly (err ==
// influx.EOF) or there's an unexpected error
break
}
if h.DatabaseTag != "" && db != "" {
m.AddTag(h.DatabaseTag, db)
}
h.acc.AddMetric(m)
}
if err != influx.EOF {
h.Log.Debugf("Error parsing the request body: %v", err.Error())
badRequest(res, err.Error())
return
}
if parseErrorCount > 0 {
var partialErrorString string
switch parseErrorCount {
case 1:
partialErrorString = fmt.Sprintf("%s", firstParseErrorStr)
case 2:
partialErrorString = fmt.Sprintf("%s (and 1 other parse error)", firstParseErrorStr)
default:
partialErrorString = fmt.Sprintf("%s (and %d other parse errors)", firstParseErrorStr, parseErrorCount-1)
}
partialWrite(res, partialErrorString)
return
}
// http request success
res.WriteHeader(http.StatusNoContent)
}
}
func tooLarge(res http.ResponseWriter) {
res.Header().Set("Content-Type", "application/json")
res.Header().Set("X-Influxdb-Version", "1.0")
res.Header().Set("X-Influxdb-Error", "http: request body too large")
res.WriteHeader(http.StatusRequestEntityTooLarge)
res.Write([]byte(`{"error":"http: request body too large"}`))
}
func badRequest(res http.ResponseWriter, errString string) {
res.Header().Set("Content-Type", "application/json")
res.Header().Set("X-Influxdb-Version", "1.0")
if errString == "" {
errString = "http: bad request"
}
res.Header().Set("X-Influxdb-Error", errString)
res.WriteHeader(http.StatusBadRequest)
res.Write([]byte(fmt.Sprintf(`{"error":%q}`, errString)))
}
func partialWrite(res http.ResponseWriter, errString string) {
res.Header().Set("Content-Type", "application/json")
res.Header().Set("X-Influxdb-Version", "1.0")
res.Header().Set("X-Influxdb-Error", errString)
res.WriteHeader(http.StatusBadRequest)
res.Write([]byte(fmt.Sprintf(`{"error":%q}`, errString)))
}
func getPrecisionMultiplier(precision string) time.Duration {
// Influxdb defaults silently to nanoseconds if precision isn't
// one of the following:
var d time.Duration
switch precision {
case "u":
d = time.Microsecond
case "ms":
d = time.Millisecond
case "s":
d = time.Second
case "m":
d = time.Minute
case "h":
d = time.Hour
default:
d = time.Nanosecond
}
return d
}
func init() {
// http_listener deprecated in 1.9
inputs.Add("http_listener", func() telegraf.Input {
return &InfluxDBListener{
ServiceAddress: ":8186",
timeFunc: time.Now,
}
})
inputs.Add("influxdb_listener", func() telegraf.Input {
return &InfluxDBListener{
ServiceAddress: ":8186",
timeFunc: time.Now,
}
})
}

108
plugins/inputs/influxdb_listener/influxdb_listener_benchmark_test.go Normal file
View File

@ -0,0 +1,108 @@
package influxdb_listener
import (
"fmt"
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
"github.com/influxdata/telegraf/internal"
"github.com/influxdata/telegraf/selfstat"
"github.com/influxdata/telegraf/testutil"
)
// newListener is the minimal InfluxDBListener construction to serve writes.
func newListener() *InfluxDBListener {
listener := &InfluxDBListener{
timeFunc: time.Now,
acc: &testutil.NopAccumulator{},
bytesRecv: selfstat.Register("influxdb_listener", "bytes_received", map[string]string{}),
writesServed: selfstat.Register("influxdb_listener", "writes_served", map[string]string{}),
MaxBodySize: internal.Size{
Size: defaultMaxBodySize,
},
}
return listener
}
func BenchmarkInfluxDBListener_serveWrite(b *testing.B) {
res := httptest.NewRecorder()
addr := "http://localhost/write?db=mydb"
benchmarks := []struct {
name string
lines string
}{
{
name: "single line, tag, and field",
lines: lines(1, 1, 1),
},
{
name: "single line, 10 tags and fields",
lines: lines(1, 10, 10),
},
{
name: "single line, 100 tags and fields",
lines: lines(1, 100, 100),
},
{
name: "1k lines, single tag and field",
lines: lines(1000, 1, 1),
},
{
name: "1k lines, 10 tags and fields",
lines: lines(1000, 10, 10),
},
{
name: "10k lines, 10 tags and fields",
lines: lines(10000, 10, 10),
},
{
name: "100k lines, 10 tags and fields",
lines: lines(100000, 10, 10),
},
}
for _, bm := range benchmarks {
b.Run(bm.name, func(b *testing.B) {
listener := newListener()
b.ResetTimer()
for n := 0; n < b.N; n++ {
req, err := http.NewRequest("POST", addr, strings.NewReader(bm.lines))
if err != nil {
b.Error(err)
}
listener.handleWrite()(res, req)
if res.Code != http.StatusNoContent {
b.Errorf("unexpected status %d", res.Code)
}
}
})
}
}
func lines(lines, numTags, numFields int) string {
lp := make([]string, lines)
for i := 0; i < lines; i++ {
tags := make([]string, numTags)
for j := 0; j < numTags; j++ {
tags[j] = fmt.Sprintf("t%d=v%d", j, j)
}
fields := make([]string, numFields)
for k := 0; k < numFields; k++ {
fields[k] = fmt.Sprintf("f%d=%d", k, k)
}
lp[i] = fmt.Sprintf("m%d,%s %s",
i,
strings.Join(tags, ","),
strings.Join(fields, ","),
)
}
return strings.Join(lp, "\n")
}

656
plugins/inputs/influxdb_listener/influxdb_listener_test.go
View File

File diff suppressed because one or more lines are too long

5
plugins/outputs/health/health.go
View File

@ -136,7 +136,7 @@ func (h *Health) Init() error {
// Connect starts the HTTP server. // Connect starts the HTTP server.
func (h *Health) Connect() error { func (h *Health) Connect() error {
authHandler := internal.AuthHandler(h.BasicUsername, h.BasicPassword, onAuthError) authHandler := internal.AuthHandler(h.BasicUsername, h.BasicPassword, "health", onAuthError)
h.server = &http.Server{ h.server = &http.Server{
Addr: h.ServiceAddress, Addr: h.ServiceAddress,
@ -168,8 +168,7 @@ func (h *Health) Connect() error {
return nil return nil
} }
func onAuthError(rw http.ResponseWriter, code int) { func onAuthError(_ http.ResponseWriter) {
http.Error(rw, http.StatusText(code), code)
} }
func (h *Health) listen() (net.Listener, error) { func (h *Health) listen() (net.Listener, error) {

6
plugins/outputs/prometheus_client/prometheus_client.go
View File

@ -156,7 +156,7 @@ func (p *PrometheusClient) Init() error {
ipRange = append(ipRange, ipNet) ipRange = append(ipRange, ipNet)
} }
authHandler := internal.AuthHandler(p.BasicUsername, p.BasicPassword, onAuthError) authHandler := internal.AuthHandler(p.BasicUsername, p.BasicPassword, "prometheus", onAuthError)
rangeHandler := internal.IPRangeHandler(ipRange, onError) rangeHandler := internal.IPRangeHandler(ipRange, onError)
promHandler := promhttp.HandlerFor(registry, promhttp.HandlerOpts{ErrorHandling: promhttp.ContinueOnError}) promHandler := promhttp.HandlerFor(registry, promhttp.HandlerOpts{ErrorHandling: promhttp.ContinueOnError})
@ -219,9 +219,7 @@ func (p *PrometheusClient) Connect() error {
return nil return nil
} }
func onAuthError(rw http.ResponseWriter, code int) { func onAuthError(_ http.ResponseWriter) {
rw.Header().Set("WWW-Authenticate", `Basic realm="Restricted"`)
http.Error(rw, http.StatusText(code), code)
} }
func onError(rw http.ResponseWriter, code int) { func onError(rw http.ResponseWriter, code int) {

4
plugins/parsers/csv/parser.go
View File

@ -13,6 +13,8 @@ import (
"github.com/influxdata/telegraf/metric" "github.com/influxdata/telegraf/metric"
) )
type TimeFunc func() time.Time
type Parser struct { type Parser struct {
MetricName string MetricName string
HeaderRowCount int HeaderRowCount int
@ -31,7 +33,7 @@ type Parser struct {
TimeFunc func() time.Time TimeFunc func() time.Time
} }
func (p *Parser) SetTimeFunc(fn metric.TimeFunc) { func (p *Parser) SetTimeFunc(fn TimeFunc) {
p.TimeFunc = fn p.TimeFunc = fn
} }

6
plugins/parsers/dropwizard/parser.go
View File

@ -17,6 +17,8 @@ import (
var fieldEscaper = strings.NewReplacer("\\", "\\\\", "\"", "\\\"") var fieldEscaper = strings.NewReplacer("\\", "\\\\", "\"", "\\\"")
var keyEscaper = strings.NewReplacer(" ", "\\ ", ",", "\\,", "=", "\\=") var keyEscaper = strings.NewReplacer(" ", "\\ ", ",", "\\,", "=", "\\=")
type TimeFunc func() time.Time
// Parser parses json inputs containing dropwizard metrics, // Parser parses json inputs containing dropwizard metrics,
// either top-level or embedded inside a json field. // either top-level or embedded inside a json field.
// This parser is using gjson for retrieving paths within the json file. // This parser is using gjson for retrieving paths within the json file.
@ -48,7 +50,7 @@ type parser struct {
separator string separator string
templateEngine *templating.Engine templateEngine *templating.Engine
timeFunc metric.TimeFunc timeFunc TimeFunc
// seriesParser parses line protocol measurement + tags // seriesParser parses line protocol measurement + tags
seriesParser *influx.Parser seriesParser *influx.Parser
@ -267,6 +269,6 @@ func (p *parser) readDWMetrics(metricType string, dwms interface{}, metrics []te
return metrics return metrics
} }
func (p *parser) SetTimeFunc(f metric.TimeFunc) { func (p *parser) SetTimeFunc(f TimeFunc) {
p.timeFunc = f p.timeFunc = f
} }

10
plugins/parsers/dropwizard/parser_test.go
View File

@ -13,7 +13,7 @@ import (
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
) )
var TimeFunc = func() time.Time { var testTimeFunc = func() time.Time {
return time.Unix(0, 0) return time.Unix(0, 0)
} }
@ -528,7 +528,7 @@ func TestDropWizard(t *testing.T) {
map[string]interface{}{ map[string]interface{}{
"value": 42.0, "value": 42.0,
}, },
TimeFunc(), testTimeFunc(),
), ),
), ),
}, },
@ -547,7 +547,7 @@ func TestDropWizard(t *testing.T) {
map[string]interface{}{ map[string]interface{}{
"value": 42.0, "value": 42.0,
}, },
TimeFunc(), testTimeFunc(),
), ),
), ),
}, },
@ -573,7 +573,7 @@ func TestDropWizard(t *testing.T) {
map[string]interface{}{ map[string]interface{}{
"value": 42.0, "value": 42.0,
}, },
TimeFunc(), testTimeFunc(),
), ),
), ),
}, },
@ -584,7 +584,7 @@ func TestDropWizard(t *testing.T) {
for _, tt := range tests { for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) { t.Run(tt.name, func(t *testing.T) {
parser := NewParser() parser := NewParser()
parser.SetTimeFunc(TimeFunc) parser.SetTimeFunc(testTimeFunc)
metrics, err := parser.Parse(tt.input) metrics, err := parser.Parse(tt.input)
tt.errFunc(t, err) tt.errFunc(t, err)

62
plugins/parsers/influx/handler.go
View File

@ -10,43 +10,53 @@ import (
"github.com/influxdata/telegraf/metric" "github.com/influxdata/telegraf/metric"
) )
// MetricHandler implements the Handler interface and produces telegraf.Metric.
type MetricHandler struct { type MetricHandler struct {
builder *metric.Builder err error
err error timePrecision time.Duration
precision time.Duration timeFunc TimeFunc
metric telegraf.Metric
} }
func NewMetricHandler() *MetricHandler { func NewMetricHandler() *MetricHandler {
return &MetricHandler{ return &MetricHandler{
builder: metric.NewBuilder(), timePrecision: time.Nanosecond,
precision: time.Nanosecond, timeFunc: time.Now,
} }
} }
func (h *MetricHandler) SetTimeFunc(f metric.TimeFunc) { func (h *MetricHandler) SetTimePrecision(p time.Duration) {
h.builder.TimeFunc = f h.timePrecision = p
// When the timestamp is omitted from the metric, the timestamp
// comes from the server clock, truncated to the nearest unit of
// measurement provided in precision.
//
// When a timestamp is provided in the metric, precsision is
// overloaded to hold the unit of measurement of the timestamp.
} }
func (h *MetricHandler) SetTimePrecision(precision time.Duration) { func (h *MetricHandler) SetTimeFunc(f TimeFunc) {
h.builder.TimePrecision = precision h.timeFunc = f
h.precision = precision
} }
func (h *MetricHandler) Metric() (telegraf.Metric, error) { func (h *MetricHandler) Metric() (telegraf.Metric, error) {
m, err := h.builder.Metric() if h.metric.Time().IsZero() {
h.builder.Reset() h.metric.SetTime(h.timeFunc().Truncate(h.timePrecision))
return m, err }
return h.metric, nil
} }
func (h *MetricHandler) SetMeasurement(name []byte) error { func (h *MetricHandler) SetMeasurement(name []byte) error {
h.builder.SetName(nameUnescape(name)) var err error
return nil h.metric, err = metric.New(nameUnescape(name),
nil, nil, time.Time{})
return err
} }
func (h *MetricHandler) AddTag(key []byte, value []byte) error { func (h *MetricHandler) AddTag(key []byte, value []byte) error {
tk := unescape(key) tk := unescape(key)
tv := unescape(value) tv := unescape(value)
h.builder.AddTag(tk, tv) h.metric.AddTag(tk, tv)
return nil return nil
} }
@ -59,7 +69,7 @@ func (h *MetricHandler) AddInt(key []byte, value []byte) error {
} }
return err return err
} }
h.builder.AddField(fk, fv) h.metric.AddField(fk, fv)
return nil return nil
} }
@ -72,7 +82,7 @@ func (h *MetricHandler) AddUint(key []byte, value []byte) error {
} }
return err return err
} }
h.builder.AddField(fk, fv) h.metric.AddField(fk, fv)
return nil return nil
} }
@ -85,14 +95,14 @@ func (h *MetricHandler) AddFloat(key []byte, value []byte) error {
} }
return err return err
} }
h.builder.AddField(fk, fv) h.metric.AddField(fk, fv)
return nil return nil
} }
func (h *MetricHandler) AddString(key []byte, value []byte) error { func (h *MetricHandler) AddString(key []byte, value []byte) error {
fk := unescape(key) fk := unescape(key)
fv := stringFieldUnescape(value) fv := stringFieldUnescape(value)
h.builder.AddField(fk, fv) h.metric.AddField(fk, fv)
return nil return nil
} }
@ -102,7 +112,7 @@ func (h *MetricHandler) AddBool(key []byte, value []byte) error {
if err != nil { if err != nil {
return errors.New("unparseable bool") return errors.New("unparseable bool")
} }
h.builder.AddField(fk, fv) h.metric.AddField(fk, fv)
return nil return nil
} }
@ -114,11 +124,9 @@ func (h *MetricHandler) SetTimestamp(tm []byte) error {
} }
return err return err
} }
ns := v * int64(h.precision)
h.builder.SetTime(time.Unix(0, ns)) //time precision is overloaded to mean time unit here
ns := v * int64(h.timePrecision)
h.metric.SetTime(time.Unix(0, ns))
return nil return nil
} }
func (h *MetricHandler) Reset() {
h.builder.Reset()
}

34919
plugins/parsers/influx/machine.go
View File

File diff suppressed because it is too large Load Diff

160
plugins/parsers/influx/machine.go.rl
View File

@ -2,6 +2,7 @@ package influx
import ( import (
"errors" "errors"
"io"
) )
var ( var (
@ -70,8 +71,8 @@ action goto_align {
fgoto align; fgoto align;
} }
action found_metric { action begin_metric {
foundMetric = true m.beginMetric = true
} }
action name { action name {
@ -84,11 +85,11 @@ action name {
} }
action tagkey { action tagkey {
key = m.text() m.key = m.text()
} }
action tagvalue { action tagvalue {
err = m.handler.AddTag(key, m.text()) err = m.handler.AddTag(m.key, m.text())
if err != nil { if err != nil {
fhold; fhold;
fnext discard_line; fnext discard_line;
@ -97,11 +98,11 @@ action tagvalue {
} }
action fieldkey { action fieldkey {
key = m.text() m.key = m.text()
} }
action integer { action integer {
err = m.handler.AddInt(key, m.text()) err = m.handler.AddInt(m.key, m.text())
if err != nil { if err != nil {
fhold; fhold;
fnext discard_line; fnext discard_line;
@ -110,7 +111,7 @@ action integer {
} }
action unsigned { action unsigned {
err = m.handler.AddUint(key, m.text()) err = m.handler.AddUint(m.key, m.text())
if err != nil { if err != nil {
fhold; fhold;
fnext discard_line; fnext discard_line;
@ -119,7 +120,7 @@ action unsigned {
} }
action float { action float {
err = m.handler.AddFloat(key, m.text()) err = m.handler.AddFloat(m.key, m.text())
if err != nil { if err != nil {
fhold; fhold;
fnext discard_line; fnext discard_line;
@ -128,7 +129,7 @@ action float {
} }
action bool { action bool {
err = m.handler.AddBool(key, m.text()) err = m.handler.AddBool(m.key, m.text())
if err != nil { if err != nil {
fhold; fhold;
fnext discard_line; fnext discard_line;
@ -137,7 +138,7 @@ action bool {
} }
action string { action string {
err = m.handler.AddString(key, m.text()) err = m.handler.AddString(m.key, m.text())
if err != nil { if err != nil {
fhold; fhold;
fnext discard_line; fnext discard_line;
@ -161,15 +162,20 @@ action incr_newline {
} }
action eol { action eol {
m.finishMetric = true
fnext align; fnext align;
fbreak; fbreak;
} }
action finish_metric {
m.finishMetric = true
}
ws = ws =
[\t\v\f ]; [\t\v\f ];
newline = newline =
'\r'? '\n' %to(incr_newline); '\r'? '\n' >incr_newline;
non_zero_digit = non_zero_digit =
[1-9]; [1-9];
@ -273,7 +279,7 @@ line_without_term =
main := main :=
(line_with_term* (line_with_term*
(line_with_term | line_without_term?) (line_with_term | line_without_term?)
) >found_metric ) >begin_metric %eof(finish_metric)
; ;
# The discard_line machine discards the current line. Useful for recovering # The discard_line machine discards the current line. Useful for recovering
@ -299,7 +305,7 @@ align :=
# Series is a machine for matching measurement+tagset # Series is a machine for matching measurement+tagset
series := series :=
(measurement >err(name_error) tagset eol_break?) (measurement >err(name_error) tagset eol_break?)
>found_metric >begin_metric
; ;
}%% }%%
@ -317,14 +323,17 @@ type Handler interface {
} }
type machine struct { type machine struct {
data []byte data []byte
cs int cs int
p, pe, eof int p, pe, eof int
pb int pb int
lineno int lineno int
sol int sol int
handler Handler handler Handler
initState int initState int
key []byte
beginMetric bool
finishMetric bool
} }
func NewMachine(handler Handler) *machine { func NewMachine(handler Handler) *machine {
@ -368,6 +377,9 @@ func (m *machine) SetData(data []byte) {
m.sol = 0 m.sol = 0
m.pe = len(data) m.pe = len(data)
m.eof = len(data) m.eof = len(data)
m.key = nil
m.beginMetric = false
m.finishMetric = false
%% write init; %% write init;
m.cs = m.initState m.cs = m.initState
@ -382,10 +394,15 @@ func (m *machine) Next() error {
return EOF return EOF
} }
var err error m.key = nil
var key []byte m.beginMetric = false
foundMetric := false m.finishMetric = false
return m.exec()
}
func (m *machine) exec() error {
var err error
%% write exec; %% write exec;
if err != nil { if err != nil {
@ -405,7 +422,7 @@ func (m *machine) Next() error {
// //
// Otherwise we have successfully parsed a metric line, so if we are at // Otherwise we have successfully parsed a metric line, so if we are at
// the EOF we will report it the next call. // the EOF we will report it the next call.
if !foundMetric && m.p == m.pe && m.pe == m.eof { if !m.beginMetric && m.p == m.pe && m.pe == m.eof {
return EOF return EOF
} }
@ -437,3 +454,96 @@ func (m *machine) Column() int {
func (m *machine) text() []byte { func (m *machine) text() []byte {
return m.data[m.pb:m.p] return m.data[m.pb:m.p]
} }
type streamMachine struct {
machine *machine
reader io.Reader
}
func NewStreamMachine(r io.Reader, handler Handler) *streamMachine {
m := &streamMachine{
machine: NewMachine(handler),
reader: r,
}
m.machine.SetData(make([]byte, 1024))
m.machine.pe = 0
m.machine.eof = -1
return m
}
func (m *streamMachine) Next() error {
// Check if we are already at EOF, this should only happen if called again
// after already returning EOF.
if m.machine.p == m.machine.pe && m.machine.pe == m.machine.eof {
return EOF
}
copy(m.machine.data, m.machine.data[m.machine.p:])
m.machine.pe = m.machine.pe - m.machine.p
m.machine.sol = m.machine.sol - m.machine.p
m.machine.pb = 0
m.machine.p = 0
m.machine.eof = -1
m.machine.key = nil
m.machine.beginMetric = false
m.machine.finishMetric = false
for {
// Expand the buffer if it is full
if m.machine.pe == len(m.machine.data) {
expanded := make([]byte, 2 * len(m.machine.data))
copy(expanded, m.machine.data)
m.machine.data = expanded
}
n, err := m.reader.Read(m.machine.data[m.machine.pe:])
if n == 0 && err == io.EOF {
m.machine.eof = m.machine.pe
} else if err != nil && err != io.EOF {
return err
}
m.machine.pe += n
err = m.machine.exec()
if err != nil {
return err
}
// If we have successfully parsed a full metric line break out
if m.machine.finishMetric {
break
}
}
return nil
}
// Position returns the current byte offset into the data.
func (m *streamMachine) Position() int {
return m.machine.Position()
}
// LineOffset returns the byte offset of the current line.
func (m *streamMachine) LineOffset() int {
return m.machine.LineOffset()
}
// LineNumber returns the current line number. Lines are counted based on the
// regular expression `\r?\n`.
func (m *streamMachine) LineNumber() int {
return m.machine.LineNumber()
}
// Column returns the current column.
func (m *streamMachine) Column() int {
return m.machine.Column()
}
// LineText returns the text of the current line that has been parsed so far.
func (m *streamMachine) LineText() string {
return string(m.machine.data[0:m.machine.p])
}

508
plugins/parsers/influx/machine_test.go
View File

@ -1,8 +1,10 @@
package influx_test package influx_test
import ( import (
"bytes"
"errors" "errors"
"fmt" "fmt"
"io"
"testing" "testing"
"github.com/influxdata/telegraf/plugins/parsers/influx" "github.com/influxdata/telegraf/plugins/parsers/influx"
@ -14,41 +16,59 @@ type TestingHandler struct {
} }
func (h *TestingHandler) SetMeasurement(name []byte) error { func (h *TestingHandler) SetMeasurement(name []byte) error {
n := make([]byte, len(name))
copy(n, name)
mname := Result{ mname := Result{
Name: Measurement, Name: Measurement,
Value: name, Value: n,
} }
h.results = append(h.results, mname) h.results = append(h.results, mname)
return nil return nil
} }
func (h *TestingHandler) AddTag(key []byte, value []byte) error { func (h *TestingHandler) AddTag(key []byte, value []byte) error {
k := make([]byte, len(key))
copy(k, key)
v := make([]byte, len(value))
copy(v, value)
tagkey := Result{ tagkey := Result{
Name: TagKey, Name: TagKey,
Value: key, Value: k,
} }
tagvalue := Result{ tagvalue := Result{
Name: TagValue, Name: TagValue,
Value: value, Value: v,
} }
h.results = append(h.results, tagkey, tagvalue) h.results = append(h.results, tagkey, tagvalue)
return nil return nil
} }
func (h *TestingHandler) AddInt(key []byte, value []byte) error { func (h *TestingHandler) AddInt(key []byte, value []byte) error {
k := make([]byte, len(key))
copy(k, key)
v := make([]byte, len(value))
copy(v, value)
fieldkey := Result{ fieldkey := Result{
Name: FieldKey, Name: FieldKey,
Value: key, Value: k,
} }
fieldvalue := Result{ fieldvalue := Result{
Name: FieldInt, Name: FieldInt,
Value: value, Value: v,
} }
h.results = append(h.results, fieldkey, fieldvalue) h.results = append(h.results, fieldkey, fieldvalue)
return nil return nil
} }
func (h *TestingHandler) AddUint(key []byte, value []byte) error { func (h *TestingHandler) AddUint(key []byte, value []byte) error {
k := make([]byte, len(key))
copy(k, key)
v := make([]byte, len(value))
copy(v, value)
fieldkey := Result{ fieldkey := Result{
Name: FieldKey, Name: FieldKey,
Value: key, Value: key,
@ -62,48 +82,66 @@ func (h *TestingHandler) AddUint(key []byte, value []byte) error {
} }
func (h *TestingHandler) AddFloat(key []byte, value []byte) error { func (h *TestingHandler) AddFloat(key []byte, value []byte) error {
k := make([]byte, len(key))
copy(k, key)
v := make([]byte, len(value))
copy(v, value)
fieldkey := Result{ fieldkey := Result{
Name: FieldKey, Name: FieldKey,
Value: key, Value: k,
} }
fieldvalue := Result{ fieldvalue := Result{
Name: FieldFloat, Name: FieldFloat,
Value: value, Value: v,
} }
h.results = append(h.results, fieldkey, fieldvalue) h.results = append(h.results, fieldkey, fieldvalue)
return nil return nil
} }
func (h *TestingHandler) AddString(key []byte, value []byte) error { func (h *TestingHandler) AddString(key []byte, value []byte) error {
k := make([]byte, len(key))
copy(k, key)
v := make([]byte, len(value))
copy(v, value)
fieldkey := Result{ fieldkey := Result{
Name: FieldKey, Name: FieldKey,
Value: key, Value: k,
} }
fieldvalue := Result{ fieldvalue := Result{
Name: FieldString, Name: FieldString,
Value: value, Value: v,
} }
h.results = append(h.results, fieldkey, fieldvalue) h.results = append(h.results, fieldkey, fieldvalue)
return nil return nil
} }
func (h *TestingHandler) AddBool(key []byte, value []byte) error { func (h *TestingHandler) AddBool(key []byte, value []byte) error {
k := make([]byte, len(key))
copy(k, key)
v := make([]byte, len(value))
copy(v, value)
fieldkey := Result{ fieldkey := Result{
Name: FieldKey, Name: FieldKey,
Value: key, Value: k,
} }
fieldvalue := Result{ fieldvalue := Result{
Name: FieldBool, Name: FieldBool,
Value: value, Value: v,
} }
h.results = append(h.results, fieldkey, fieldvalue) h.results = append(h.results, fieldkey, fieldvalue)
return nil return nil
} }
func (h *TestingHandler) SetTimestamp(tm []byte) error { func (h *TestingHandler) SetTimestamp(tm []byte) error {
t := make([]byte, len(tm))
copy(t, tm)
timestamp := Result{ timestamp := Result{
Name: Timestamp, Name: Timestamp,
Value: tm, Value: t,
} }
h.results = append(h.results, timestamp) h.results = append(h.results, timestamp)
return nil return nil
@ -1676,63 +1714,64 @@ func TestMachine(t *testing.T) {
} }
} }
var positionTests = []struct {
name string
input []byte
lineno int
column int
}{
{
name: "empty string",
input: []byte(""),
lineno: 1,
column: 1,
},
{
name: "minimal",
input: []byte("cpu value=42"),
lineno: 1,
column: 13,
},
{
name: "one newline",
input: []byte("cpu value=42\ncpu value=42"),
lineno: 2,
column: 13,
},
{
name: "several newlines",
input: []byte("cpu value=42\n\n\n"),
lineno: 4,
column: 1,
},
{
name: "error on second line",
input: []byte("cpu value=42\ncpu value=invalid"),
lineno: 2,
column: 11,
},
{
name: "error after comment line",
input: []byte("cpu value=42\n# comment\ncpu value=invalid"),
lineno: 3,
column: 11,
},
{
name: "dos line endings",
input: []byte("cpu value=42\r\ncpu value=invalid"),
lineno: 2,
column: 11,
},
{
name: "mac line endings not supported",
input: []byte("cpu value=42\rcpu value=invalid"),
lineno: 1,
column: 14,
},
}
func TestMachinePosition(t *testing.T) { func TestMachinePosition(t *testing.T) {
var tests = []struct { for _, tt := range positionTests {
name string
input []byte
lineno int
column int
}{
{
name: "empty string",
input: []byte(""),
lineno: 1,
column: 1,
},
{
name: "minimal",
input: []byte("cpu value=42"),
lineno: 1,
column: 13,
},
{
name: "one newline",
input: []byte("cpu value=42\ncpu value=42"),
lineno: 2,
column: 13,
},
{
name: "several newlines",
input: []byte("cpu value=42\n\n\n"),
lineno: 4,
column: 1,
},
{
name: "error on second line",
input: []byte("cpu value=42\ncpu value=invalid"),
lineno: 2,
column: 11,
},
{
name: "error after comment line",
input: []byte("cpu value=42\n# comment\ncpu value=invalid"),
lineno: 3,
column: 11,
},
{
name: "dos line endings",
input: []byte("cpu value=42\r\ncpu value=invalid"),
lineno: 2,
column: 11,
},
{
name: "mac line endings not supported",
input: []byte("cpu value=42\rcpu value=invalid"),
lineno: 1,
column: 14,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) { t.Run(tt.name, func(t *testing.T) {
handler := &TestingHandler{} handler := &TestingHandler{}
fsm := influx.NewMachine(handler) fsm := influx.NewMachine(handler)
@ -1932,135 +1971,136 @@ func (h *MockHandler) SetTimestamp(tm []byte) error {
return h.SetTimestampF(tm) return h.SetTimestampF(tm)
} }
var errorRecoveryTests = []struct {
name string
input []byte
handler *MockHandler
results []Result
}{
{
name: "integer",
input: []byte("cpu value=43i\ncpu value=42i"),
handler: &MockHandler{
SetMeasurementF: func(name []byte) error {
return nil
},
AddIntF: func(name, value []byte) error {
if string(value) != "42i" {
return errors.New("handler error")
}
return nil
},
},
results: []Result{
{
Name: Measurement,
Value: []byte("cpu"),
},
{
Name: Error,
err: errors.New("handler error"),
},
{
Name: Measurement,
Value: []byte("cpu"),
},
{
Name: FieldKey,
Value: []byte("value"),
},
{
Name: FieldInt,
Value: []byte("42i"),
},
{
Name: Success,
},
},
},
{
name: "integer with timestamp",
input: []byte("cpu value=43i 1516241192000000000\ncpu value=42i"),
handler: &MockHandler{
SetMeasurementF: func(name []byte) error {
return nil
},
AddIntF: func(name, value []byte) error {
if string(value) != "42i" {
return errors.New("handler error")
}
return nil
},
},
results: []Result{
{
Name: Measurement,
Value: []byte("cpu"),
},
{
Name: Error,
err: errors.New("handler error"),
},
{
Name: Measurement,
Value: []byte("cpu"),
},
{
Name: FieldKey,
Value: []byte("value"),
},
{
Name: FieldInt,
Value: []byte("42i"),
},
{
Name: Success,
},
},
},
{
name: "unsigned",
input: []byte("cpu value=43u\ncpu value=42u"),
handler: &MockHandler{
SetMeasurementF: func(name []byte) error {
return nil
},
AddUintF: func(name, value []byte) error {
if string(value) != "42u" {
return errors.New("handler error")
}
return nil
},
},
results: []Result{
{
Name: Measurement,
Value: []byte("cpu"),
},
{
Name: Error,
err: errors.New("handler error"),
},
{
Name: Measurement,
Value: []byte("cpu"),
},
{
Name: FieldKey,
Value: []byte("value"),
},
{
Name: FieldUint,
Value: []byte("42u"),
},
{
Name: Success,
},
},
},
}
func TestHandlerErrorRecovery(t *testing.T) { func TestHandlerErrorRecovery(t *testing.T) {
var tests = []struct { for _, tt := range errorRecoveryTests {
name string
input []byte
handler *MockHandler
results []Result
}{
{
name: "integer",
input: []byte("cpu value=43i\ncpu value=42i"),
handler: &MockHandler{
SetMeasurementF: func(name []byte) error {
return nil
},
AddIntF: func(name, value []byte) error {
if string(value) != "42i" {
return errors.New("handler error")
}
return nil
},
},
results: []Result{
{
Name: Measurement,
Value: []byte("cpu"),
},
{
Name: Error,
err: errors.New("handler error"),
},
{
Name: Measurement,
Value: []byte("cpu"),
},
{
Name: FieldKey,
Value: []byte("value"),
},
{
Name: FieldInt,
Value: []byte("42i"),
},
{
Name: Success,
},
},
},
{
name: "integer with timestamp",
input: []byte("cpu value=43i 1516241192000000000\ncpu value=42i"),
handler: &MockHandler{
SetMeasurementF: func(name []byte) error {
return nil
},
AddIntF: func(name, value []byte) error {
if string(value) != "42i" {
return errors.New("handler error")
}
return nil
},
},
results: []Result{
{
Name: Measurement,
Value: []byte("cpu"),
},
{
Name: Error,
err: errors.New("handler error"),
},
{
Name: Measurement,
Value: []byte("cpu"),
},
{
Name: FieldKey,
Value: []byte("value"),
},
{
Name: FieldInt,
Value: []byte("42i"),
},
{
Name: Success,
},
},
},
{
name: "unsigned",
input: []byte("cpu value=43u\ncpu value=42u"),
handler: &MockHandler{
SetMeasurementF: func(name []byte) error {
return nil
},
AddUintF: func(name, value []byte) error {
if string(value) != "42u" {
return errors.New("handler error")
}
return nil
},
},
results: []Result{
{
Name: Measurement,
Value: []byte("cpu"),
},
{
Name: Error,
err: errors.New("handler error"),
},
{
Name: Measurement,
Value: []byte("cpu"),
},
{
Name: FieldKey,
Value: []byte("value"),
},
{
Name: FieldUint,
Value: []byte("42u"),
},
{
Name: Success,
},
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) { t.Run(tt.name, func(t *testing.T) {
fsm := influx.NewMachine(tt.handler) fsm := influx.NewMachine(tt.handler)
fsm.SetData(tt.input) fsm.SetData(tt.input)
@ -2078,3 +2118,79 @@ func TestHandlerErrorRecovery(t *testing.T) {
}) })
} }
} }
func TestStreamMachine(t *testing.T) {
type testcase struct {
name string
input io.Reader
results []Result
err error
}
var tc []testcase
for _, tt := range tests {
tc = append(tc, testcase{
name: tt.name,
input: bytes.NewBuffer([]byte(tt.input)),
results: tt.results,
err: tt.err,
})
}
for _, tt := range tc {
t.Run(tt.name, func(t *testing.T) {
handler := &TestingHandler{}
fsm := influx.NewStreamMachine(tt.input, handler)
// Parse only up to 20 metrics; to avoid any bugs where the parser
// isn't terminated.
for i := 0; i < 20; i++ {
err := fsm.Next()
if err != nil && err == influx.EOF {
break
}
handler.Result(err)
}
results := handler.Results()
require.Equal(t, tt.results, results)
})
}
}
func TestStreamMachinePosition(t *testing.T) {
type testcase struct {
name string
input io.Reader
lineno int
column int
}
var tc []testcase
for _, tt := range positionTests {
tc = append(tc, testcase{
name: tt.name,
input: bytes.NewBuffer([]byte(tt.input)),
lineno: tt.lineno,
column: tt.column,
})
}
for _, tt := range tc {
t.Run(tt.name, func(t *testing.T) {
handler := &TestingHandler{}
fsm := influx.NewStreamMachine(tt.input, handler)
// Parse until an error or eof
for i := 0; i < 20; i++ {
err := fsm.Next()
if err != nil {
break
}
}
require.Equal(t, tt.lineno, fsm.LineNumber(), "lineno")
require.Equal(t, tt.column, fsm.Column(), "column")
})
}
}

104
plugins/parsers/influx/parser.go
View File

@ -3,8 +3,10 @@ package influx
import ( import (
"errors" "errors"
"fmt" "fmt"
"io"
"strings" "strings"
"sync" "sync"
"time"
"github.com/influxdata/telegraf" "github.com/influxdata/telegraf"
) )
@ -17,6 +19,9 @@ var (
ErrNoMetric = errors.New("no metric in line") ErrNoMetric = errors.New("no metric in line")
) )
type TimeFunc func() time.Time
// ParseError indicates a error in the parsing of the text.
type ParseError struct { type ParseError struct {
Offset int Offset int
LineOffset int LineOffset int
@ -38,6 +43,8 @@ func (e *ParseError) Error() string {
return fmt.Sprintf("metric parse error: %s at %d:%d: %q", e.msg, e.LineNumber, e.Column, buffer) return fmt.Sprintf("metric parse error: %s at %d:%d: %q", e.msg, e.LineNumber, e.Column, buffer)
} }
// Parser is an InfluxDB Line Protocol parser that implements the
// parsers.Parser interface.
type Parser struct { type Parser struct {
DefaultTags map[string]string DefaultTags map[string]string
@ -62,6 +69,10 @@ func NewSeriesParser(handler *MetricHandler) *Parser {
} }
} }
func (h *Parser) SetTimeFunc(f TimeFunc) {
h.handler.SetTimeFunc(f)
}
func (p *Parser) Parse(input []byte) ([]telegraf.Metric, error) { func (p *Parser) Parse(input []byte) ([]telegraf.Metric, error) {
p.Lock() p.Lock()
defer p.Unlock() defer p.Unlock()
@ -75,7 +86,6 @@ func (p *Parser) Parse(input []byte) ([]telegraf.Metric, error) {
} }
if err != nil { if err != nil {
p.handler.Reset()
return nil, &ParseError{ return nil, &ParseError{
Offset: p.machine.Position(), Offset: p.machine.Position(),
LineOffset: p.machine.LineOffset(), LineOffset: p.machine.LineOffset(),
@ -88,7 +98,6 @@ func (p *Parser) Parse(input []byte) ([]telegraf.Metric, error) {
metric, err := p.handler.Metric() metric, err := p.handler.Metric()
if err != nil { if err != nil {
p.handler.Reset()
return nil, err return nil, err
} }
@ -126,10 +135,93 @@ func (p *Parser) applyDefaultTags(metrics []telegraf.Metric) {
} }
for _, m := range metrics { for _, m := range metrics {
for k, v := range p.DefaultTags { p.applyDefaultTagsSingle(m)
if !m.HasTag(k) { }
m.AddTag(k, v) }
}
func (p *Parser) applyDefaultTagsSingle(metric telegraf.Metric) {
for k, v := range p.DefaultTags {
if !metric.HasTag(k) {
metric.AddTag(k, v)
} }
} }
} }
// StreamParser is an InfluxDB Line Protocol parser. It is not safe for
// concurrent use in multiple goroutines.
type StreamParser struct {
machine *streamMachine
handler *MetricHandler
}
func NewStreamParser(r io.Reader) *StreamParser {
handler := NewMetricHandler()
return &StreamParser{
machine: NewStreamMachine(r, handler),
handler: handler,
}
}
// SetTimeFunc changes the function used to determine the time of metrics
// without a timestamp. The default TimeFunc is time.Now. Useful mostly for
// testing, or perhaps if you want all metrics to have the same timestamp.
func (h *StreamParser) SetTimeFunc(f TimeFunc) {
h.handler.SetTimeFunc(f)
}
func (h *StreamParser) SetTimePrecision(u time.Duration) {
h.handler.SetTimePrecision(u)
}
// Next parses the next item from the stream. You can repeat calls to this
// function until it returns EOF.
func (p *StreamParser) Next() (telegraf.Metric, error) {
err := p.machine.Next()
if err == EOF {
return nil, EOF
}
if err != nil {
return nil, &ParseError{
Offset: p.machine.Position(),
LineOffset: p.machine.LineOffset(),
LineNumber: p.machine.LineNumber(),
Column: p.machine.Column(),
msg: err.Error(),
buf: p.machine.LineText(),
}
}
metric, err := p.handler.Metric()
if err != nil {
return nil, err
}
return metric, nil
}
// Position returns the current byte offset into the data.
func (p *StreamParser) Position() int {
return p.machine.Position()
}
// LineOffset returns the byte offset of the current line.
func (p *StreamParser) LineOffset() int {
return p.machine.LineOffset()
}
// LineNumber returns the current line number. Lines are counted based on the
// regular expression `\r?\n`.
func (p *StreamParser) LineNumber() int {
return p.machine.LineNumber()
}
// Column returns the current column.
func (p *StreamParser) Column() int {
return p.machine.Column()
}
// LineText returns the text of the current line that has been parsed so far.
func (p *StreamParser) LineText() string {
return p.machine.LineText()
}

162
plugins/parsers/influx/parser_test.go
View File

@ -1,6 +1,7 @@
package influx package influx
import ( import (
"bytes"
"strconv" "strconv"
"strings" "strings"
"testing" "testing"
@ -8,6 +9,7 @@ import (
"github.com/influxdata/telegraf" "github.com/influxdata/telegraf"
"github.com/influxdata/telegraf/metric" "github.com/influxdata/telegraf/metric"
"github.com/influxdata/telegraf/testutil"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
) )
@ -23,12 +25,11 @@ var DefaultTime = func() time.Time {
} }
var ptests = []struct { var ptests = []struct {
name string name string
input []byte input []byte
timeFunc func() time.Time timeFunc func() time.Time
precision time.Duration metrics []telegraf.Metric
metrics []telegraf.Metric err error
err error
}{ }{
{ {
name: "minimal", name: "minimal",
@ -495,7 +496,7 @@ var ptests = []struct {
err: nil, err: nil,
}, },
{ {
name: "no timestamp full precision", name: "no timestamp",
input: []byte("cpu value=42"), input: []byte("cpu value=42"),
timeFunc: func() time.Time { timeFunc: func() time.Time {
return time.Unix(42, 123456789) return time.Unix(42, 123456789)
@ -514,27 +515,6 @@ var ptests = []struct {
}, },
err: nil, err: nil,
}, },
{
name: "no timestamp partial precision",
input: []byte("cpu value=42"),
timeFunc: func() time.Time {
return time.Unix(42, 123456789)
},
precision: 1 * time.Millisecond,
metrics: []telegraf.Metric{
Metric(
metric.New(
"cpu",
map[string]string{},
map[string]interface{}{
"value": 42.0,
},
time.Unix(42, 123000000),
),
),
},
err: nil,
},
{ {
name: "multiple lines", name: "multiple lines",
input: []byte("cpu value=42\ncpu value=42"), input: []byte("cpu value=42\ncpu value=42"),
@ -651,14 +631,11 @@ func TestParser(t *testing.T) {
for _, tt := range ptests { for _, tt := range ptests {
t.Run(tt.name, func(t *testing.T) { t.Run(tt.name, func(t *testing.T) {
handler := NewMetricHandler() handler := NewMetricHandler()
handler.SetTimeFunc(DefaultTime)
if tt.timeFunc != nil {
handler.SetTimeFunc(tt.timeFunc)
}
if tt.precision > 0 {
handler.SetTimePrecision(tt.precision)
}
parser := NewParser(handler) parser := NewParser(handler)
parser.SetTimeFunc(DefaultTime)
if tt.timeFunc != nil {
parser.SetTimeFunc(tt.timeFunc)
}
metrics, err := parser.Parse(tt.input) metrics, err := parser.Parse(tt.input)
require.Equal(t, tt.err, err) require.Equal(t, tt.err, err)
@ -688,14 +665,41 @@ func BenchmarkParser(b *testing.B) {
} }
} }
func TestStreamParser(t *testing.T) {
for _, tt := range ptests {
t.Run(tt.name, func(t *testing.T) {
r := bytes.NewBuffer(tt.input)
parser := NewStreamParser(r)
parser.SetTimeFunc(DefaultTime)
if tt.timeFunc != nil {
parser.SetTimeFunc(tt.timeFunc)
}
var i int
for {
m, err := parser.Next()
if err != nil {
if err == EOF {
break
}
require.Equal(t, tt.err, err)
break
}
testutil.RequireMetricEqual(t, tt.metrics[i], m)
i++
}
})
}
}
func TestSeriesParser(t *testing.T) { func TestSeriesParser(t *testing.T) {
var tests = []struct { var tests = []struct {
name string name string
input []byte input []byte
timeFunc func() time.Time timeFunc func() time.Time
precision time.Duration metrics []telegraf.Metric
metrics []telegraf.Metric err error
err error
}{ }{
{ {
name: "empty", name: "empty",
@ -749,14 +753,10 @@ func TestSeriesParser(t *testing.T) {
for _, tt := range tests { for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) { t.Run(tt.name, func(t *testing.T) {
handler := NewMetricHandler() handler := NewMetricHandler()
handler.SetTimeFunc(DefaultTime)
if tt.timeFunc != nil {
handler.SetTimeFunc(tt.timeFunc)
}
if tt.precision > 0 {
handler.SetTimePrecision(tt.precision)
}
parser := NewSeriesParser(handler) parser := NewSeriesParser(handler)
if tt.timeFunc != nil {
parser.SetTimeFunc(tt.timeFunc)
}
metrics, err := parser.Parse(tt.input) metrics, err := parser.Parse(tt.input)
require.Equal(t, tt.err, err) require.Equal(t, tt.err, err)
@ -791,6 +791,11 @@ func TestParserErrorString(t *testing.T) {
input: []byte("cpu " + strings.Repeat("ab", maxErrorBufferSize) + "=invalid\ncpu value=42"), input: []byte("cpu " + strings.Repeat("ab", maxErrorBufferSize) + "=invalid\ncpu value=42"),
errString: "metric parse error: expected field at 1:2054: \"cpu " + strings.Repeat("ab", maxErrorBufferSize)[:maxErrorBufferSize-4] + "...\"", errString: "metric parse error: expected field at 1:2054: \"cpu " + strings.Repeat("ab", maxErrorBufferSize)[:maxErrorBufferSize-4] + "...\"",
}, },
{
name: "multiple line error",
input: []byte("cpu value=42\ncpu value=invalid\ncpu value=42\ncpu value=invalid"),
errString: `metric parse error: expected field at 2:11: "cpu value=invalid"`,
},
} }
for _, tt := range ptests { for _, tt := range ptests {
@ -803,3 +808,64 @@ func TestParserErrorString(t *testing.T) {
}) })
} }
} }
func TestStreamParserErrorString(t *testing.T) {
var ptests = []struct {
name string
input []byte
errs []string
}{
{
name: "multiple line error",
input: []byte("cpu value=42\ncpu value=invalid\ncpu value=42"),
errs: []string{
`metric parse error: expected field at 2:11: "cpu value="`,
},
},
{
name: "handler error",
input: []byte("cpu value=9223372036854775808i\ncpu value=42"),
errs: []string{
`metric parse error: value out of range at 1:31: "cpu value=9223372036854775808i"`,
},
},
{
name: "buffer too long",
input: []byte("cpu " + strings.Repeat("ab", maxErrorBufferSize) + "=invalid\ncpu value=42"),
errs: []string{
"metric parse error: expected field at 1:2054: \"cpu " + strings.Repeat("ab", maxErrorBufferSize)[:maxErrorBufferSize-4] + "...\"",
},
},
{
name: "multiple errors",
input: []byte("foo value=1asdf2.0\nfoo value=2.0\nfoo value=3asdf2.0\nfoo value=4.0"),
errs: []string{
`metric parse error: expected field at 1:12: "foo value=1"`,
`metric parse error: expected field at 3:12: "foo value=3"`,
},
},
}
for _, tt := range ptests {
t.Run(tt.name, func(t *testing.T) {
parser := NewStreamParser(bytes.NewBuffer(tt.input))
var errs []error
for i := 0; i < 20; i++ {
_, err := parser.Next()
if err == EOF {
break
}
if err != nil {
errs = append(errs, err)
}
}
require.Equal(t, len(tt.errs), len(errs))
for i, err := range errs {
require.Equal(t, tt.errs[i], err.Error())
}
})
}
}