Add line protocol uint64 support (#3946)

This commit is contained in:
Matt 2018-03-28 19:43:25 -04:00 committed by Daniel Nelson
parent 741abbf590
commit 006ccbf05b
12 changed files with 8046 additions and 6754 deletions

View File

@ -4,6 +4,7 @@ BRANCH := $(shell git rev-parse --abbrev-ref HEAD)
COMMIT := $(shell git rev-parse --short HEAD)
GOFILES ?= $(shell git ls-files '*.go')
GOFMT ?= $(shell gofmt -l $(filter-out plugins/parsers/influx/machine.go, $(GOFILES)))
BUILDFLAGS ?=
ifdef GOBIN
PATH := $(GOBIN):$(PATH)
@ -35,7 +36,7 @@ deps:
gdm restore
telegraf:
go build -i -o $(TELEGRAF) -ldflags "$(LDFLAGS)" ./cmd/telegraf/telegraf.go
go build -i -o $(TELEGRAF) -ldflags "$(LDFLAGS)" $(BUILDFLAGS) ./cmd/telegraf/telegraf.go
go-install:
go install -ldflags "-w -s $(LDFLAGS)" ./cmd/telegraf
@ -61,6 +62,9 @@ fmtcheck:
fi
@echo '[INFO] done.'
uint64:
BUILDFLAGS="-tags uint64" $(MAKE) all
lint:
golint ./...
@ -99,4 +103,4 @@ docker-image:
plugins/parsers/influx/machine.go: plugins/parsers/influx/machine.go.rl
ragel -Z -G2 $^ -o $@
.PHONY: deps telegraf install test test-windows lint vet test-all package clean docker-image fmtcheck
.PHONY: deps telegraf install test test-windows lint vet test-all package clean docker-image fmtcheck uint64

View File

@ -11,6 +11,16 @@ import (
const MaxInt = int(^uint(0) >> 1)
// enableUint64Support will enable uint64 support if set to true.
var enableUint64Support = false
// EnableUintSupport manually enables uint support for convertValue.
// This function will be removed in the future and only exists for unit tests during the
// transition.
func EnableUintSupport() {
enableUint64Support = true
}
type metric struct {
name string
tags []*telegraf.Tag
@ -265,11 +275,14 @@ func convertField(v interface{}) interface{} {
return int64(MaxInt)
}
case uint64:
if v <= uint64(MaxInt) {
return int64(v)
} else {
return int64(MaxInt)
if enableUint64Support == false {
if v <= uint64(MaxInt) {
return int64(v)
} else {
return int64(MaxInt)
}
}
return uint64(v)
case []byte:
return string(v)
case int32:

7
metric/uint_support.go Normal file
View File

@ -0,0 +1,7 @@
// +build uint64
package metric
func init() {
EnableUintSupport()
}

View File

@ -63,6 +63,12 @@ func parseIntBytes(b []byte, base int, bitSize int) (i int64, err error) {
return strconv.ParseInt(s, base, bitSize)
}
// parseUintBytes is a zero-alloc wrapper around strconv.ParseUint.
func parseUintBytes(b []byte, base int, bitSize int) (i uint64, err error) {
s := unsafeBytesToString(b)
return strconv.ParseUint(s, base, bitSize)
}
// parseFloatBytes is a zero-alloc wrapper around strconv.ParseFloat.
func parseFloatBytes(b []byte, bitSize int) (float64, error) {
s := unsafeBytesToString(b)

View File

@ -54,6 +54,16 @@ func (h *MetricHandler) AddInt(key []byte, value []byte) {
h.builder.AddField(fk, fv)
}
func (h *MetricHandler) AddUint(key []byte, value []byte) {
fk := unescape(key)
fv, err := parseUintBytes(bytes.TrimSuffix(value, []byte("u")), 10, 64)
if err != nil {
log.Errorf("E! Received unparseable uint value: %q", value)
return
}
h.builder.AddField(fk, fv)
}
func (h *MetricHandler) AddFloat(key []byte, value []byte) {
fk := unescape(key)
fv, err := parseFloatBytes(value, 64)

File diff suppressed because it is too large Load Diff

View File

@ -89,6 +89,10 @@ action integer {
m.handler.AddInt(key, m.text())
}
action unsigned {
m.handler.AddUint(key, m.text())
}
action float {
m.handler.AddFloat(key, m.text())
}
@ -114,6 +118,9 @@ non_zero_digit =
integer =
'-'? ( digit | ( non_zero_digit digit* ) );
unsigned =
( digit | ( non_zero_digit digit* ) );
number =
( integer ( '.' digit* )? ) | ( '.' digit* );
@ -135,6 +142,9 @@ fieldfloat =
fieldinteger =
(integer 'i') >begin %integer;
fieldunsigned =
(unsigned 'u') >begin %unsigned;
false =
"false" | "FALSE" | "False" | "F" | "f";
@ -153,7 +163,7 @@ fieldstring =
fieldstringquoted =
'"' fieldstring '"';
fieldvalue = fieldinteger | fieldfloat | fieldstringquoted | fieldbool;
fieldvalue = fieldinteger | fieldunsigned | fieldfloat | fieldstringquoted | fieldbool;
field =
fieldkey '=' fieldvalue;

View File

@ -43,6 +43,18 @@ func (h *TestingHandler) AddInt(key []byte, value []byte) {
h.results = append(h.results, fieldkey, fieldvalue)
}
func (h *TestingHandler) AddUint(key []byte, value []byte) {
fieldkey := Result{
Name: FieldKey,
Value: key,
}
fieldvalue := Result{
Name: FieldUint,
Value: value,
}
h.results = append(h.results, fieldkey, fieldvalue)
}
func (h *TestingHandler) AddFloat(key []byte, value []byte) {
fieldkey := Result{
Name: FieldKey,
@ -113,6 +125,9 @@ func (h *BenchmarkingHandler) AddTag(key []byte, value []byte) {
func (h *BenchmarkingHandler) AddInt(key []byte, value []byte) {
}
func (h *BenchmarkingHandler) AddUint(key []byte, value []byte) {
}
func (h *BenchmarkingHandler) AddFloat(key []byte, value []byte) {
}

View File

@ -19,6 +19,7 @@ type Handler interface {
SetMeasurement(name []byte)
AddTag(key []byte, value []byte)
AddInt(key []byte, value []byte)
AddUint(key []byte, value []byte)
AddFloat(key []byte, value []byte)
AddString(key []byte, value []byte)
AddBool(key []byte, value []byte)

View File

@ -16,6 +16,12 @@ func Metric(v telegraf.Metric, err error) telegraf.Metric {
return v
}
const (
Uint64Overflow uint64 = 9223372036854775808
Uint64Max uint64 = 18446744073709551615
Uint64Test uint64 = 42
)
var DefaultTime = func() time.Time {
return time.Unix(42, 0)
}
@ -256,6 +262,57 @@ var ptests = []struct {
},
err: nil,
},
{
name: "field uint",
input: []byte("cpu value=42u"),
metrics: []telegraf.Metric{
Metric(
metric.New(
"cpu",
map[string]string{},
map[string]interface{}{
"value": Uint64Test,
},
time.Unix(42, 0),
),
),
},
err: nil,
},
{
name: "field uint int overflow",
input: []byte("cpu value=9223372036854775808u"),
metrics: []telegraf.Metric{
Metric(
metric.New(
"cpu",
map[string]string{},
map[string]interface{}{
"value": Uint64Overflow,
},
time.Unix(42, 0),
),
),
},
err: nil,
},
{
name: "field uint maximum",
input: []byte("cpu value=18446744073709551615u"),
metrics: []telegraf.Metric{
Metric(
metric.New(
"cpu",
map[string]string{},
map[string]interface{}{
"value": Uint64Max,
},
time.Unix(42, 0),
),
),
},
err: nil,
},
{
name: "field boolean",
input: []byte("cpu value=true"),

View File

@ -237,6 +237,8 @@ func (s *Serializer) writeMetric(w io.Writer, m telegraf.Metric) error {
func appendFieldValue(buf []byte, value interface{}) ([]byte, error) {
switch v := value.(type) {
case uint64:
return appendUintField(buf, v), nil
case int64:
return appendIntField(buf, v), nil
case float64:
@ -257,6 +259,10 @@ func appendFieldValue(buf []byte, value interface{}) ([]byte, error) {
return buf, ErrInvalidFieldType
}
func appendUintField(buf []byte, value uint64) []byte {
return append(strconv.AppendUint(buf, value, 10), 'u')
}
func appendIntField(buf []byte, value int64) []byte {
return append(strconv.AppendInt(buf, value, 10), 'i')
}

View File

@ -11,12 +11,20 @@ import (
)
func MustMetric(v telegraf.Metric, err error) telegraf.Metric {
// Force uint support to be enabled for testing.
metric.EnableUintSupport()
if err != nil {
panic(err)
}
return v
}
const (
Uint64Overflow uint64 = 9223372036854775808
Uint64Max uint64 = 18446744073709551615
Uint64Test uint64 = 42
)
var tests = []struct {
name string
maxBytes int
@ -128,6 +136,48 @@ var tests = []struct {
),
output: []byte("cpu value=42i 0\n"),
},
{
name: "uint field",
input: MustMetric(
metric.New(
"cpu",
map[string]string{},
map[string]interface{}{
"value": Uint64Test,
},
time.Unix(0, 0),
),
),
output: []byte("cpu value=42u 0\n"),
},
{
name: "uint field int64 overflow",
input: MustMetric(
metric.New(
"cpu",
map[string]string{},
map[string]interface{}{
"value": Uint64Overflow,
},
time.Unix(0, 0),
),
),
output: []byte("cpu value=9223372036854775808u 0\n"),
},
{
name: "uint field uint64 max",
input: MustMetric(
metric.New(
"cpu",
map[string]string{},
map[string]interface{}{
"value": Uint64Max,
},
time.Unix(0, 0),
),
),
output: []byte("cpu value=18446744073709551615u 0\n"),
},
{
name: "bool field",
input: MustMetric(