Add support for datadog events to statsd input (#5791)

This commit is contained in:
Jorge Landivar
2019-05-14 18:20:35 -05:00
committed by Daniel Nelson
parent 9318d47a38
commit 8f3ed45797
7 changed files with 852 additions and 177 deletions

View File

@@ -1,8 +1,6 @@
package statsd
import (
"bytes"
"errors"
"fmt"
"net"
"testing"
@@ -17,8 +15,8 @@ const (
testMsg = "test.tcp.msg:100|c"
)
func newTestTcpListener() (*Statsd, chan *bytes.Buffer) {
in := make(chan *bytes.Buffer, 1500)
func newTestTCPListener() (*Statsd, chan input) {
in := make(chan input, 1500)
listener := &Statsd{
Protocol: "tcp",
ServiceAddress: "localhost:8125",
@@ -35,7 +33,7 @@ func NewTestStatsd() *Statsd {
// Make data structures
s.done = make(chan struct{})
s.in = make(chan *bytes.Buffer, s.AllowedPendingMessages)
s.in = make(chan input, s.AllowedPendingMessages)
s.gauges = make(map[string]cachedgauge)
s.counters = make(map[string]cachedcounter)
s.sets = make(map[string]cachedset)
@@ -189,7 +187,7 @@ func BenchmarkTCP(b *testing.B) {
// Valid lines should be parsed and their values should be cached
func TestParse_ValidLines(t *testing.T) {
s := NewTestStatsd()
valid_lines := []string{
validLines := []string{
"valid:45|c",
"valid:45|s",
"valid:45|g",
@@ -197,7 +195,7 @@ func TestParse_ValidLines(t *testing.T) {
"valid.timer:45|h",
}
for _, line := range valid_lines {
for _, line := range validLines {
err := s.parseStatsdLine(line)
if err != nil {
t.Errorf("Parsing line %s should not have resulted in an error\n", line)
@@ -210,7 +208,7 @@ func TestParse_Gauges(t *testing.T) {
s := NewTestStatsd()
// Test that gauge +- values work
valid_lines := []string{
validLines := []string{
"plus.minus:100|g",
"plus.minus:-10|g",
"plus.minus:+30|g",
@@ -228,7 +226,7 @@ func TestParse_Gauges(t *testing.T) {
"scientific.notation.minus:4.7E-5|g",
}
for _, line := range valid_lines {
for _, line := range validLines {
err := s.parseStatsdLine(line)
if err != nil {
t.Errorf("Parsing line %s should not have resulted in an error\n", line)
@@ -274,7 +272,7 @@ func TestParse_Gauges(t *testing.T) {
}
for _, test := range validations {
err := test_validate_gauge(test.name, test.value, s.gauges)
err := testValidateGauge(test.name, test.value, s.gauges)
if err != nil {
t.Error(err.Error())
}
@@ -286,7 +284,7 @@ func TestParse_Sets(t *testing.T) {
s := NewTestStatsd()
// Test that sets work
valid_lines := []string{
validLines := []string{
"unique.user.ids:100|s",
"unique.user.ids:100|s",
"unique.user.ids:100|s",
@@ -306,7 +304,7 @@ func TestParse_Sets(t *testing.T) {
"string.sets:bar|s",
}
for _, line := range valid_lines {
for _, line := range validLines {
err := s.parseStatsdLine(line)
if err != nil {
t.Errorf("Parsing line %s should not have resulted in an error\n", line)
@@ -336,7 +334,7 @@ func TestParse_Sets(t *testing.T) {
}
for _, test := range validations {
err := test_validate_set(test.name, test.value, s.sets)
err := testValidateSet(test.name, test.value, s.sets)
if err != nil {
t.Error(err.Error())
}
@@ -348,7 +346,7 @@ func TestParse_Counters(t *testing.T) {
s := NewTestStatsd()
// Test that counters work
valid_lines := []string{
validLines := []string{
"small.inc:1|c",
"big.inc:100|c",
"big.inc:1|c",
@@ -363,7 +361,7 @@ func TestParse_Counters(t *testing.T) {
"negative.test:-5|c",
}
for _, line := range valid_lines {
for _, line := range validLines {
err := s.parseStatsdLine(line)
if err != nil {
t.Errorf("Parsing line %s should not have resulted in an error\n", line)
@@ -401,7 +399,7 @@ func TestParse_Counters(t *testing.T) {
}
for _, test := range validations {
err := test_validate_counter(test.name, test.value, s.counters)
err := testValidateCounter(test.name, test.value, s.counters)
if err != nil {
t.Error(err.Error())
}
@@ -415,7 +413,7 @@ func TestParse_Timings(t *testing.T) {
acc := &testutil.Accumulator{}
// Test that counters work
valid_lines := []string{
validLines := []string{
"test.timing:1|ms",
"test.timing:11|ms",
"test.timing:1|ms",
@@ -423,7 +421,7 @@ func TestParse_Timings(t *testing.T) {
"test.timing:1|ms",
}
for _, line := range valid_lines {
for _, line := range validLines {
err := s.parseStatsdLine(line)
if err != nil {
t.Errorf("Parsing line %s should not have resulted in an error\n", line)
@@ -464,7 +462,7 @@ func TestParseScientificNotation(t *testing.T) {
// Invalid lines should return an error
func TestParse_InvalidLines(t *testing.T) {
s := NewTestStatsd()
invalid_lines := []string{
invalidLines := []string{
"i.dont.have.a.pipe:45g",
"i.dont.have.a.colon45|c",
"invalid.metric.type:45|e",
@@ -475,7 +473,7 @@ func TestParse_InvalidLines(t *testing.T) {
"invalid.value:d11|c",
"invalid.value:1d1|c",
}
for _, line := range invalid_lines {
for _, line := range invalidLines {
err := s.parseStatsdLine(line)
if err == nil {
t.Errorf("Parsing line %s should have resulted in an error\n", line)
@@ -486,21 +484,21 @@ func TestParse_InvalidLines(t *testing.T) {
// Invalid sample rates should be ignored and not applied
func TestParse_InvalidSampleRate(t *testing.T) {
s := NewTestStatsd()
invalid_lines := []string{
invalidLines := []string{
"invalid.sample.rate:45|c|0.1",
"invalid.sample.rate.2:45|c|@foo",
"invalid.sample.rate:45|g|@0.1",
"invalid.sample.rate:45|s|@0.1",
}
for _, line := range invalid_lines {
for _, line := range invalidLines {
err := s.parseStatsdLine(line)
if err != nil {
t.Errorf("Parsing line %s should not have resulted in an error\n", line)
}
}
counter_validations := []struct {
counterValidations := []struct {
name string
value int64
cache map[string]cachedcounter
@@ -517,19 +515,19 @@ func TestParse_InvalidSampleRate(t *testing.T) {
},
}
for _, test := range counter_validations {
err := test_validate_counter(test.name, test.value, test.cache)
for _, test := range counterValidations {
err := testValidateCounter(test.name, test.value, test.cache)
if err != nil {
t.Error(err.Error())
}
}
err := test_validate_gauge("invalid_sample_rate", 45, s.gauges)
err := testValidateGauge("invalid_sample_rate", 45, s.gauges)
if err != nil {
t.Error(err.Error())
}
err = test_validate_set("invalid_sample_rate", 1, s.sets)
err = testValidateSet("invalid_sample_rate", 1, s.sets)
if err != nil {
t.Error(err.Error())
}
@@ -538,12 +536,12 @@ func TestParse_InvalidSampleRate(t *testing.T) {
// Names should be parsed like . -> _
func TestParse_DefaultNameParsing(t *testing.T) {
s := NewTestStatsd()
valid_lines := []string{
validLines := []string{
"valid:1|c",
"valid.foo-bar:11|c",
}
for _, line := range valid_lines {
for _, line := range validLines {
err := s.parseStatsdLine(line)
if err != nil {
t.Errorf("Parsing line %s should not have resulted in an error\n", line)
@@ -565,7 +563,7 @@ func TestParse_DefaultNameParsing(t *testing.T) {
}
for _, test := range validations {
err := test_validate_counter(test.name, test.value, s.counters)
err := testValidateCounter(test.name, test.value, s.counters)
if err != nil {
t.Error(err.Error())
}
@@ -607,7 +605,7 @@ func TestParse_Template(t *testing.T) {
// Validate counters
for _, test := range validations {
err := test_validate_counter(test.name, test.value, s.counters)
err := testValidateCounter(test.name, test.value, s.counters)
if err != nil {
t.Error(err.Error())
}
@@ -649,7 +647,7 @@ func TestParse_TemplateFilter(t *testing.T) {
// Validate counters
for _, test := range validations {
err := test_validate_counter(test.name, test.value, s.counters)
err := testValidateCounter(test.name, test.value, s.counters)
if err != nil {
t.Error(err.Error())
}
@@ -687,7 +685,7 @@ func TestParse_TemplateSpecificity(t *testing.T) {
// Validate counters
for _, test := range validations {
err := test_validate_counter(test.name, test.value, s.counters)
err := testValidateCounter(test.name, test.value, s.counters)
if err != nil {
t.Error(err.Error())
}
@@ -723,7 +721,7 @@ func TestParse_TemplateFields(t *testing.T) {
}
}
counter_tests := []struct {
counterTests := []struct {
name string
value int64
field string
@@ -745,14 +743,14 @@ func TestParse_TemplateFields(t *testing.T) {
},
}
// Validate counters
for _, test := range counter_tests {
err := test_validate_counter(test.name, test.value, s.counters, test.field)
for _, test := range counterTests {
err := testValidateCounter(test.name, test.value, s.counters, test.field)
if err != nil {
t.Error(err.Error())
}
}
gauge_tests := []struct {
gaugeTests := []struct {
name string
value float64
field string
@@ -769,14 +767,14 @@ func TestParse_TemplateFields(t *testing.T) {
},
}
// Validate gauges
for _, test := range gauge_tests {
err := test_validate_gauge(test.name, test.value, s.gauges, test.field)
for _, test := range gaugeTests {
err := testValidateGauge(test.name, test.value, s.gauges, test.field)
if err != nil {
t.Error(err.Error())
}
}
set_tests := []struct {
setTests := []struct {
name string
value int64
field string
@@ -793,8 +791,8 @@ func TestParse_TemplateFields(t *testing.T) {
},
}
// Validate sets
for _, test := range set_tests {
err := test_validate_set(test.name, test.value, s.sets, test.field)
for _, test := range setTests {
err := testValidateSet(test.name, test.value, s.sets, test.field)
if err != nil {
t.Error(err.Error())
}
@@ -864,7 +862,7 @@ func TestParse_Tags(t *testing.T) {
// Test that DataDog tags are parsed
func TestParse_DataDogTags(t *testing.T) {
s := NewTestStatsd()
s.ParseDataDogTags = true
s.DataDogExtensions = true
lines := []string{
"my_counter:1|c|#host:localhost,environment:prod,endpoint:/:tenant?/oauth/ro",
@@ -873,24 +871,28 @@ func TestParse_DataDogTags(t *testing.T) {
"my_timer:3|ms|@0.1|#live,host:localhost",
}
testTags := map[string]map[string]string{
expectedTags := map[string]map[string]string{
"my_counter": {
"host": "localhost",
"environment": "prod",
"endpoint": "/:tenant?/oauth/ro",
"metric_type": "counter",
},
"my_gauge": {
"live": "",
"live": "true",
"metric_type": "gauge",
},
"my_set": {
"host": "localhost",
"host": "localhost",
"metric_type": "set",
},
"my_timer": {
"live": "",
"host": "localhost",
"live": "true",
"host": "localhost",
"metric_type": "timing",
},
}
@@ -901,18 +903,16 @@ func TestParse_DataDogTags(t *testing.T) {
}
}
sourceTags := map[string]map[string]string{
actualTags := map[string]map[string]string{
"my_gauge": tagsForItem(s.gauges),
"my_counter": tagsForItem(s.counters),
"my_set": tagsForItem(s.sets),
"my_timer": tagsForItem(s.timings),
}
for statName, tags := range testTags {
for k, v := range tags {
otherValue := sourceTags[statName][k]
if sourceTags[statName][k] != v {
t.Errorf("Error with %s, tag %s: %s != %s", statName, k, v, otherValue)
for name, tags := range expectedTags {
for expectedK, expectedV := range tags {
if expectedV != actualTags[name][expectedK] {
t.Errorf("failed: expected: %#v != %#v", tags, actualTags[name])
}
}
}
@@ -945,8 +945,8 @@ func TestParseName(t *testing.T) {
s := NewTestStatsd()
tests := []struct {
in_name string
out_name string
inName string
outName string
}{
{
"foobar",
@@ -963,9 +963,9 @@ func TestParseName(t *testing.T) {
}
for _, test := range tests {
name, _, _ := s.parseName(test.in_name)
if name != test.out_name {
t.Errorf("Expected: %s, got %s", test.out_name, name)
name, _, _ := s.parseName(test.inName)
if name != test.outName {
t.Errorf("Expected: %s, got %s", test.outName, name)
}
}
@@ -973,8 +973,8 @@ func TestParseName(t *testing.T) {
s.MetricSeparator = "."
tests = []struct {
in_name string
out_name string
inName string
outName string
}{
{
"foobar",
@@ -991,9 +991,9 @@ func TestParseName(t *testing.T) {
}
for _, test := range tests {
name, _, _ := s.parseName(test.in_name)
if name != test.out_name {
t.Errorf("Expected: %s, got %s", test.out_name, name)
name, _, _ := s.parseName(test.inName)
if name != test.outName {
t.Errorf("Expected: %s, got %s", test.outName, name)
}
}
}
@@ -1004,12 +1004,12 @@ func TestParse_MeasurementsWithSameName(t *testing.T) {
s := NewTestStatsd()
// Test that counters work
valid_lines := []string{
validLines := []string{
"test.counter,host=localhost:1|c",
"test.counter,host=localhost,region=west:1|c",
}
for _, line := range valid_lines {
for _, line := range validLines {
err := s.parseStatsdLine(line)
if err != nil {
t.Errorf("Parsing line %s should not have resulted in an error\n", line)
@@ -1024,7 +1024,7 @@ func TestParse_MeasurementsWithSameName(t *testing.T) {
// Test that measurements with multiple bits, are treated as different outputs
// but are equal to their single-measurement representation
func TestParse_MeasurementsWithMultipleValues(t *testing.T) {
single_lines := []string{
singleLines := []string{
"valid.multiple:0|ms|@0.1",
"valid.multiple:0|ms|",
"valid.multiple:1|ms",
@@ -1050,7 +1050,7 @@ func TestParse_MeasurementsWithMultipleValues(t *testing.T) {
"valid.multiple.mixed:1|g",
}
multiple_lines := []string{
multipleLines := []string{
"valid.multiple:0|ms|@0.1:0|ms|:1|ms",
"valid.multiple.duplicate:1|c:1|c:2|c:1|c",
"valid.multiple.duplicate:1|h:1|h:2|h:1|h",
@@ -1059,28 +1059,28 @@ func TestParse_MeasurementsWithMultipleValues(t *testing.T) {
"valid.multiple.mixed:1|c:1|ms:2|s:1|g",
}
s_single := NewTestStatsd()
s_multiple := NewTestStatsd()
sSingle := NewTestStatsd()
sMultiple := NewTestStatsd()
for _, line := range single_lines {
err := s_single.parseStatsdLine(line)
for _, line := range singleLines {
err := sSingle.parseStatsdLine(line)
if err != nil {
t.Errorf("Parsing line %s should not have resulted in an error\n", line)
}
}
for _, line := range multiple_lines {
err := s_multiple.parseStatsdLine(line)
for _, line := range multipleLines {
err := sMultiple.parseStatsdLine(line)
if err != nil {
t.Errorf("Parsing line %s should not have resulted in an error\n", line)
}
}
if len(s_single.timings) != 3 {
t.Errorf("Expected 3 measurement, found %d", len(s_single.timings))
if len(sSingle.timings) != 3 {
t.Errorf("Expected 3 measurement, found %d", len(sSingle.timings))
}
if cachedtiming, ok := s_single.timings["metric_type=timingvalid_multiple"]; !ok {
if cachedtiming, ok := sSingle.timings["metric_type=timingvalid_multiple"]; !ok {
t.Errorf("Expected cached measurement with hash 'metric_type=timingvalid_multiple' not found")
} else {
if cachedtiming.name != "valid_multiple" {
@@ -1100,60 +1100,60 @@ func TestParse_MeasurementsWithMultipleValues(t *testing.T) {
}
}
// test if s_single and s_multiple did compute the same stats for valid.multiple.duplicate
if err := test_validate_set("valid_multiple_duplicate", 2, s_single.sets); err != nil {
// test if sSingle and sMultiple did compute the same stats for valid.multiple.duplicate
if err := testValidateSet("valid_multiple_duplicate", 2, sSingle.sets); err != nil {
t.Error(err.Error())
}
if err := test_validate_set("valid_multiple_duplicate", 2, s_multiple.sets); err != nil {
if err := testValidateSet("valid_multiple_duplicate", 2, sMultiple.sets); err != nil {
t.Error(err.Error())
}
if err := test_validate_counter("valid_multiple_duplicate", 5, s_single.counters); err != nil {
if err := testValidateCounter("valid_multiple_duplicate", 5, sSingle.counters); err != nil {
t.Error(err.Error())
}
if err := test_validate_counter("valid_multiple_duplicate", 5, s_multiple.counters); err != nil {
if err := testValidateCounter("valid_multiple_duplicate", 5, sMultiple.counters); err != nil {
t.Error(err.Error())
}
if err := test_validate_gauge("valid_multiple_duplicate", 1, s_single.gauges); err != nil {
if err := testValidateGauge("valid_multiple_duplicate", 1, sSingle.gauges); err != nil {
t.Error(err.Error())
}
if err := test_validate_gauge("valid_multiple_duplicate", 1, s_multiple.gauges); err != nil {
if err := testValidateGauge("valid_multiple_duplicate", 1, sMultiple.gauges); err != nil {
t.Error(err.Error())
}
// test if s_single and s_multiple did compute the same stats for valid.multiple.mixed
if err := test_validate_set("valid_multiple_mixed", 1, s_single.sets); err != nil {
// test if sSingle and sMultiple did compute the same stats for valid.multiple.mixed
if err := testValidateSet("valid_multiple_mixed", 1, sSingle.sets); err != nil {
t.Error(err.Error())
}
if err := test_validate_set("valid_multiple_mixed", 1, s_multiple.sets); err != nil {
if err := testValidateSet("valid_multiple_mixed", 1, sMultiple.sets); err != nil {
t.Error(err.Error())
}
if err := test_validate_counter("valid_multiple_mixed", 1, s_single.counters); err != nil {
if err := testValidateCounter("valid_multiple_mixed", 1, sSingle.counters); err != nil {
t.Error(err.Error())
}
if err := test_validate_counter("valid_multiple_mixed", 1, s_multiple.counters); err != nil {
if err := testValidateCounter("valid_multiple_mixed", 1, sMultiple.counters); err != nil {
t.Error(err.Error())
}
if err := test_validate_gauge("valid_multiple_mixed", 1, s_single.gauges); err != nil {
if err := testValidateGauge("valid_multiple_mixed", 1, sSingle.gauges); err != nil {
t.Error(err.Error())
}
if err := test_validate_gauge("valid_multiple_mixed", 1, s_multiple.gauges); err != nil {
if err := testValidateGauge("valid_multiple_mixed", 1, sMultiple.gauges); err != nil {
t.Error(err.Error())
}
}
// Tests low-level functionality of timings when multiple fields is enabled
// and a measurement template has been defined which can parse field names
func TestParse_Timings_MultipleFieldsWithTemplate(t *testing.T) {
func TestParse_TimingsMultipleFieldsWithTemplate(t *testing.T) {
s := NewTestStatsd()
s.Templates = []string{"measurement.field"}
s.Percentiles = []int{90}
@@ -1204,7 +1204,7 @@ func TestParse_Timings_MultipleFieldsWithTemplate(t *testing.T) {
// Tests low-level functionality of timings when multiple fields is enabled
// but a measurement template hasn't been defined so we can't parse field names
// In this case the behaviour should be the same as normal behaviour
func TestParse_Timings_MultipleFieldsWithoutTemplate(t *testing.T) {
func TestParse_TimingsMultipleFieldsWithoutTemplate(t *testing.T) {
s := NewTestStatsd()
s.Templates = []string{}
s.Percentiles = []int{90}
@@ -1420,14 +1420,14 @@ func TestParse_Gauges_Delete(t *testing.T) {
t.Errorf("Parsing line %s should not have resulted in an error\n", line)
}
err = test_validate_gauge("current_users", 100, s.gauges)
err = testValidateGauge("current_users", 100, s.gauges)
if err != nil {
t.Error(err.Error())
}
s.Gather(fakeacc)
err = test_validate_gauge("current_users", 100, s.gauges)
err = testValidateGauge("current_users", 100, s.gauges)
if err == nil {
t.Error("current_users_gauge metric should have been deleted")
}
@@ -1446,14 +1446,14 @@ func TestParse_Sets_Delete(t *testing.T) {
t.Errorf("Parsing line %s should not have resulted in an error\n", line)
}
err = test_validate_set("unique_user_ids", 1, s.sets)
err = testValidateSet("unique_user_ids", 1, s.sets)
if err != nil {
t.Error(err.Error())
}
s.Gather(fakeacc)
err = test_validate_set("unique_user_ids", 1, s.sets)
err = testValidateSet("unique_user_ids", 1, s.sets)
if err == nil {
t.Error("unique_user_ids_set metric should have been deleted")
}
@@ -1472,14 +1472,14 @@ func TestParse_Counters_Delete(t *testing.T) {
t.Errorf("Parsing line %s should not have resulted in an error\n", line)
}
err = test_validate_counter("total_users", 100, s.counters)
err = testValidateCounter("total_users", 100, s.counters)
if err != nil {
t.Error(err.Error())
}
s.Gather(fakeacc)
err = test_validate_counter("total_users", 100, s.counters)
err = testValidateCounter("total_users", 100, s.counters)
if err == nil {
t.Error("total_users_counter metric should have been deleted")
}
@@ -1504,8 +1504,7 @@ func TestParseKeyValue(t *testing.T) {
}
// Test utility functions
func test_validate_set(
func testValidateSet(
name string,
value int64,
cache map[string]cachedset,
@@ -1527,17 +1526,16 @@ func test_validate_set(
}
}
if !found {
return errors.New(fmt.Sprintf("Test Error: Metric name %s not found\n", name))
return fmt.Errorf("test Error: Metric name %s not found", name)
}
if value != int64(len(metric.fields[f])) {
return errors.New(fmt.Sprintf("Measurement: %s, expected %d, actual %d\n",
name, value, len(metric.fields[f])))
return fmt.Errorf("measurement: %s, expected %d, actual %d", name, value, len(metric.fields[f]))
}
return nil
}
func test_validate_counter(
func testValidateCounter(
name string,
valueExpected int64,
cache map[string]cachedcounter,
@@ -1559,17 +1557,16 @@ func test_validate_counter(
}
}
if !found {
return errors.New(fmt.Sprintf("Test Error: Metric name %s not found\n", name))
return fmt.Errorf("test Error: Metric name %s not found", name)
}
if valueExpected != valueActual {
return errors.New(fmt.Sprintf("Measurement: %s, expected %d, actual %d\n",
name, valueExpected, valueActual))
return fmt.Errorf("measurement: %s, expected %d, actual %d", name, valueExpected, valueActual)
}
return nil
}
func test_validate_gauge(
func testValidateGauge(
name string,
valueExpected float64,
cache map[string]cachedgauge,
@@ -1591,12 +1588,11 @@ func test_validate_gauge(
}
}
if !found {
return errors.New(fmt.Sprintf("Test Error: Metric name %s not found\n", name))
return fmt.Errorf("test Error: Metric name %s not found", name)
}
if valueExpected != valueActual {
return errors.New(fmt.Sprintf("Measurement: %s, expected %f, actual %f\n",
name, valueExpected, valueActual))
return fmt.Errorf("Measurement: %s, expected %f, actual %f", name, valueExpected, valueActual)
}
return nil
}