Statsd: unit tests for gauges, sets, counters

This commit is contained in:
Cameron Sparr 2015-10-05 16:19:08 -06:00
parent eb2a4dc724
commit d84a258b0a
3 changed files with 483 additions and 17 deletions

View File

@ -75,5 +75,5 @@ implementation. In short, the telegraf statsd listener will accept:
- `users.unique:101|s` - `users.unique:101|s`
- `users.unique:101|s` - `users.unique:101|s`
- `users.unique:102|s` <- would result in a count of 2 for `users.unique` - `users.unique:102|s` <- would result in a count of 2 for `users.unique`
- Timers - Timings
- TODO - TODO

View File

@ -1,6 +1,7 @@
package statsd package statsd
import ( import (
"errors"
"log" "log"
"net" "net"
"strconv" "strconv"
@ -44,6 +45,20 @@ type Statsd struct {
} }
} }
func NewStatsd() *Statsd {
s := Statsd{}
// Make data structures
s.done = make(chan struct{})
s.in = make(chan string, s.AllowedPendingMessages)
s.inmetrics = make(chan metric, s.AllowedPendingMessages)
s.gauges = make(map[string]cachedmetric)
s.counters = make(map[string]cachedmetric)
s.sets = make(map[string]cachedmetric)
return &s
}
// One statsd metric, form is <bucket>:<value>|<mtype>|@<samplerate> // One statsd metric, form is <bucket>:<value>|<mtype>|@<samplerate>
type metric struct { type metric struct {
name string name string
@ -201,7 +216,7 @@ func (s *Statsd) parser() error {
// parseStatsdLine will parse the given statsd line, validating it as it goes. // parseStatsdLine will parse the given statsd line, validating it as it goes.
// If the line is valid, it will be cached for the next call to Gather() // If the line is valid, it will be cached for the next call to Gather()
func (s *Statsd) parseStatsdLine(line string) { func (s *Statsd) parseStatsdLine(line string) error {
s.Lock() s.Lock()
defer s.Unlock() defer s.Unlock()
@ -209,21 +224,21 @@ func (s *Statsd) parseStatsdLine(line string) {
m := metric{} m := metric{}
parts1 := strings.Split(line, "|") parts1 := strings.Split(line, "|")
if len(parts1) < 2 { if len(parts1) < 2 {
log.Printf("Error splitting '|', Unable to parse metric: %s\n", line) log.Printf("Error: splitting '|', Unable to parse metric: %s\n", line)
return return errors.New("Error Parsing statsd line")
} else if len(parts1) > 2 { } else if len(parts1) > 2 {
sr := parts1[2] sr := parts1[2]
errmsg := "Error: parsing sample rate, %s, it must be in format like: " +
"@0.1, @0.5, etc. Ignoring sample rate for line: %s\n"
if strings.Contains(sr, "@") && len(sr) > 1 { if strings.Contains(sr, "@") && len(sr) > 1 {
samplerate, err := strconv.ParseFloat(sr[1:], 64) samplerate, err := strconv.ParseFloat(sr[1:], 64)
if err != nil { if err != nil {
log.Printf("Error parsing sample rate: %s\n", err.Error()) log.Printf(errmsg, err.Error(), line)
} else { } else {
m.samplerate = samplerate m.samplerate = samplerate
} }
} else { } else {
msg := "Error parsing sample rate, it must be in format like: " + log.Printf(errmsg, "", line)
"@0.1, @0.5, etc. Ignoring sample rate for line: %s\n"
log.Printf(msg, line)
} }
} }
@ -232,15 +247,15 @@ func (s *Statsd) parseStatsdLine(line string) {
case "g", "c", "s", "ms", "h": case "g", "c", "s", "ms", "h":
m.mtype = parts1[1] m.mtype = parts1[1]
default: default:
log.Printf("Statsd Metric type %s unsupported", parts1[1]) log.Printf("Error: Statsd Metric type %s unsupported", parts1[1])
return return errors.New("Error Parsing statsd line")
} }
// Validate splitting the rest of the line on ":" // Validate splitting the rest of the line on ":"
parts2 := strings.Split(parts1[0], ":") parts2 := strings.Split(parts1[0], ":")
if len(parts2) != 2 { if len(parts2) != 2 {
log.Printf("Error splitting ':', Unable to parse metric: %s\n", line) log.Printf("Error: splitting ':', Unable to parse metric: %s\n", line)
return return errors.New("Error Parsing statsd line")
} }
m.bucket = parts2[0] m.bucket = parts2[0]
@ -248,14 +263,14 @@ func (s *Statsd) parseStatsdLine(line string) {
if strings.ContainsAny(parts2[1], "-+") { if strings.ContainsAny(parts2[1], "-+") {
if m.mtype != "g" { if m.mtype != "g" {
log.Printf("Error: +- values are only supported for gauges: %s\n", line) log.Printf("Error: +- values are only supported for gauges: %s\n", line)
return return errors.New("Error Parsing statsd line")
} }
m.additive = true m.additive = true
} }
v, err := strconv.ParseInt(parts2[1], 10, 64) v, err := strconv.ParseInt(parts2[1], 10, 64)
if err != nil { if err != nil {
log.Printf("Error: parsing value to int64: %s\n", line) log.Printf("Error: parsing value to int64: %s\n", line)
return return errors.New("Error Parsing statsd line")
} }
// If a sample rate is given with a counter, divide value by the rate // If a sample rate is given with a counter, divide value by the rate
if m.samplerate != 0 && m.mtype == "c" { if m.samplerate != 0 && m.mtype == "c" {
@ -278,6 +293,7 @@ func (s *Statsd) parseStatsdLine(line string) {
log.Printf(dropwarn, line) log.Printf(dropwarn, line)
} }
} }
return nil
} }
// parseName parses the given bucket name with the list of bucket maps in the // parseName parses the given bucket name with the list of bucket maps in the

View File

@ -1,11 +1,461 @@
package statsd package statsd
import ( import (
"errors"
"fmt"
"testing" "testing"
"github.com/influxdb/telegraf/testutil"
) )
func TestListen(t *testing.T) { // Invalid lines should return an error
if false { func TestParse_InvalidLines(t *testing.T) {
t.Errorf("Test failed!") s := NewStatsd()
invalid_lines := []string{
"i.dont.have.a.pipe:45g",
"i.dont.have.a.colon45|c",
"invalid.metric.type:45|e",
"invalid.plus.minus.non.gauge:+10|c",
"invalid.plus.minus.non.gauge:+10|s",
"invalid.plus.minus.non.gauge:+10|ms",
"invalid.plus.minus.non.gauge:+10|h",
"invalid.plus.minus.non.gauge:-10|c",
"invalid.value:foobar|c",
"invalid.value:d11|c",
"invalid.value:1d1|c",
"invalid.value:1.1|c",
}
for _, line := range invalid_lines {
err := s.parseStatsdLine(line)
if err == nil {
t.Errorf("Parsing line %s should have resulted in an error\n", line)
}
} }
} }
// Invalid sample rates should be ignored and not applied
func TestParse_InvalidSampleRate(t *testing.T) {
s := NewStatsd()
invalid_lines := []string{
"invalid.sample.rate:45|c|0.1",
"invalid.sample.rate.2:45|c|@foo",
"invalid.sample.rate:45|g|@0.1",
"invalid.sample.rate:45|s|@0.1",
}
for _, line := range invalid_lines {
err := s.parseStatsdLine(line)
if err != nil {
t.Errorf("Parsing line %s should not have resulted in an error\n", line)
}
}
validations := []struct {
name string
value int64
cache map[string]cachedmetric
}{
{
"invalid_sample_rate_counter",
45,
s.counters,
},
{
"invalid_sample_rate_2_counter",
45,
s.counters,
},
{
"invalid_sample_rate_gauge",
45,
s.gauges,
},
{
"invalid_sample_rate_set",
1,
s.sets,
},
}
for _, test := range validations {
err := test_validate_value(test.name, test.value, test.cache)
if err != nil {
t.Error(err.Error())
}
}
}
// Names should be parsed like . -> _ and - -> __
func TestParse_DefaultNameParsing(t *testing.T) {
s := NewStatsd()
valid_lines := []string{
"valid:1|c",
"valid.foo-bar:11|c",
}
for _, line := range valid_lines {
err := s.parseStatsdLine(line)
if err != nil {
t.Errorf("Parsing line %s should not have resulted in an error\n", line)
}
}
validations := []struct {
name string
value int64
}{
{
"valid_counter",
1,
},
{
"valid_foo__bar_counter",
11,
},
}
for _, test := range validations {
err := test_validate_value(test.name, test.value, s.counters)
if err != nil {
t.Error(err.Error())
}
}
}
// Test that name mappings match and work
func TestParse_NameMap(t *testing.T) {
if false {
t.Errorf("TODO")
}
}
// Test that name map tags are applied properly
func TestParse_NameMapTags(t *testing.T) {
if false {
t.Errorf("TODO")
}
}
// Valid lines should be parsed and their values should be cached
func TestParse_ValidLines(t *testing.T) {
s := NewStatsd()
valid_lines := []string{
"valid:45|c",
"valid:45|s",
"valid:45|g",
// TODO(cam): timings
//"valid.timer:45|ms",
//"valid.timer:45|h",
}
for _, line := range valid_lines {
err := s.parseStatsdLine(line)
if err != nil {
t.Errorf("Parsing line %s should not have resulted in an error\n", line)
}
}
validations := []struct {
name string
value int64
cache map[string]cachedmetric
}{
{
"valid_counter",
45,
s.counters,
},
{
"valid_set",
1,
s.sets,
},
{
"valid_gauge",
45,
s.gauges,
},
}
for _, test := range validations {
err := test_validate_value(test.name, test.value, test.cache)
if err != nil {
t.Error(err.Error())
}
}
}
// Tests low-level functionality of gauges
func TestParse_Gauges(t *testing.T) {
s := NewStatsd()
// Test that gauge +- values work
valid_lines := []string{
"plus.minus:100|g",
"plus.minus:-10|g",
"plus.minus:+30|g",
"plus.plus:100|g",
"plus.plus:+100|g",
"plus.plus:+100|g",
"minus.minus:100|g",
"minus.minus:-100|g",
"minus.minus:-100|g",
"lone.plus:+100|g",
"lone.minus:-100|g",
"overwrite:100|g",
"overwrite:300|g",
}
for _, line := range valid_lines {
err := s.parseStatsdLine(line)
if err != nil {
t.Errorf("Parsing line %s should not have resulted in an error\n", line)
}
}
validations := []struct {
name string
value int64
}{
{
"plus_minus_gauge",
120,
},
{
"plus_plus_gauge",
300,
},
{
"minus_minus_gauge",
-100,
},
{
"lone_plus_gauge",
100,
},
{
"lone_minus_gauge",
-100,
},
{
"overwrite_gauge",
300,
},
}
for _, test := range validations {
err := test_validate_value(test.name, test.value, s.gauges)
if err != nil {
t.Error(err.Error())
}
}
}
// Tests low-level functionality of sets
func TestParse_Sets(t *testing.T) {
s := NewStatsd()
// Test that sets work
valid_lines := []string{
"unique.user.ids:100|s",
"unique.user.ids:100|s",
"unique.user.ids:100|s",
"unique.user.ids:100|s",
"unique.user.ids:100|s",
"unique.user.ids:101|s",
"unique.user.ids:102|s",
"unique.user.ids:102|s",
"unique.user.ids:123456789|s",
"oneuser.id:100|s",
"oneuser.id:100|s",
}
for _, line := range valid_lines {
err := s.parseStatsdLine(line)
if err != nil {
t.Errorf("Parsing line %s should not have resulted in an error\n", line)
}
}
validations := []struct {
name string
value int64
}{
{
"unique_user_ids_set",
4,
},
{
"oneuser_id_set",
1,
},
}
for _, test := range validations {
err := test_validate_value(test.name, test.value, s.sets)
if err != nil {
t.Error(err.Error())
}
}
}
// Tests low-level functionality of counters
func TestParse_Counters(t *testing.T) {
s := NewStatsd()
// Test that counters work
valid_lines := []string{
"small.inc:1|c",
"big.inc:100|c",
"big.inc:1|c",
"big.inc:100000|c",
"big.inc:1000000|c",
"small.inc:1|c",
"zero.init:0|c",
"sample.rate:1|c|@0.1",
"sample.rate:1|c",
}
for _, line := range valid_lines {
err := s.parseStatsdLine(line)
if err != nil {
t.Errorf("Parsing line %s should not have resulted in an error\n", line)
}
}
validations := []struct {
name string
value int64
}{
{
"small_inc_counter",
2,
},
{
"big_inc_counter",
1100101,
},
{
"zero_init_counter",
0,
},
{
"sample_rate_counter",
11,
},
}
for _, test := range validations {
err := test_validate_value(test.name, test.value, s.counters)
if err != nil {
t.Error(err.Error())
}
}
}
// Tests low-level functionality of timings
func TestParse_Timings(t *testing.T) {
if false {
t.Errorf("TODO")
}
}
// Tests the delete_gauges option
func TestParse_Gauges_Delete(t *testing.T) {
s := NewStatsd()
s.DeleteGauges = true
fakeacc := &testutil.Accumulator{}
var err error
line := "current.users:100|g"
err = s.parseStatsdLine(line)
if err != nil {
t.Errorf("Parsing line %s should not have resulted in an error\n", line)
}
err = test_validate_value("current_users_gauge", 100, s.gauges)
if err != nil {
t.Error(err.Error())
}
s.Gather(fakeacc)
err = test_validate_value("current_users_gauge", 100, s.gauges)
if err == nil {
t.Error("current_users_gauge metric should have been deleted")
}
}
// Tests the delete_sets option
func TestParse_Sets_Delete(t *testing.T) {
s := NewStatsd()
s.DeleteSets = true
fakeacc := &testutil.Accumulator{}
var err error
line := "unique.user.ids:100|s"
err = s.parseStatsdLine(line)
if err != nil {
t.Errorf("Parsing line %s should not have resulted in an error\n", line)
}
err = test_validate_value("unique_user_ids_set", 1, s.sets)
if err != nil {
t.Error(err.Error())
}
s.Gather(fakeacc)
err = test_validate_value("unique_user_ids_set", 1, s.sets)
if err == nil {
t.Error("unique_user_ids_set metric should have been deleted")
}
}
// Tests the delete_counters option
func TestParse_Counters_Delete(t *testing.T) {
s := NewStatsd()
s.DeleteCounters = true
fakeacc := &testutil.Accumulator{}
var err error
line := "total.users:100|c"
err = s.parseStatsdLine(line)
if err != nil {
t.Errorf("Parsing line %s should not have resulted in an error\n", line)
}
err = test_validate_value("total_users_counter", 100, s.counters)
if err != nil {
t.Error(err.Error())
}
s.Gather(fakeacc)
err = test_validate_value("total_users_counter", 100, s.counters)
if err == nil {
t.Error("total_users_counter metric should have been deleted")
}
}
// Integration test the listener starting up and receiving UDP packets
func TestListen(t *testing.T) {
if false {
t.Errorf("TODO")
}
}
// Test utility functions
func test_validate_value(name string, value int64, cache map[string]cachedmetric) error {
metric, ok := cache[name]
if !ok {
return errors.New(fmt.Sprintf("Test Error: Metric name %s not found\n", name))
}
if value != metric.value {
return errors.New(fmt.Sprintf("Measurement: %s, expected %d, actual %d",
name, value, metric.value))
}
return nil
}