snmp: use a shared global translation cache

Prevents the same data from being looked up multiple times. Also prevents multiple simultaneous lookups.

closes #2115
closes #2104

CHANGELOG.md

@@ -7,6 +7,23 @@
 ### Bugfixes
 
 ## v1.1 [unreleased]
+- [#1949](https://github.com/influxdata/telegraf/issues/1949): Fix windows `net` plugin.
+
+## v1.1.1 [unreleased]
+## v1.1.2 [2016-12-12]
+
+### Bugfixes
+
+- [#2007](https://github.com/influxdata/telegraf/issues/2007): Make snmptranslate not required when using numeric OID.
+- [#2104](https://github.com/influxdata/telegraf/issues/2104): Add a global snmp translation cache.
+
+## v1.1.1 [2016-11-14]
+
+### Bugfixes
+
+- [#2023](https://github.com/influxdata/telegraf/issues/2023): Fix issue parsing toml durations with single quotes.
+
+## v1.1.0 [2016-11-07]
 
 ### Release Notes
 

internal/internal.go

@@ -35,8 +35,9 @@ type Duration struct {
 // UnmarshalTOML parses the duration from the TOML config file
 func (d *Duration) UnmarshalTOML(b []byte) error {
 	var err error
+	b = bytes.Trim(b, `'`)
 
-	// see if we can straight convert it
+	// see if we can directly convert it
 	d.Duration, err = time.ParseDuration(string(b))
 	if err == nil {
 		return nil

internal/internal_test.go

@@ -142,6 +142,10 @@ func TestDuration(t *testing.T) {
 	d.UnmarshalTOML([]byte(`1s`))
 	assert.Equal(t, time.Second, d.Duration)
 
+	d = Duration{}
+	d.UnmarshalTOML([]byte(`'1s'`))
+	assert.Equal(t, time.Second, d.Duration)
+
 	d = Duration{}
 	d.UnmarshalTOML([]byte(`10`))
 	assert.Equal(t, 10*time.Second, d.Duration)

plugins/inputs/snmp/snmp.go

@@ -8,6 +8,7 @@ import (
 	"os/exec"
 	"strconv"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/influxdata/telegraf"
@@ -199,65 +200,22 @@ func (t *Table) init() error {
 	return nil
 }
 
-// init() populates Fields if a table OID is provided.
+// initBuild initializes the table if it has an OID configured. If so, the
+// net-snmp tools will be used to look up the OID and auto-populate the table's
+// fields.
 func (t *Table) initBuild() error {
 	if t.Oid == "" {
 		return nil
 	}
 
-	mibName, _, oidText, _, err := snmpTranslate(t.Oid)
+	_, _, oidText, fields, err := snmpTable(t.Oid)
 	if err != nil {
-		return Errorf(err, "translating %s", t.Oid)
+		return Errorf(err, "initializing table %s", t.Oid)
 	}
 	if t.Name == "" {
 		t.Name = oidText
 	}
-	mibPrefix := mibName + "::"
-	oidFullName := mibPrefix + oidText
-
-	// first attempt to get the table's tags
-	tagOids := map[string]struct{}{}
-	// We have to guess that the "entry" oid is `t.Oid+".1"`. snmptable and snmptranslate don't seem to have a way to provide the info.
-	if out, err := execCmd("snmptranslate", "-Td", oidFullName+".1"); err == nil {
-		lines := bytes.Split(out, []byte{'\n'})
-		for _, line := range lines {
-			if !bytes.HasPrefix(line, []byte("  INDEX")) {
-				continue
-			}
-
-			i := bytes.Index(line, []byte("{ "))
-			if i == -1 { // parse error
-				continue
-			}
-			line = line[i+2:]
-			i = bytes.Index(line, []byte(" }"))
-			if i == -1 { // parse error
-				continue
-			}
-			line = line[:i]
-			for _, col := range bytes.Split(line, []byte(", ")) {
-				tagOids[mibPrefix+string(col)] = struct{}{}
-			}
-		}
-	}
-
-	// this won't actually try to run a query. The `-Ch` will just cause it to dump headers.
-	out, err := execCmd("snmptable", "-Ch", "-Cl", "-c", "public", "127.0.0.1", oidFullName)
-	if err != nil {
-		return Errorf(err, "getting table columns for %s", t.Oid)
-	}
-	cols := bytes.SplitN(out, []byte{'\n'}, 2)[0]
-	if len(cols) == 0 {
-		return fmt.Errorf("unable to get columns for table %s", t.Oid)
-	}
-	for _, col := range bytes.Split(cols, []byte{' '}) {
-		if len(col) == 0 {
-			continue
-		}
-		col := string(col)
-		_, isTag := tagOids[mibPrefix+col]
-		t.Fields = append(t.Fields, Field{Name: col, Oid: mibPrefix + col, IsTag: isTag})
-	}
+	t.Fields = append(t.Fields, fields...)
 
 	return nil
 }
@@ -841,13 +799,141 @@ func fieldConvert(conv string, v interface{}) (interface{}, error) {
 	return nil, fmt.Errorf("invalid conversion type '%s'", conv)
 }
 
+type snmpTableCache struct {
+	mibName string
+	oidNum  string
+	oidText string
+	fields  []Field
+	err     error
+}
+
+var snmpTableCaches map[string]snmpTableCache
+var snmpTableCachesLock sync.Mutex
+
+// snmpTable resolves the given OID as a table, providing information about the
+// table and fields within.
+func snmpTable(oid string) (mibName string, oidNum string, oidText string, fields []Field, err error) {
+	snmpTableCachesLock.Lock()
+	if snmpTableCaches == nil {
+		snmpTableCaches = map[string]snmpTableCache{}
+	}
+
+	var stc snmpTableCache
+	var ok bool
+	if stc, ok = snmpTableCaches[oid]; !ok {
+		stc.mibName, stc.oidNum, stc.oidText, stc.fields, stc.err = snmpTableCall(oid)
+		snmpTableCaches[oid] = stc
+	}
+
+	snmpTableCachesLock.Unlock()
+	return stc.mibName, stc.oidNum, stc.oidText, stc.fields, stc.err
+}
+
+func snmpTableCall(oid string) (mibName string, oidNum string, oidText string, fields []Field, err error) {
+	mibName, oidNum, oidText, _, err = snmpTranslate(oid)
+	if err != nil {
+		return "", "", "", nil, Errorf(err, "translating")
+	}
+
+	mibPrefix := mibName + "::"
+	oidFullName := mibPrefix + oidText
+
+	// first attempt to get the table's tags
+	tagOids := map[string]struct{}{}
+	// We have to guess that the "entry" oid is `oid+".1"`. snmptable and snmptranslate don't seem to have a way to provide the info.
+	if out, err := execCmd("snmptranslate", "-Td", oidFullName+".1"); err == nil {
+		lines := bytes.Split(out, []byte{'\n'})
+		for _, line := range lines {
+			if !bytes.HasPrefix(line, []byte("  INDEX")) {
+				continue
+			}
+
+			i := bytes.Index(line, []byte("{ "))
+			if i == -1 { // parse error
+				continue
+			}
+			line = line[i+2:]
+			i = bytes.Index(line, []byte(" }"))
+			if i == -1 { // parse error
+				continue
+			}
+			line = line[:i]
+			for _, col := range bytes.Split(line, []byte(", ")) {
+				tagOids[mibPrefix+string(col)] = struct{}{}
+			}
+		}
+	}
+
+	// this won't actually try to run a query. The `-Ch` will just cause it to dump headers.
+	out, err := execCmd("snmptable", "-Ch", "-Cl", "-c", "public", "127.0.0.1", oidFullName)
+	if err != nil {
+		return "", "", "", nil, Errorf(err, "getting table columns")
+	}
+	cols := bytes.SplitN(out, []byte{'\n'}, 2)[0]
+	if len(cols) == 0 {
+		return "", "", "", nil, fmt.Errorf("could not find any columns in table")
+	}
+	for _, col := range bytes.Split(cols, []byte{' '}) {
+		if len(col) == 0 {
+			continue
+		}
+		col := string(col)
+		_, isTag := tagOids[mibPrefix+col]
+		fields = append(fields, Field{Name: col, Oid: mibPrefix + col, IsTag: isTag})
+	}
+
+	return mibName, oidNum, oidText, fields, err
+}
+
+type snmpTranslateCache struct {
+	mibName    string
+	oidNum     string
+	oidText    string
+	conversion string
+	err        error
+}
+
+var snmpTranslateCachesLock sync.Mutex
+var snmpTranslateCaches map[string]snmpTranslateCache
+
 // snmpTranslate resolves the given OID.
 func snmpTranslate(oid string) (mibName string, oidNum string, oidText string, conversion string, err error) {
+	snmpTranslateCachesLock.Lock()
+	if snmpTranslateCaches == nil {
+		snmpTranslateCaches = map[string]snmpTranslateCache{}
+	}
+
+	var stc snmpTranslateCache
+	var ok bool
+	if stc, ok = snmpTranslateCaches[oid]; !ok {
+		// This will result in only one call to snmptranslate running at a time.
+		// We could speed it up by putting a lock in snmpTranslateCache and then
+		// returning it immediately, and multiple callers would then release the
+		// snmpTranslateCachesLock and instead wait on the individual
+		// snmpTranlsation.Lock to release. But I don't know that the extra complexity
+		// is worth it. Especially when it would slam the system pretty hard if lots
+		// of lookups are being perfomed.
+
+		stc.mibName, stc.oidNum, stc.oidText, stc.conversion, stc.err = snmpTranslateCall(oid)
+		snmpTranslateCaches[oid] = stc
+	}
+
+	snmpTranslateCachesLock.Unlock()
+
+	return stc.mibName, stc.oidNum, stc.oidText, stc.conversion, stc.err
+}
+
+func snmpTranslateCall(oid string) (mibName string, oidNum string, oidText string, conversion string, err error) {
 	var out []byte
 	if strings.ContainsAny(oid, ":abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ") {
 		out, err = execCmd("snmptranslate", "-Td", "-Ob", oid)
 	} else {
 		out, err = execCmd("snmptranslate", "-Td", "-Ob", "-m", "all", oid)
+		if err, ok := err.(*exec.Error); ok && err.Err == exec.ErrNotFound {
+			// Silently discard error if snmptranslate not found and we have a numeric OID.
+			// Meaning we can get by without the lookup.
+			return "", oid, oid, "", nil
+		}
 	}
 	if err != nil {
 		return "", "", "", "", err

plugins/inputs/snmp/snmp_test.go

@@ -4,6 +4,7 @@ package snmp
 import (
 	"fmt"
 	"net"
+	"os/exec"
 	"sync"
 	"testing"
 	"time"
@@ -198,6 +199,56 @@ func TestSnmpInit(t *testing.T) {
 	}, s.Fields[0])
 }
 
+func TestSnmpInit_noTranslate(t *testing.T) {
+	// override execCommand so it returns exec.ErrNotFound
+	defer func(ec func(string, ...string) *exec.Cmd) { execCommand = ec }(execCommand)
+	execCommand = func(_ string, _ ...string) *exec.Cmd {
+		return exec.Command("snmptranslateExecErrNotFound")
+	}
+
+	s := &Snmp{
+		Fields: []Field{
+			{Oid: ".1.1.1.1", Name: "one", IsTag: true},
+			{Oid: ".1.1.1.2", Name: "two"},
+			{Oid: ".1.1.1.3"},
+		},
+		Tables: []Table{
+			{Fields: []Field{
+				{Oid: ".1.1.1.4", Name: "four", IsTag: true},
+				{Oid: ".1.1.1.5", Name: "five"},
+				{Oid: ".1.1.1.6"},
+			}},
+		},
+	}
+
+	err := s.init()
+	require.NoError(t, err)
+
+	assert.Equal(t, ".1.1.1.1", s.Fields[0].Oid)
+	assert.Equal(t, "one", s.Fields[0].Name)
+	assert.Equal(t, true, s.Fields[0].IsTag)
+
+	assert.Equal(t, ".1.1.1.2", s.Fields[1].Oid)
+	assert.Equal(t, "two", s.Fields[1].Name)
+	assert.Equal(t, false, s.Fields[1].IsTag)
+
+	assert.Equal(t, ".1.1.1.3", s.Fields[2].Oid)
+	assert.Equal(t, ".1.1.1.3", s.Fields[2].Name)
+	assert.Equal(t, false, s.Fields[2].IsTag)
+
+	assert.Equal(t, ".1.1.1.4", s.Tables[0].Fields[0].Oid)
+	assert.Equal(t, "four", s.Tables[0].Fields[0].Name)
+	assert.Equal(t, true, s.Tables[0].Fields[0].IsTag)
+
+	assert.Equal(t, ".1.1.1.5", s.Tables[0].Fields[1].Oid)
+	assert.Equal(t, "five", s.Tables[0].Fields[1].Name)
+	assert.Equal(t, false, s.Tables[0].Fields[1].IsTag)
+
+	assert.Equal(t, ".1.1.1.6", s.Tables[0].Fields[2].Oid)
+	assert.Equal(t, ".1.1.1.6", s.Tables[0].Fields[2].Name)
+	assert.Equal(t, false, s.Tables[0].Fields[2].IsTag)
+}
+
 func TestGetSNMPConnection_v2(t *testing.T) {
 	s := &Snmp{
 		Timeout:   internal.Duration{Duration: 3 * time.Second},
@@ -597,6 +648,71 @@ func TestFieldConvert(t *testing.T) {
 	}
 }
 
+func TestSnmpTranslateCache_miss(t *testing.T) {
+	snmpTranslateCaches = nil
+	oid := "IF-MIB::ifPhysAddress.1"
+	mibName, oidNum, oidText, conversion, err := snmpTranslate(oid)
+	assert.Len(t, snmpTranslateCaches, 1)
+	stc := snmpTranslateCaches[oid]
+	require.NotNil(t, stc)
+	assert.Equal(t, mibName, stc.mibName)
+	assert.Equal(t, oidNum, stc.oidNum)
+	assert.Equal(t, oidText, stc.oidText)
+	assert.Equal(t, conversion, stc.conversion)
+	assert.Equal(t, err, stc.err)
+}
+
+func TestSnmpTranslateCache_hit(t *testing.T) {
+	snmpTranslateCaches = map[string]snmpTranslateCache{
+		"foo": snmpTranslateCache{
+			mibName:    "a",
+			oidNum:     "b",
+			oidText:    "c",
+			conversion: "d",
+			err:        fmt.Errorf("e"),
+		},
+	}
+	mibName, oidNum, oidText, conversion, err := snmpTranslate("foo")
+	assert.Equal(t, "a", mibName)
+	assert.Equal(t, "b", oidNum)
+	assert.Equal(t, "c", oidText)
+	assert.Equal(t, "d", conversion)
+	assert.Equal(t, fmt.Errorf("e"), err)
+	snmpTranslateCaches = nil
+}
+
+func TestSnmpTableCache_miss(t *testing.T) {
+	snmpTableCaches = nil
+	oid := ".1.0.0.0"
+	mibName, oidNum, oidText, fields, err := snmpTable(oid)
+	assert.Len(t, snmpTableCaches, 1)
+	stc := snmpTableCaches[oid]
+	require.NotNil(t, stc)
+	assert.Equal(t, mibName, stc.mibName)
+	assert.Equal(t, oidNum, stc.oidNum)
+	assert.Equal(t, oidText, stc.oidText)
+	assert.Equal(t, fields, stc.fields)
+	assert.Equal(t, err, stc.err)
+}
+
+func TestSnmpTableCache_hit(t *testing.T) {
+	snmpTableCaches = map[string]snmpTableCache{
+		"foo": snmpTableCache{
+			mibName: "a",
+			oidNum:  "b",
+			oidText: "c",
+			fields:  []Field{{Name: "d"}},
+			err:     fmt.Errorf("e"),
+		},
+	}
+	mibName, oidNum, oidText, fields, err := snmpTable("foo")
+	assert.Equal(t, "a", mibName)
+	assert.Equal(t, "b", oidNum)
+	assert.Equal(t, "c", oidText)
+	assert.Equal(t, []Field{{Name: "d"}}, fields)
+	assert.Equal(t, fmt.Errorf("e"), err)
+}
+
 func TestError(t *testing.T) {
 	e := fmt.Errorf("nested error")
 	err := Errorf(e, "top error %d", 123)