dom
/
telegraf
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Add support for custom attributes to vsphere input (#5971)

This commit is contained in:
Pontus Rydin 2019-08-14 20:03:33 -04:00 committed by Daniel Nelson
parent 5e0c63f2e6
commit 2755595019
6 changed files with 205 additions and 57 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
plugins/inputs/vsphere/README.md
View File

@ -118,9 +118,13 @@ vm_metric_exclude = [ "*" ]
"storageAdapter.write.average",
"sys.uptime.latest",
]
## Collect IP addresses? Valid values are "ipv4" and "ipv6"
# ip_addresses = ["ipv6", "ipv4" ]
# host_metric_exclude = [] ## Nothing excluded by default
# host_instances = true ## true by default
## Clusters
# cluster_include = [ "/*/host/**"] # Inventory path to clusters to collect (by default all are collected)
# cluster_metric_include = [] ## if omitted or empty, all metrics are collected
@ -173,6 +177,17 @@ vm_metric_exclude = [ "*" ]
## the plugin. Setting this flag to "false" will send values as floats to
## preserve the full precision when averaging takes place.
# use_int_samples = true
## Custom attributes from vCenter can be very useful for queries in order to slice the
## metrics along different dimension and for forming ad-hoc relationships. They are disabled
## by default, since they can add a considerable amount of tags to the resulting metrics. To
## enable, simply set custom_attribute_exlude to [] (empty set) and use custom_attribute_include
## to select the attributes you want to include.
# by default, since they can add a considerable amount of tags to the resulting metrics. To
# enable, simply set custom_attribute_exlude to [] (empty set) and use custom_attribute_include
# to select the attributes you want to include.
# custom_attribute_include = []
# custom_attribute_exclude = ["*"] # Default is to exclude everything
## Optional SSL Config
# ssl_ca = "/path/to/cafile"
@ -241,7 +256,7 @@ to a file system. A vSphere inventory has a structure similar to this:
#### Using Inventory Paths
Using familiar UNIX-style paths, one could select e.g. VM2 with the path ```/DC0/vm/VM2```.
Often, we want to select a group of resource, such as all the VMs in a folder. We could use the path ```/DC0/vm/Folder1/*``` for that.
Often, we want to select a group of resource, such as all the VMs in a folder. We could use the path ```/DC0/vm/Folder1/*``` for that.
Another possibility is to select objects using a partial name, such as ```/DC0/vm/Folder1/hadoop*``` yielding all vms in Folder1 with a name starting with "hadoop".

15
plugins/inputs/vsphere/client.go
View File

@ -305,3 +305,18 @@ func (c *Client) ListResources(ctx context.Context, root *view.ContainerView, ki
defer cancel1()
return root.Retrieve(ctx1, kind, ps, dst)
}
func (c *Client) GetCustomFields(ctx context.Context) (map[int32]string, error) {
ctx1, cancel1 := context.WithTimeout(ctx, c.Timeout)
defer cancel1()
cfm := object.NewCustomFieldsManager(c.Client.Client)
fields, err := cfm.Field(ctx1)
if err != nil {
return nil, err
}
r := make(map[int32]string)
for _, f := range fields {
r[f.Key] = f.Name
}
return r, nil
}

173
plugins/inputs/vsphere/endpoint.go
View File

@ -26,6 +26,10 @@ import (
var isolateLUN = regexp.MustCompile(".*/([^/]+)/?$")
var isIPv4 = regexp.MustCompile("^(?:[0-9]{1,3}\\.){3}[0-9]{1,3}$")
var isIPv6 = regexp.MustCompile("^(?:[A-Fa-f0-9]{0,4}:){1,7}[A-Fa-f0-9]{1,4}$")
const metricLookback = 3 // Number of time periods to look back at for non-realtime metrics
const rtMetricLookback = 3 // Number of time periods to look back at for realtime metrics
@ -37,16 +41,19 @@ const maxMetadataSamples = 100 // Number of resources to sample for metric metad
// Endpoint is a high-level representation of a connected vCenter endpoint. It is backed by the lower
// level Client type.
type Endpoint struct {
Parent *VSphere
URL *url.URL
resourceKinds map[string]*resourceKind
hwMarks *TSCache
lun2ds map[string]string
discoveryTicker *time.Ticker
collectMux sync.RWMutex
initialized bool
clientFactory *ClientFactory
busy sync.Mutex
Parent *VSphere
URL *url.URL
resourceKinds map[string]*resourceKind
hwMarks *TSCache
lun2ds map[string]string
discoveryTicker *time.Ticker
collectMux sync.RWMutex
initialized bool
clientFactory *ClientFactory
busy sync.Mutex
customFields map[int32]string
customAttrFilter filter.Filter
customAttrEnabled bool
}
type resourceKind struct {
@ -80,12 +87,14 @@ type metricEntry struct {
type objectMap map[string]objectRef
type objectRef struct {
name string
altID string
ref types.ManagedObjectReference
parentRef *types.ManagedObjectReference //Pointer because it must be nillable
guest string
dcname string
name string
altID string
ref types.ManagedObjectReference
parentRef *types.ManagedObjectReference //Pointer because it must be nillable
guest string
dcname string
customValues map[string]string
lookup map[string]string
}
func (e *Endpoint) getParent(obj *objectRef, res *resourceKind) (*objectRef, bool) {
@ -101,12 +110,14 @@ func (e *Endpoint) getParent(obj *objectRef, res *resourceKind) (*objectRef, boo
// as parameters.
func NewEndpoint(ctx context.Context, parent *VSphere, url *url.URL) (*Endpoint, error) {
e := Endpoint{
URL: url,
Parent: parent,
hwMarks: NewTSCache(1 * time.Hour),
lun2ds: make(map[string]string),
initialized: false,
clientFactory: NewClientFactory(ctx, url, parent),
URL: url,
Parent: parent,
hwMarks: NewTSCache(1 * time.Hour),
lun2ds: make(map[string]string),
initialized: false,
clientFactory: NewClientFactory(ctx, url, parent),
customAttrFilter: newFilterOrPanic(parent.CustomAttributeInclude, parent.CustomAttributeExclude),
customAttrEnabled: anythingEnabled(parent.CustomAttributeExclude),
}
e.resourceKinds = map[string]*resourceKind{
@ -259,6 +270,20 @@ func (e *Endpoint) initalDiscovery(ctx context.Context) {
}
func (e *Endpoint) init(ctx context.Context) error {
client, err := e.clientFactory.GetClient(ctx)
if err != nil {
return err
}
// Initial load of custom field metadata
if e.customAttrEnabled {
fields, err := client.GetCustomFields(ctx)
if err != nil {
log.Println("W! [inputs.vsphere] Could not load custom field metadata")
} else {
e.customFields = fields
}
}
if e.Parent.ObjectDiscoveryInterval.Duration > 0 {
@ -427,6 +452,16 @@ func (e *Endpoint) discover(ctx context.Context) error {
}
}
// Load custom field metadata
var fields map[int32]string
if e.customAttrEnabled {
fields, err = client.GetCustomFields(ctx)
if err != nil {
log.Println("W! [inputs.vsphere] Could not load custom field metadata")
fields = nil
}
}
// Atomically swap maps
e.collectMux.Lock()
defer e.collectMux.Unlock()
@ -436,6 +471,10 @@ func (e *Endpoint) discover(ctx context.Context) error {
}
e.lun2ds = l2d
if fields != nil {
e.customFields = fields
}
sw.Stop()
SendInternalCounterWithTags("discovered_objects", e.URL.Host, map[string]string{"type": "instance-total"}, numRes)
return nil
@ -609,14 +648,77 @@ func getVMs(ctx context.Context, e *Endpoint, filter *ResourceFilter) (objectMap
}
guest := "unknown"
uuid := ""
lookup := make(map[string]string)
// Extract host name
if r.Guest != nil && r.Guest.HostName != "" {
lookup["guesthostname"] = r.Guest.HostName
}
// Collect network information
for _, net := range r.Guest.Net {
if net.DeviceConfigId == -1 {
continue
}
if net.IpConfig == nil || net.IpConfig.IpAddress == nil {
continue
}
ips := make(map[string][]string)
for _, ip := range net.IpConfig.IpAddress {
addr := ip.IpAddress
for _, ipType := range e.Parent.IpAddresses {
if !(ipType == "ipv4" && isIPv4.MatchString(addr) ||
ipType == "ipv6" && isIPv6.MatchString(addr)) {
continue
}
// By convention, we want the preferred addresses to appear first in the array.
if _, ok := ips[ipType]; !ok {
ips[ipType] = make([]string, 0)
}
if ip.State == "preferred" {
ips[ipType] = append([]string{addr}, ips[ipType]...)
} else {
ips[ipType] = append(ips[ipType], addr)
}
}
}
for ipType, ipList := range ips {
lookup["nic/"+strconv.Itoa(int(net.DeviceConfigId))+"/"+ipType] = strings.Join(ipList, ",")
}
}
// Sometimes Config is unknown and returns a nil pointer
//
if r.Config != nil {
guest = cleanGuestID(r.Config.GuestId)
uuid = r.Config.Uuid
}
cvs := make(map[string]string)
if e.customAttrEnabled {
for _, cv := range r.Summary.CustomValue {
val := cv.(*types.CustomFieldStringValue)
if val.Value == "" {
continue
}
key, ok := e.customFields[val.Key]
if !ok {
log.Printf("W! [inputs.vsphere] Metadata for custom field %d not found. Skipping", val.Key)
continue
}
if e.customAttrFilter.Match(key) {
cvs[key] = val.Value
}
}
}
m[r.ExtensibleManagedObject.Reference().Value] = objectRef{
name: r.Name, ref: r.ExtensibleManagedObject.Reference(), parentRef: r.Runtime.Host, guest: guest, altID: uuid}
name: r.Name,
ref: r.ExtensibleManagedObject.Reference(),
parentRef: r.Runtime.Host,
guest: guest,
altID: uuid,
customValues: cvs,
lookup: lookup,
}
}
return m, nil
}
@ -1032,6 +1134,9 @@ func (e *Endpoint) populateTags(objectRef *objectRef, resourceType string, resou
if objectRef.guest != "" {
t["guest"] = objectRef.guest
}
if gh := objectRef.lookup["guesthostname"]; gh != "" {
t["guesthostname"] = gh
}
if c, ok := e.resourceKinds["cluster"].objects[parent.parentRef.Value]; ok {
t["clustername"] = c.name
}
@ -1062,6 +1167,17 @@ func (e *Endpoint) populateTags(objectRef *objectRef, resourceType string, resou
t["disk"] = cleanDiskTag(instance)
} else if strings.HasPrefix(name, "net.") {
t["interface"] = instance
// Add IP addresses to NIC data.
if resourceType == "vm" && objectRef.lookup != nil {
key := "nic/" + t["interface"] + "/"
if ip, ok := objectRef.lookup[key+"ipv6"]; ok {
t["ipv6"] = ip
}
if ip, ok := objectRef.lookup[key+"ipv4"]; ok {
t["ipv4"] = ip
}
}
} else if strings.HasPrefix(name, "storageAdapter.") {
t["adapter"] = instance
} else if strings.HasPrefix(name, "storagePath.") {
@ -1076,6 +1192,15 @@ func (e *Endpoint) populateTags(objectRef *objectRef, resourceType string, resou
// default
t["instance"] = v.Instance
}
// Fill in custom values if they exist
if objectRef.customValues != nil {
for k, v := range objectRef.customValues {
if v != "" {
t[k] = v
}
}
}
}
func (e *Endpoint) makeMetricIdentifier(prefix, metric string) (string, string) {

5
plugins/inputs/vsphere/finder.go
View File

@ -231,8 +231,9 @@ func init() {
}
addFields = map[string][]string{
"HostSystem": {"parent"},
"VirtualMachine": {"runtime.host", "config.guestId", "config.uuid", "runtime.powerState"},
"HostSystem": {"parent"},
"VirtualMachine": {"runtime.host", "config.guestId", "config.uuid", "runtime.powerState",
"summary.customValue", "guest.net", "guest.hostName"},
"Datastore": {"parent", "info"},
"ClusterComputeResource": {"parent"},
"Datacenter": {"parent"},

18
plugins/inputs/vsphere/vsphere.go
View File

@ -40,7 +40,10 @@ type VSphere struct {
DatastoreMetricExclude []string
DatastoreInclude []string
Separator string
CustomAttributeInclude []string
CustomAttributeExclude []string
UseIntSamples bool
IpAddresses []string
MaxQueryObjects int
MaxQueryMetrics int
@ -155,6 +158,8 @@ var sampleConfig = `
"storageAdapter.write.average",
"sys.uptime.latest",
]
## Collect IP addresses? Valid values are "ipv4" and "ipv6"
# ip_addresses = ["ipv6", "ipv4" ]
# host_metric_exclude = [] ## Nothing excluded by default
# host_instances = true ## true by default
@ -173,7 +178,7 @@ var sampleConfig = `
datacenter_metric_exclude = [ "*" ] ## Datacenters are not collected by default.
# datacenter_instances = false ## false by default for Datastores only
## Plugin Settings
## Plugin Settings
## separator character to use for measurement and field names (default: "_")
# separator = "_"
@ -208,6 +213,14 @@ var sampleConfig = `
## preserve the full precision when averaging takes place.
# use_int_samples = true
## Custom attributes from vCenter can be very useful for queries in order to slice the
## metrics along different dimension and for forming ad-hoc relationships. They are disabled
## by default, since they can add a considerable amount of tags to the resulting metrics. To
## enable, simply set custom_attribute_exlude to [] (empty set) and use custom_attribute_include
## to select the attributes you want to include.
# custom_attribute_include = []
# custom_attribute_exclude = ["*"]
## Optional SSL Config
# ssl_ca = "/path/to/cafile"
# ssl_cert = "/path/to/certfile"
@ -321,7 +334,10 @@ func init() {
DatastoreMetricExclude: nil,
DatastoreInclude: []string{"/*/datastore/**"},
Separator: "_",
CustomAttributeInclude: []string{},
CustomAttributeExclude: []string{"*"},
UseIntSamples: true,
IpAddresses: []string{},
MaxQueryObjects: 256,
MaxQueryMetrics: 256,

34
plugins/inputs/vsphere/vsphere_test.go
View File

@ -6,7 +6,6 @@ import (
"fmt"
"regexp"
"sort"
"strings"
"sync"
"sync/atomic"
"testing"
@ -256,34 +255,6 @@ func TestThrottledExecutor(t *testing.T) {
require.Equal(t, int64(5), max, "Wrong number of goroutines spawned")
}
func TestTimeout(t *testing.T) {
// Don't run test on 32-bit machines due to bug in simulator.
// https://github.com/vmware/govmomi/issues/1330
var i int
if unsafe.Sizeof(i) < 8 {
return
}
m, s, err := createSim()
if err != nil {
t.Fatal(err)
}
defer m.Remove()
defer s.Close()
v := defaultVSphere()
var acc testutil.Accumulator
v.Vcenters = []string{s.URL.String()}
v.Timeout = internal.Duration{Duration: 1 * time.Nanosecond}
require.NoError(t, v.Start(nil)) // We're not using the Accumulator, so it can be nil.
defer v.Stop()
err = v.Gather(&acc)
// The accumulator must contain exactly one error and it must be a deadline exceeded.
require.Equal(t, 1, len(acc.Errors))
require.True(t, strings.Contains(acc.Errors[0].Error(), "context deadline exceeded"))
}
func TestMaxQuery(t *testing.T) {
// Don't run test on 32-bit machines due to bug in simulator.
// https://github.com/vmware/govmomi/issues/1330
@ -414,6 +385,11 @@ func TestFinder(t *testing.T) {
require.NoError(t, err)
require.Equal(t, 2, len(vm))
vm = []mo.VirtualMachine{}
err = f.Find(ctx, "VirtualMachine", "/DC0/**/DC0_H0_VM*", &vm)
require.NoError(t, err)
require.Equal(t, 2, len(vm))
vm = []mo.VirtualMachine{}
err = f.Find(ctx, "VirtualMachine", "/**/vm/**", &vm)
require.NoError(t, err)