Improve scalability of vsphere input (#5113)

plugins/inputs/vsphere/vsphere_test.go

@@ -7,8 +7,11 @@ import (
 	"regexp"
 	"sort"
 	"strings"
+	"sync"
+	"sync/atomic"
 	"testing"
 	"time"
+	"unsafe"
 
 	"github.com/influxdata/telegraf/internal"
 	itls "github.com/influxdata/telegraf/internal/tls"
@@ -175,6 +178,8 @@ func defaultVSphere() *VSphere {
 		ObjectDiscoveryInterval: internal.Duration{Duration: time.Second * 300},
 		Timeout:                 internal.Duration{Duration: time.Second * 20},
 		ForceDiscoverOnInit:     true,
+		DiscoverConcurrency:     1,
+		CollectConcurrency:      1,
 	}
 }
 
@@ -205,32 +210,43 @@ func TestParseConfig(t *testing.T) {
 }
 
 func TestWorkerPool(t *testing.T) {
-	wp := NewWorkerPool(100)
-	ctx := context.Background()
-	wp.Run(ctx, func(ctx context.Context, p interface{}) interface{} {
-		return p.(int) * 2
-	}, 10)
-
-	n := 100000
-	wp.Fill(ctx, func(ctx context.Context, f PushFunc) {
-		for i := 0; i < n; i++ {
-			f(ctx, i)
-		}
-	})
-	results := make([]int, n)
-	i := 0
-	wp.Drain(ctx, func(ctx context.Context, p interface{}) bool {
-		results[i] = p.(int)
-		i++
-		return true
-	})
+	max := int64(0)
+	ngr := int64(0)
+	n := 10000
+	var mux sync.Mutex
+	results := make([]int, 0, n)
+	te := NewThrottledExecutor(5)
+	for i := 0; i < n; i++ {
+		func(i int) {
+			te.Run(context.Background(), func() {
+				atomic.AddInt64(&ngr, 1)
+				mux.Lock()
+				defer mux.Unlock()
+				results = append(results, i*2)
+				if ngr > max {
+					max = ngr
+				}
+				time.Sleep(100 * time.Microsecond)
+				atomic.AddInt64(&ngr, -1)
+			})
+		}(i)
+	}
+	te.Wait()
 	sort.Ints(results)
 	for i := 0; i < n; i++ {
-		require.Equal(t, results[i], i*2)
+		require.Equal(t, results[i], i*2, "Some jobs didn't run")
 	}
+	require.Equal(t, int64(5), max, "Wrong number of goroutines spawned")
 }
 
 func TestTimeout(t *testing.T) {
+	// Don't run test on 32-bit machines due to bug in simulator.
+	// https://github.com/vmware/govmomi/issues/1330
+	var i int
+	if unsafe.Sizeof(i) < 8 {
+		return
+	}
+
 	m, s, err := createSim()
 	if err != nil {
 		t.Fatal(err)
@@ -245,7 +261,7 @@ func TestTimeout(t *testing.T) {
 	require.NoError(t, v.Start(nil)) // We're not using the Accumulator, so it can be nil.
 	defer v.Stop()
 	err = v.Gather(&acc)
-	require.NotNil(t, err, "Error should not be nil here")
+	require.True(t, len(acc.Errors) > 0, "Errors should not be empty here")
 
 	// The accumulator must contain exactly one error and it must be a deadline exceeded.
 	require.Equal(t, 1, len(acc.Errors))
@@ -253,6 +269,12 @@ func TestTimeout(t *testing.T) {
 }
 
 func TestMaxQuery(t *testing.T) {
+	// Don't run test on 32-bit machines due to bug in simulator.
+	// https://github.com/vmware/govmomi/issues/1330
+	var i int
+	if unsafe.Sizeof(i) < 8 {
+		return
+	}
 	m, s, err := createSim()
 	if err != nil {
 		t.Fatal(err)
@@ -290,6 +312,13 @@ func TestMaxQuery(t *testing.T) {
 }
 
 func TestAll(t *testing.T) {
+	// Don't run test on 32-bit machines due to bug in simulator.
+	// https://github.com/vmware/govmomi/issues/1330
+	var i int
+	if unsafe.Sizeof(i) < 8 {
+		return
+	}
+
 	m, s, err := createSim()
 	if err != nil {
 		t.Fatal(err)
@@ -300,7 +329,8 @@ func TestAll(t *testing.T) {
 	var acc testutil.Accumulator
 	v := defaultVSphere()
 	v.Vcenters = []string{s.URL.String()}
-	v.Start(nil) // We're not using the Accumulator, so it can be nil.
+	v.Start(&acc)
 	defer v.Stop()
 	require.NoError(t, v.Gather(&acc))
+	require.Equal(t, 0, len(acc.Errors), fmt.Sprintf("Errors found: %s", acc.Errors))
 }