Add per-host shard metrics in mongodb input (#3819)

plugins/inputs/mongodb/README.md

@@ -109,8 +109,18 @@ Error in input [mongodb]: not authorized on admin to execute command { serverSta
     - storage_size (integer)
     - type (string)
 
+- mongodb_shard_stats
+  - tags:
+    - hostname
+  - fields:
+    - in_use (integer)
+    - available (integer)
+    - created (integer)
+    - refreshing (integer)
+
 ### Example Output:
 ```
 mongodb,hostname=127.0.0.1:27017 active_reads=0i,active_writes=0i,commands_per_sec=6i,deletes_per_sec=0i,flushes_per_sec=0i,getmores_per_sec=1i,inserts_per_sec=0i,jumbo_chunks=0i,member_status="PRI",net_in_bytes=851i,net_out_bytes=23904i,open_connections=6i,percent_cache_dirty=0,percent_cache_used=0,queries_per_sec=2i,queued_reads=0i,queued_writes=0i,repl_commands_per_sec=0i,repl_deletes_per_sec=0i,repl_getmores_per_sec=0i,repl_inserts_per_sec=0i,repl_lag=0i,repl_queries_per_sec=0i,repl_updates_per_sec=0i,resident_megabytes=67i,state="PRIMARY",total_available=0i,total_created=0i,total_in_use=0i,total_refreshing=0i,ttl_deletes_per_sec=0i,ttl_passes_per_sec=0i,updates_per_sec=0i,vsize_megabytes=729i,wtcache_app_threads_page_read_count=4i,wtcache_app_threads_page_read_time=18i,wtcache_app_threads_page_write_count=6i,wtcache_bytes_read_into=10075i,wtcache_bytes_written_from=115711i,wtcache_current_bytes=86038i,wtcache_max_bytes_configured=1073741824i,wtcache_pages_evicted_by_app_thread=0i,wtcache_pages_queued_for_eviction=0i,wtcache_server_evicting_pages=0i,wtcache_tracked_dirty_bytes=0i,wtcache_worker_thread_evictingpages=0i 1522798796000000000
 mongodb_db_stats,db_name=local,hostname=127.0.0.1:27017 avg_obj_size=818.625,collections=5i,data_size=6549i,index_size=86016i,indexes=4i,num_extents=0i,objects=8i,ok=1i,storage_size=118784i,type="db_stat" 1522799074000000000
+mongodb_shard_stats,hostname=127.0.0.1:27017,in_use=3i,available=3i,created=4i,refreshing=0i 1522799074000000000
 ```

plugins/inputs/mongodb/mongodb_data.go

@@ -9,10 +9,11 @@ import (
 )
 
 type MongodbData struct {
-	StatLine *StatLine
+	StatLine      *StatLine
-	Fields   map[string]interface{}
+	Fields        map[string]interface{}
-	Tags     map[string]string
+	Tags          map[string]string
-	DbData   []DbData
+	DbData        []DbData
+	ShardHostData []DbData
 }
 
 type DbData struct {
@@ -74,6 +75,13 @@ var DefaultShardStats = map[string]string{
 	"total_refreshing": "TotalRefreshing",
 }
 
+var ShardHostStats = map[string]string{
+	"in_use":     "InUse",
+	"available":  "Available",
+	"created":    "Created",
+	"refreshing": "Refreshing",
+}
+
 var MmapStats = map[string]string{
 	"mapped_megabytes":     "Mapped",
 	"non-mapped_megabytes": "NonMapped",
@@ -128,6 +136,22 @@ func (d *MongodbData) AddDbStats() {
 	}
 }
 
+func (d *MongodbData) AddShardHostStats() {
+	for host, hostStat := range d.StatLine.ShardHostStatsLines {
+		hostStatLine := reflect.ValueOf(&hostStat).Elem()
+		newDbData := &DbData{
+			Name:   host,
+			Fields: make(map[string]interface{}),
+		}
+		newDbData.Fields["type"] = "shard_host_stat"
+		for k, v := range ShardHostStats {
+			val := hostStatLine.FieldByName(v).Interface()
+			newDbData.Fields[k] = val
+		}
+		d.ShardHostData = append(d.ShardHostData, *newDbData)
+	}
+}
+
 func (d *MongodbData) AddDefaultStats() {
 	statLine := reflect.ValueOf(d.StatLine).Elem()
 	d.addStat(statLine, DefaultStats)
@@ -179,4 +203,14 @@ func (d *MongodbData) flush(acc telegraf.Accumulator) {
 		)
 		db.Fields = make(map[string]interface{})
 	}
+	for _, host := range d.ShardHostData {
+		d.Tags["hostname"] = host.Name
+		acc.AddFields(
+			"mongodb_shard_stats",
+			host.Fields,
+			d.Tags,
+			d.StatLine.Time,
+		)
+		host.Fields = make(map[string]interface{})
+	}
 }

plugins/inputs/mongodb/mongodb_data_test.go

@@ -120,6 +120,42 @@ func TestAddShardStats(t *testing.T) {
 	}
 }
 
+func TestAddShardHostStats(t *testing.T) {
+	expectedHosts := []string{"hostA", "hostB"}
+	hostStatLines := map[string]ShardHostStatLine{}
+	for _, host := range expectedHosts {
+		hostStatLines[host] = ShardHostStatLine{
+			InUse:      0,
+			Available:  0,
+			Created:    0,
+			Refreshing: 0,
+		}
+	}
+
+	d := NewMongodbData(
+		&StatLine{
+			ShardHostStatsLines: hostStatLines,
+		},
+		map[string]string{}, // Use empty tags, so we don't break existing tests
+	)
+
+	var acc testutil.Accumulator
+	d.AddShardHostStats()
+	d.flush(&acc)
+
+	var hostsFound []string
+	for host, _ := range hostStatLines {
+		for key, _ := range ShardHostStats {
+			assert.True(t, acc.HasInt64Field("mongodb_shard_stats", key))
+		}
+
+		assert.True(t, acc.HasTag("mongodb_shard_stats", "hostname"))
+		hostsFound = append(hostsFound, host)
+	}
+
+	assert.Equal(t, hostsFound, expectedHosts)
+}
+
 func TestStateTag(t *testing.T) {
 	d := NewMongodbData(
 		&StatLine{

plugins/inputs/mongodb/mongodb_server.go

@@ -156,6 +156,7 @@ func (s *Server) gatherData(acc telegraf.Accumulator, gatherDbStats bool) error
 		)
 		data.AddDefaultStats()
 		data.AddDbStats()
+		data.AddShardHostStats()
 		data.flush(acc)
 	}
 	return nil

plugins/inputs/mongodb/mongostat.go

@@ -125,12 +125,27 @@ type WiredTiger struct {
 
 // ShardStats stores information from shardConnPoolStats.
 type ShardStats struct {
+	ShardStatsData `bson:",inline"`
+	Hosts          map[string]ShardHostStatsData `bson:"hosts"`
+}
+
+// ShardStatsData is the total Shard Stats from shardConnPoolStats database command.
+type ShardStatsData struct {
 	TotalInUse      int64 `bson:"totalInUse"`
 	TotalAvailable  int64 `bson:"totalAvailable"`
 	TotalCreated    int64 `bson:"totalCreated"`
 	TotalRefreshing int64 `bson:"totalRefreshing"`
 }
 
+// ShardHostStatsData is the host-specific stats
+// from shardConnPoolStats database command.
+type ShardHostStatsData struct {
+	InUse      int64 `bson:"inUse"`
+	Available  int64 `bson:"available"`
+	Created    int64 `bson:"created"`
+	Refreshing int64 `bson:"refreshing"`
+}
+
 type ConcurrentTransactions struct {
 	Write ConcurrentTransStats `bson:"write"`
 	Read  ConcurrentTransStats `bson:"read"`
@@ -469,6 +484,9 @@ type StatLine struct {
 
 	// Shard stats
 	TotalInUse, TotalAvailable, TotalCreated, TotalRefreshing int64
+
+	// Shard Hosts stats field
+	ShardHostStatsLines map[string]ShardHostStatLine
 }
 
 type DbStatLine struct {
@@ -484,6 +502,13 @@ type DbStatLine struct {
 	Ok          int64
 }
 
+type ShardHostStatLine struct {
+	InUse      int64
+	Available  int64
+	Created    int64
+	Refreshing int64
+}
+
 func parseLocks(stat ServerStatus) map[string]LockUsage {
 	returnVal := map[string]LockUsage{}
 	for namespace, lockInfo := range stat.Locks {
@@ -809,6 +834,17 @@ func NewStatLine(oldMongo, newMongo MongoStatus, key string, all bool, sampleSec
 	returnVal.TotalAvailable = newShardStats.TotalAvailable
 	returnVal.TotalCreated = newShardStats.TotalCreated
 	returnVal.TotalRefreshing = newShardStats.TotalRefreshing
+	returnVal.ShardHostStatsLines = map[string]ShardHostStatLine{}
+	for host, stats := range newShardStats.Hosts {
+		shardStatLine := &ShardHostStatLine{
+			InUse:      stats.InUse,
+			Available:  stats.Available,
+			Created:    stats.Created,
+			Refreshing: stats.Refreshing,
+		}
+
+		returnVal.ShardHostStatsLines[host] = *shardStatLine
+	}
 
 	return returnVal
 }