Add per-host shard metrics in mongodb input (#3819)

This commit is contained in:
Jake Champlin 2018-04-10 20:10:29 -04:00 committed by Daniel Nelson
parent d44b3f6839
commit 0fc8724f88
5 changed files with 121 additions and 4 deletions

View File

@ -109,8 +109,18 @@ Error in input [mongodb]: not authorized on admin to execute command { serverSta
- storage_size (integer)
- type (string)
- mongodb_shard_stats
- tags:
- hostname
- fields:
- in_use (integer)
- available (integer)
- created (integer)
- refreshing (integer)
### Example Output:
```
mongodb,hostname=127.0.0.1:27017 active_reads=0i,active_writes=0i,commands_per_sec=6i,deletes_per_sec=0i,flushes_per_sec=0i,getmores_per_sec=1i,inserts_per_sec=0i,jumbo_chunks=0i,member_status="PRI",net_in_bytes=851i,net_out_bytes=23904i,open_connections=6i,percent_cache_dirty=0,percent_cache_used=0,queries_per_sec=2i,queued_reads=0i,queued_writes=0i,repl_commands_per_sec=0i,repl_deletes_per_sec=0i,repl_getmores_per_sec=0i,repl_inserts_per_sec=0i,repl_lag=0i,repl_queries_per_sec=0i,repl_updates_per_sec=0i,resident_megabytes=67i,state="PRIMARY",total_available=0i,total_created=0i,total_in_use=0i,total_refreshing=0i,ttl_deletes_per_sec=0i,ttl_passes_per_sec=0i,updates_per_sec=0i,vsize_megabytes=729i,wtcache_app_threads_page_read_count=4i,wtcache_app_threads_page_read_time=18i,wtcache_app_threads_page_write_count=6i,wtcache_bytes_read_into=10075i,wtcache_bytes_written_from=115711i,wtcache_current_bytes=86038i,wtcache_max_bytes_configured=1073741824i,wtcache_pages_evicted_by_app_thread=0i,wtcache_pages_queued_for_eviction=0i,wtcache_server_evicting_pages=0i,wtcache_tracked_dirty_bytes=0i,wtcache_worker_thread_evictingpages=0i 1522798796000000000
mongodb_db_stats,db_name=local,hostname=127.0.0.1:27017 avg_obj_size=818.625,collections=5i,data_size=6549i,index_size=86016i,indexes=4i,num_extents=0i,objects=8i,ok=1i,storage_size=118784i,type="db_stat" 1522799074000000000
mongodb_shard_stats,hostname=127.0.0.1:27017,in_use=3i,available=3i,created=4i,refreshing=0i 1522799074000000000
```

View File

@ -9,10 +9,11 @@ import (
)
type MongodbData struct {
StatLine *StatLine
Fields map[string]interface{}
Tags map[string]string
DbData []DbData
StatLine *StatLine
Fields map[string]interface{}
Tags map[string]string
DbData []DbData
ShardHostData []DbData
}
type DbData struct {
@ -74,6 +75,13 @@ var DefaultShardStats = map[string]string{
"total_refreshing": "TotalRefreshing",
}
var ShardHostStats = map[string]string{
"in_use": "InUse",
"available": "Available",
"created": "Created",
"refreshing": "Refreshing",
}
var MmapStats = map[string]string{
"mapped_megabytes": "Mapped",
"non-mapped_megabytes": "NonMapped",
@ -128,6 +136,22 @@ func (d *MongodbData) AddDbStats() {
}
}
func (d *MongodbData) AddShardHostStats() {
for host, hostStat := range d.StatLine.ShardHostStatsLines {
hostStatLine := reflect.ValueOf(&hostStat).Elem()
newDbData := &DbData{
Name: host,
Fields: make(map[string]interface{}),
}
newDbData.Fields["type"] = "shard_host_stat"
for k, v := range ShardHostStats {
val := hostStatLine.FieldByName(v).Interface()
newDbData.Fields[k] = val
}
d.ShardHostData = append(d.ShardHostData, *newDbData)
}
}
func (d *MongodbData) AddDefaultStats() {
statLine := reflect.ValueOf(d.StatLine).Elem()
d.addStat(statLine, DefaultStats)
@ -179,4 +203,14 @@ func (d *MongodbData) flush(acc telegraf.Accumulator) {
)
db.Fields = make(map[string]interface{})
}
for _, host := range d.ShardHostData {
d.Tags["hostname"] = host.Name
acc.AddFields(
"mongodb_shard_stats",
host.Fields,
d.Tags,
d.StatLine.Time,
)
host.Fields = make(map[string]interface{})
}
}

View File

@ -120,6 +120,42 @@ func TestAddShardStats(t *testing.T) {
}
}
func TestAddShardHostStats(t *testing.T) {
expectedHosts := []string{"hostA", "hostB"}
hostStatLines := map[string]ShardHostStatLine{}
for _, host := range expectedHosts {
hostStatLines[host] = ShardHostStatLine{
InUse: 0,
Available: 0,
Created: 0,
Refreshing: 0,
}
}
d := NewMongodbData(
&StatLine{
ShardHostStatsLines: hostStatLines,
},
map[string]string{}, // Use empty tags, so we don't break existing tests
)
var acc testutil.Accumulator
d.AddShardHostStats()
d.flush(&acc)
var hostsFound []string
for host, _ := range hostStatLines {
for key, _ := range ShardHostStats {
assert.True(t, acc.HasInt64Field("mongodb_shard_stats", key))
}
assert.True(t, acc.HasTag("mongodb_shard_stats", "hostname"))
hostsFound = append(hostsFound, host)
}
assert.Equal(t, hostsFound, expectedHosts)
}
func TestStateTag(t *testing.T) {
d := NewMongodbData(
&StatLine{

View File

@ -156,6 +156,7 @@ func (s *Server) gatherData(acc telegraf.Accumulator, gatherDbStats bool) error
)
data.AddDefaultStats()
data.AddDbStats()
data.AddShardHostStats()
data.flush(acc)
}
return nil

View File

@ -125,12 +125,27 @@ type WiredTiger struct {
// ShardStats stores information from shardConnPoolStats.
type ShardStats struct {
ShardStatsData `bson:",inline"`
Hosts map[string]ShardHostStatsData `bson:"hosts"`
}
// ShardStatsData is the total Shard Stats from shardConnPoolStats database command.
type ShardStatsData struct {
TotalInUse int64 `bson:"totalInUse"`
TotalAvailable int64 `bson:"totalAvailable"`
TotalCreated int64 `bson:"totalCreated"`
TotalRefreshing int64 `bson:"totalRefreshing"`
}
// ShardHostStatsData is the host-specific stats
// from shardConnPoolStats database command.
type ShardHostStatsData struct {
InUse int64 `bson:"inUse"`
Available int64 `bson:"available"`
Created int64 `bson:"created"`
Refreshing int64 `bson:"refreshing"`
}
type ConcurrentTransactions struct {
Write ConcurrentTransStats `bson:"write"`
Read ConcurrentTransStats `bson:"read"`
@ -469,6 +484,9 @@ type StatLine struct {
// Shard stats
TotalInUse, TotalAvailable, TotalCreated, TotalRefreshing int64
// Shard Hosts stats field
ShardHostStatsLines map[string]ShardHostStatLine
}
type DbStatLine struct {
@ -484,6 +502,13 @@ type DbStatLine struct {
Ok int64
}
type ShardHostStatLine struct {
InUse int64
Available int64
Created int64
Refreshing int64
}
func parseLocks(stat ServerStatus) map[string]LockUsage {
returnVal := map[string]LockUsage{}
for namespace, lockInfo := range stat.Locks {
@ -809,6 +834,17 @@ func NewStatLine(oldMongo, newMongo MongoStatus, key string, all bool, sampleSec
returnVal.TotalAvailable = newShardStats.TotalAvailable
returnVal.TotalCreated = newShardStats.TotalCreated
returnVal.TotalRefreshing = newShardStats.TotalRefreshing
returnVal.ShardHostStatsLines = map[string]ShardHostStatLine{}
for host, stats := range newShardStats.Hosts {
shardStatLine := &ShardHostStatLine{
InUse: stats.InUse,
Available: stats.Available,
Created: stats.Created,
Refreshing: stats.Refreshing,
}
returnVal.ShardHostStatsLines[host] = *shardStatLine
}
return returnVal
}