dom
/
telegraf
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Add latency stats to mongo input (#6733)

This commit is contained in:
Ross Lodge 2020-01-08 18:18:12 -08:00 committed by Daniel Nelson
parent 695678c4a5
commit 6f9f0b7f01
5 changed files with 285 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
plugins/inputs/mongodb/README.md
View File

@ -77,6 +77,12 @@ by running Telegraf with the `--debug` argument.
- getmores (integer)
- inserts (integer)
- jumbo_chunks (integer)
- latency_commands_count (integer)
- latency_commands (integer)
- latency_reads_count (integer)
- latency_reads (integer)
- latency_writes_count (integer)
- latency_writes (integer)
- member_status (string)
- net_in_bytes_count (integer)
- net_out_bytes_count (integer)
@ -185,6 +191,7 @@ by running Telegraf with the `--debug` argument.
### Example Output:
```
mongodb,hostname=127.0.0.1:27017 active_reads=0i,active_writes=0i,commands=1335i,commands_per_sec=7i,connections_available=814i,connections_current=5i,connections_total_created=0i,cursor_no_timeout=0i,cursor_no_timeout_count=0i,cursor_pinned=0i,cursor_pinned_count=1i,cursor_timed_out=0i,cursor_timed_out_count=0i,cursor_total=0i,cursor_total_count=1i,deletes=0i,deletes_per_sec=0i,document_deleted=0i,document_inserted=0i,document_returned=13i,document_updated=0i,flushes=5i,flushes_per_sec=0i,getmores=269i,getmores_per_sec=0i,inserts=0i,inserts_per_sec=0i,jumbo_chunks=0i,latency_commands_count=0i,latency_commands=0i,latency_reads_count=0i,latency_reads=0i,latency_writes_count=0i,latency_writes=0i,member_status="PRI",net_in_bytes=986i,net_in_bytes_count=358006i,net_out_bytes=23906i,net_out_bytes_count=661507i,open_connections=5i,percent_cache_dirty=0,percent_cache_used=0,queries=18i,queries_per_sec=3i,queued_reads=0i,queued_writes=0i,repl_commands=0i,repl_commands_per_sec=0i,repl_deletes=0i,repl_deletes_per_sec=0i,repl_getmores=0i,repl_getmores_per_sec=0i,repl_inserts=0i,repl_inserts_per_sec=0i,repl_lag=0i,repl_oplog_window_sec=24355215i,repl_queries=0i,repl_queries_per_sec=0i,repl_updates=0i,repl_updates_per_sec=0i,resident_megabytes=62i,state="PRIMARY",total_available=0i,total_created=0i,total_in_use=0i,total_refreshing=0i,ttl_deletes=0i,ttl_deletes_per_sec=0i,ttl_passes=23i,ttl_passes_per_sec=0i,updates=0i,updates_per_sec=0i,vsize_megabytes=713i,wtcache_app_threads_page_read_count=13i,wtcache_app_threads_page_read_time=74i,wtcache_app_threads_page_write_count=0i,wtcache_bytes_read_into=55271i,wtcache_bytes_written_from=125402i,wtcache_current_bytes=117050i,wtcache_max_bytes_configured=1073741824i,wtcache_pages_evicted_by_app_thread=0i,wtcache_pages_queued_for_eviction=0i,wtcache_server_evicting_pages=0i,wtcache_tracked_dirty_bytes=0i,wtcache_worker_thread_evictingpages=0i 1547159491000000000
mongodb,hostname=127.0.0.1:27017,node_type=PRI active_reads=0i,active_writes=0i,commands=1335i,commands_per_sec=7i,connections_available=814i,connections_current=5i,connections_total_created=0i,cursor_no_timeout=0i,cursor_no_timeout_count=0i,cursor_pinned=0i,cursor_pinned_count=1i,cursor_timed_out=0i,cursor_timed_out_count=0i,cursor_total=0i,cursor_total_count=1i,deletes=0i,deletes_per_sec=0i,document_deleted=0i,document_inserted=0i,document_returned=13i,document_updated=0i,flushes=5i,flushes_per_sec=0i,getmores=269i,getmores_per_sec=0i,inserts=0i,inserts_per_sec=0i,jumbo_chunks=0i,member_status="PRI",net_in_bytes=986i,net_in_bytes_count=358006i,net_out_bytes=23906i,net_out_bytes_count=661507i,open_connections=5i,percent_cache_dirty=0,percent_cache_used=0,queries=18i,queries_per_sec=3i,queued_reads=0i,queued_writes=0i,repl_commands=0i,repl_commands_per_sec=0i,repl_deletes=0i,repl_deletes_per_sec=0i,repl_getmores=0i,repl_getmores_per_sec=0i,repl_inserts=0i,repl_inserts_per_sec=0i,repl_lag=0i,repl_oplog_window_sec=24355215i,repl_queries=0i,repl_queries_per_sec=0i,repl_updates=0i,repl_updates_per_sec=0i,resident_megabytes=62i,state="PRIMARY",total_available=0i,total_created=0i,total_in_use=0i,total_refreshing=0i,ttl_deletes=0i,ttl_deletes_per_sec=0i,ttl_passes=23i,ttl_passes_per_sec=0i,updates=0i,updates_per_sec=0i,vsize_megabytes=713i,wtcache_app_threads_page_read_count=13i,wtcache_app_threads_page_read_time=74i,wtcache_app_threads_page_write_count=0i,wtcache_bytes_read_into=55271i,wtcache_bytes_written_from=125402i,wtcache_current_bytes=117050i,wtcache_max_bytes_configured=1073741824i,wtcache_pages_evicted_by_app_thread=0i,wtcache_pages_queued_for_eviction=0i,wtcache_server_evicting_pages=0i,wtcache_tracked_dirty_bytes=0i,wtcache_worker_thread_evictingpages=0i 1547159491000000000
mongodb_db_stats,db_name=admin,hostname=127.0.0.1:27017 avg_obj_size=241,collections=2i,data_size=723i,index_size=49152i,indexes=3i,num_extents=0i,objects=3i,ok=1i,storage_size=53248i,type="db_stat" 1547159491000000000
mongodb_db_stats,db_name=local,hostname=127.0.0.1:27017 avg_obj_size=813.9705882352941,collections=6i,data_size=55350i,index_size=102400i,indexes=5i,num_extents=0i,objects=68i,ok=1i,storage_size=204800i,type="db_stat" 1547159491000000000

13
plugins/inputs/mongodb/mongodb_data.go
View File

@ -86,6 +86,15 @@ var DefaultStats = map[string]string{
"connections_total_created": "TotalCreatedC",
}
var DefaultLatencyStats = map[string]string{
"latency_writes_count": "WriteOpsCnt",
"latency_writes": "WriteLatency",
"latency_reads_count": "ReadOpsCnt",
"latency_reads": "ReadLatency",
"latency_commands_count": "CommandOpsCnt",
"latency_commands": "CommandLatency",
}
var DefaultReplStats = map[string]string{
"repl_inserts": "InsertRCnt",
"repl_inserts_per_sec": "InsertR",
@ -232,6 +241,10 @@ func (d *MongodbData) AddDefaultStats() {
d.Tags["node_type"] = d.StatLine.NodeType
}
if d.StatLine.ReadLatency > 0 {
d.addStat(statLine, DefaultLatencyStats)
}
if d.StatLine.OplogStats != nil {
d.add("repl_oplog_window_sec", d.StatLine.OplogStats.TimeDiff)
}

23
plugins/inputs/mongodb/mongodb_data_test.go
View File

@ -142,6 +142,29 @@ func TestAddShardStats(t *testing.T) {
}
}
func TestAddLatencyStats(t *testing.T) {
d := NewMongodbData(
&StatLine{
CommandOpsCnt: 73,
CommandLatency: 364,
ReadOpsCnt: 113,
ReadLatency: 201,
WriteOpsCnt: 7,
WriteLatency: 55,
},
tags,
)
var acc testutil.Accumulator
d.AddDefaultStats()
d.flush(&acc)
for key := range DefaultLatencyStats {
assert.True(t, acc.HasInt64Field("mongodb", key))
}
}
func TestAddShardHostStats(t *testing.T) {
expectedHosts := []string{"hostA", "hostB"}
hostStatLines := map[string]ShardHostStatLine{}

37
plugins/inputs/mongodb/mongostat.go
View File

@ -58,6 +58,7 @@ type ServerStatus struct {
Network *NetworkStats `bson:"network"`
Opcounters *OpcountStats `bson:"opcounters"`
OpcountersRepl *OpcountStats `bson:"opcountersRepl"`
OpLatencies *OpLatenciesStats `bson:"opLatencies"`
RecordStats *DBRecordStats `bson:"recordStats"`
Mem *MemStats `bson:"mem"`
Repl *ReplStatus `bson:"repl"`
@ -314,6 +315,19 @@ type OpcountStats struct {
Command int64 `bson:"command"`
}
// OpLatenciesStats stores information related to operation latencies for the database as a whole
type OpLatenciesStats struct {
Reads *LatencyStats `bson:"reads"`
Writes *LatencyStats `bson:"writes"`
Commands *LatencyStats `bson:"commands"`
}
// LatencyStats lists total latency in microseconds and count of operations, enabling you to obtain an average
type LatencyStats struct {
Latency int64 `bson:"latency"`
Ops int64 `bson:"ops"`
}
// MetricsStats stores information related to metrics
type MetricsStats struct {
TTL *TTLStats `bson:"ttl"`
@ -493,6 +507,14 @@ type StatLine struct {
GetMore, GetMoreCnt int64
Command, CommandCnt int64
// OpLatency fields
WriteOpsCnt int64
WriteLatency int64
ReadOpsCnt int64
ReadLatency int64
CommandOpsCnt int64
CommandLatency int64
// TTL fields
Passes, PassesCnt int64
DeletedDocuments, DeletedDocumentsCnt int64
@ -684,6 +706,21 @@ func NewStatLine(oldMongo, newMongo MongoStatus, key string, all bool, sampleSec
returnVal.Command, returnVal.CommandCnt = diff(newStat.Opcounters.Command, oldStat.Opcounters.Command, sampleSecs)
}
if newStat.OpLatencies != nil {
if newStat.OpLatencies.Reads != nil {
returnVal.ReadOpsCnt = newStat.OpLatencies.Reads.Ops
returnVal.ReadLatency = newStat.OpLatencies.Reads.Latency
}
if newStat.OpLatencies.Writes != nil {
returnVal.WriteOpsCnt = newStat.OpLatencies.Writes.Ops
returnVal.WriteLatency = newStat.OpLatencies.Writes.Latency
}
if newStat.OpLatencies.Commands != nil {
returnVal.CommandOpsCnt = newStat.OpLatencies.Commands.Ops
returnVal.CommandLatency = newStat.OpLatencies.Commands.Latency
}
}
if newStat.Metrics != nil && oldStat.Metrics != nil {
if newStat.Metrics.TTL != nil && oldStat.Metrics.TTL != nil {
returnVal.Passes, returnVal.PassesCnt = diff(newStat.Metrics.TTL.Passes, oldStat.Metrics.TTL.Passes, sampleSecs)

205
plugins/inputs/mongodb/mongostat_test.go Normal file
View File

@ -0,0 +1,205 @@
package mongodb
import (
"testing"
//"time"
//"github.com/influxdata/telegraf/testutil"
"github.com/stretchr/testify/assert"
)
func TestLatencyStats(t *testing.T) {
sl := NewStatLine(
MongoStatus{
ServerStatus: &ServerStatus{
Connections: &ConnectionStats{},
Mem: &MemStats{
Bits: 0,
Resident: 0,
Virtual: 0,
Supported: false,
Mapped: 0,
MappedWithJournal: 0,
},
},
},
MongoStatus{
ServerStatus: &ServerStatus{
Connections: &ConnectionStats{},
Mem: &MemStats{
Bits: 0,
Resident: 0,
Virtual: 0,
Supported: false,
Mapped: 0,
MappedWithJournal: 0,
},
OpLatencies: &OpLatenciesStats{
Reads: &LatencyStats{
Ops: 0,
Latency: 0,
},
Writes: &LatencyStats{
Ops: 0,
Latency: 0,
},
Commands: &LatencyStats{
Ops: 0,
Latency: 0,
},
},
},
},
"foo",
true,
60,
)
assert.Equal(t, sl.CommandLatency, int64(0))
assert.Equal(t, sl.ReadLatency, int64(0))
assert.Equal(t, sl.WriteLatency, int64(0))
assert.Equal(t, sl.CommandOpsCnt, int64(0))
assert.Equal(t, sl.ReadOpsCnt, int64(0))
assert.Equal(t, sl.WriteOpsCnt, int64(0))
}
func TestLatencyStatsDiffZero(t *testing.T) {
sl := NewStatLine(
MongoStatus{
ServerStatus: &ServerStatus{
Connections: &ConnectionStats{},
Mem: &MemStats{
Bits: 0,
Resident: 0,
Virtual: 0,
Supported: false,
Mapped: 0,
MappedWithJournal: 0,
},
OpLatencies: &OpLatenciesStats{
Reads: &LatencyStats{
Ops: 0,
Latency: 0,
},
Writes: &LatencyStats{
Ops: 0,
Latency: 0,
},
Commands: &LatencyStats{
Ops: 0,
Latency: 0,
},
},
},
},
MongoStatus{
ServerStatus: &ServerStatus{
Connections: &ConnectionStats{},
Mem: &MemStats{
Bits: 0,
Resident: 0,
Virtual: 0,
Supported: false,
Mapped: 0,
MappedWithJournal: 0,
},
OpLatencies: &OpLatenciesStats{
Reads: &LatencyStats{
Ops: 0,
Latency: 0,
},
Writes: &LatencyStats{
Ops: 0,
Latency: 0,
},
Commands: &LatencyStats{
Ops: 0,
Latency: 0,
},
},
},
},
"foo",
true,
60,
)
assert.Equal(t, sl.CommandLatency, int64(0))
assert.Equal(t, sl.ReadLatency, int64(0))
assert.Equal(t, sl.WriteLatency, int64(0))
assert.Equal(t, sl.CommandOpsCnt, int64(0))
assert.Equal(t, sl.ReadOpsCnt, int64(0))
assert.Equal(t, sl.WriteOpsCnt, int64(0))
}
func TestLatencyStatsDiff(t *testing.T) {
sl := NewStatLine(
MongoStatus{
ServerStatus: &ServerStatus{
Connections: &ConnectionStats{},
Mem: &MemStats{
Bits: 0,
Resident: 0,
Virtual: 0,
Supported: false,
Mapped: 0,
MappedWithJournal: 0,
},
OpLatencies: &OpLatenciesStats{
Reads: &LatencyStats{
Ops: 4189041956,
Latency: 2255922322753,
},
Writes: &LatencyStats{
Ops: 1691019457,
Latency: 494478256915,
},
Commands: &LatencyStats{
Ops: 1019150402,
Latency: 59177710371,
},
},
},
},
MongoStatus{
ServerStatus: &ServerStatus{
Connections: &ConnectionStats{},
Mem: &MemStats{
Bits: 0,
Resident: 0,
Virtual: 0,
Supported: false,
Mapped: 0,
MappedWithJournal: 0,
},
OpLatencies: &OpLatenciesStats{
Reads: &LatencyStats{
Ops: 4189049884,
Latency: 2255946760057,
},
Writes: &LatencyStats{
Ops: 1691021287,
Latency: 494479456987,
},
Commands: &LatencyStats{
Ops: 1019152861,
Latency: 59177981552,
},
},
},
},
"foo",
true,
60,
)
assert.Equal(t, sl.CommandLatency, int64(59177981552))
assert.Equal(t, sl.ReadLatency, int64(2255946760057))
assert.Equal(t, sl.WriteLatency, int64(494479456987))
assert.Equal(t, sl.CommandOpsCnt, int64(1019152861))
assert.Equal(t, sl.ReadOpsCnt, int64(4189049884))
assert.Equal(t, sl.WriteOpsCnt, int64(1691021287))
}