From 6f9f0b7f01e2e9ff6b73e266ec14e39c9c95ba78 Mon Sep 17 00:00:00 2001 From: Ross Lodge Date: Wed, 8 Jan 2020 18:18:12 -0800 Subject: [PATCH] Add latency stats to mongo input (#6733) --- plugins/inputs/mongodb/README.md | 7 + plugins/inputs/mongodb/mongodb_data.go | 13 ++ plugins/inputs/mongodb/mongodb_data_test.go | 23 +++ plugins/inputs/mongodb/mongostat.go | 37 ++++ plugins/inputs/mongodb/mongostat_test.go | 205 ++++++++++++++++++++ 5 files changed, 285 insertions(+) create mode 100644 plugins/inputs/mongodb/mongostat_test.go diff --git a/plugins/inputs/mongodb/README.md b/plugins/inputs/mongodb/README.md index f154f333b..5449e3b4b 100644 --- a/plugins/inputs/mongodb/README.md +++ b/plugins/inputs/mongodb/README.md @@ -77,6 +77,12 @@ by running Telegraf with the `--debug` argument. - getmores (integer) - inserts (integer) - jumbo_chunks (integer) + - latency_commands_count (integer) + - latency_commands (integer) + - latency_reads_count (integer) + - latency_reads (integer) + - latency_writes_count (integer) + - latency_writes (integer) - member_status (string) - net_in_bytes_count (integer) - net_out_bytes_count (integer) @@ -185,6 +191,7 @@ by running Telegraf with the `--debug` argument. ### Example Output: ``` +mongodb,hostname=127.0.0.1:27017 active_reads=0i,active_writes=0i,commands=1335i,commands_per_sec=7i,connections_available=814i,connections_current=5i,connections_total_created=0i,cursor_no_timeout=0i,cursor_no_timeout_count=0i,cursor_pinned=0i,cursor_pinned_count=1i,cursor_timed_out=0i,cursor_timed_out_count=0i,cursor_total=0i,cursor_total_count=1i,deletes=0i,deletes_per_sec=0i,document_deleted=0i,document_inserted=0i,document_returned=13i,document_updated=0i,flushes=5i,flushes_per_sec=0i,getmores=269i,getmores_per_sec=0i,inserts=0i,inserts_per_sec=0i,jumbo_chunks=0i,latency_commands_count=0i,latency_commands=0i,latency_reads_count=0i,latency_reads=0i,latency_writes_count=0i,latency_writes=0i,member_status="PRI",net_in_bytes=986i,net_in_bytes_count=358006i,net_out_bytes=23906i,net_out_bytes_count=661507i,open_connections=5i,percent_cache_dirty=0,percent_cache_used=0,queries=18i,queries_per_sec=3i,queued_reads=0i,queued_writes=0i,repl_commands=0i,repl_commands_per_sec=0i,repl_deletes=0i,repl_deletes_per_sec=0i,repl_getmores=0i,repl_getmores_per_sec=0i,repl_inserts=0i,repl_inserts_per_sec=0i,repl_lag=0i,repl_oplog_window_sec=24355215i,repl_queries=0i,repl_queries_per_sec=0i,repl_updates=0i,repl_updates_per_sec=0i,resident_megabytes=62i,state="PRIMARY",total_available=0i,total_created=0i,total_in_use=0i,total_refreshing=0i,ttl_deletes=0i,ttl_deletes_per_sec=0i,ttl_passes=23i,ttl_passes_per_sec=0i,updates=0i,updates_per_sec=0i,vsize_megabytes=713i,wtcache_app_threads_page_read_count=13i,wtcache_app_threads_page_read_time=74i,wtcache_app_threads_page_write_count=0i,wtcache_bytes_read_into=55271i,wtcache_bytes_written_from=125402i,wtcache_current_bytes=117050i,wtcache_max_bytes_configured=1073741824i,wtcache_pages_evicted_by_app_thread=0i,wtcache_pages_queued_for_eviction=0i,wtcache_server_evicting_pages=0i,wtcache_tracked_dirty_bytes=0i,wtcache_worker_thread_evictingpages=0i 1547159491000000000 mongodb,hostname=127.0.0.1:27017,node_type=PRI active_reads=0i,active_writes=0i,commands=1335i,commands_per_sec=7i,connections_available=814i,connections_current=5i,connections_total_created=0i,cursor_no_timeout=0i,cursor_no_timeout_count=0i,cursor_pinned=0i,cursor_pinned_count=1i,cursor_timed_out=0i,cursor_timed_out_count=0i,cursor_total=0i,cursor_total_count=1i,deletes=0i,deletes_per_sec=0i,document_deleted=0i,document_inserted=0i,document_returned=13i,document_updated=0i,flushes=5i,flushes_per_sec=0i,getmores=269i,getmores_per_sec=0i,inserts=0i,inserts_per_sec=0i,jumbo_chunks=0i,member_status="PRI",net_in_bytes=986i,net_in_bytes_count=358006i,net_out_bytes=23906i,net_out_bytes_count=661507i,open_connections=5i,percent_cache_dirty=0,percent_cache_used=0,queries=18i,queries_per_sec=3i,queued_reads=0i,queued_writes=0i,repl_commands=0i,repl_commands_per_sec=0i,repl_deletes=0i,repl_deletes_per_sec=0i,repl_getmores=0i,repl_getmores_per_sec=0i,repl_inserts=0i,repl_inserts_per_sec=0i,repl_lag=0i,repl_oplog_window_sec=24355215i,repl_queries=0i,repl_queries_per_sec=0i,repl_updates=0i,repl_updates_per_sec=0i,resident_megabytes=62i,state="PRIMARY",total_available=0i,total_created=0i,total_in_use=0i,total_refreshing=0i,ttl_deletes=0i,ttl_deletes_per_sec=0i,ttl_passes=23i,ttl_passes_per_sec=0i,updates=0i,updates_per_sec=0i,vsize_megabytes=713i,wtcache_app_threads_page_read_count=13i,wtcache_app_threads_page_read_time=74i,wtcache_app_threads_page_write_count=0i,wtcache_bytes_read_into=55271i,wtcache_bytes_written_from=125402i,wtcache_current_bytes=117050i,wtcache_max_bytes_configured=1073741824i,wtcache_pages_evicted_by_app_thread=0i,wtcache_pages_queued_for_eviction=0i,wtcache_server_evicting_pages=0i,wtcache_tracked_dirty_bytes=0i,wtcache_worker_thread_evictingpages=0i 1547159491000000000 mongodb_db_stats,db_name=admin,hostname=127.0.0.1:27017 avg_obj_size=241,collections=2i,data_size=723i,index_size=49152i,indexes=3i,num_extents=0i,objects=3i,ok=1i,storage_size=53248i,type="db_stat" 1547159491000000000 mongodb_db_stats,db_name=local,hostname=127.0.0.1:27017 avg_obj_size=813.9705882352941,collections=6i,data_size=55350i,index_size=102400i,indexes=5i,num_extents=0i,objects=68i,ok=1i,storage_size=204800i,type="db_stat" 1547159491000000000 diff --git a/plugins/inputs/mongodb/mongodb_data.go b/plugins/inputs/mongodb/mongodb_data.go index 279dbb138..09bacdae1 100644 --- a/plugins/inputs/mongodb/mongodb_data.go +++ b/plugins/inputs/mongodb/mongodb_data.go @@ -86,6 +86,15 @@ var DefaultStats = map[string]string{ "connections_total_created": "TotalCreatedC", } +var DefaultLatencyStats = map[string]string{ + "latency_writes_count": "WriteOpsCnt", + "latency_writes": "WriteLatency", + "latency_reads_count": "ReadOpsCnt", + "latency_reads": "ReadLatency", + "latency_commands_count": "CommandOpsCnt", + "latency_commands": "CommandLatency", +} + var DefaultReplStats = map[string]string{ "repl_inserts": "InsertRCnt", "repl_inserts_per_sec": "InsertR", @@ -232,6 +241,10 @@ func (d *MongodbData) AddDefaultStats() { d.Tags["node_type"] = d.StatLine.NodeType } + if d.StatLine.ReadLatency > 0 { + d.addStat(statLine, DefaultLatencyStats) + } + if d.StatLine.OplogStats != nil { d.add("repl_oplog_window_sec", d.StatLine.OplogStats.TimeDiff) } diff --git a/plugins/inputs/mongodb/mongodb_data_test.go b/plugins/inputs/mongodb/mongodb_data_test.go index bbc882a26..711b3eef2 100644 --- a/plugins/inputs/mongodb/mongodb_data_test.go +++ b/plugins/inputs/mongodb/mongodb_data_test.go @@ -142,6 +142,29 @@ func TestAddShardStats(t *testing.T) { } } +func TestAddLatencyStats(t *testing.T) { + d := NewMongodbData( + &StatLine{ + CommandOpsCnt: 73, + CommandLatency: 364, + ReadOpsCnt: 113, + ReadLatency: 201, + WriteOpsCnt: 7, + WriteLatency: 55, + }, + tags, + ) + + var acc testutil.Accumulator + + d.AddDefaultStats() + d.flush(&acc) + + for key := range DefaultLatencyStats { + assert.True(t, acc.HasInt64Field("mongodb", key)) + } +} + func TestAddShardHostStats(t *testing.T) { expectedHosts := []string{"hostA", "hostB"} hostStatLines := map[string]ShardHostStatLine{} diff --git a/plugins/inputs/mongodb/mongostat.go b/plugins/inputs/mongodb/mongostat.go index 1658fc071..985627c87 100644 --- a/plugins/inputs/mongodb/mongostat.go +++ b/plugins/inputs/mongodb/mongostat.go @@ -58,6 +58,7 @@ type ServerStatus struct { Network *NetworkStats `bson:"network"` Opcounters *OpcountStats `bson:"opcounters"` OpcountersRepl *OpcountStats `bson:"opcountersRepl"` + OpLatencies *OpLatenciesStats `bson:"opLatencies"` RecordStats *DBRecordStats `bson:"recordStats"` Mem *MemStats `bson:"mem"` Repl *ReplStatus `bson:"repl"` @@ -314,6 +315,19 @@ type OpcountStats struct { Command int64 `bson:"command"` } +// OpLatenciesStats stores information related to operation latencies for the database as a whole +type OpLatenciesStats struct { + Reads *LatencyStats `bson:"reads"` + Writes *LatencyStats `bson:"writes"` + Commands *LatencyStats `bson:"commands"` +} + +// LatencyStats lists total latency in microseconds and count of operations, enabling you to obtain an average +type LatencyStats struct { + Latency int64 `bson:"latency"` + Ops int64 `bson:"ops"` +} + // MetricsStats stores information related to metrics type MetricsStats struct { TTL *TTLStats `bson:"ttl"` @@ -493,6 +507,14 @@ type StatLine struct { GetMore, GetMoreCnt int64 Command, CommandCnt int64 + // OpLatency fields + WriteOpsCnt int64 + WriteLatency int64 + ReadOpsCnt int64 + ReadLatency int64 + CommandOpsCnt int64 + CommandLatency int64 + // TTL fields Passes, PassesCnt int64 DeletedDocuments, DeletedDocumentsCnt int64 @@ -684,6 +706,21 @@ func NewStatLine(oldMongo, newMongo MongoStatus, key string, all bool, sampleSec returnVal.Command, returnVal.CommandCnt = diff(newStat.Opcounters.Command, oldStat.Opcounters.Command, sampleSecs) } + if newStat.OpLatencies != nil { + if newStat.OpLatencies.Reads != nil { + returnVal.ReadOpsCnt = newStat.OpLatencies.Reads.Ops + returnVal.ReadLatency = newStat.OpLatencies.Reads.Latency + } + if newStat.OpLatencies.Writes != nil { + returnVal.WriteOpsCnt = newStat.OpLatencies.Writes.Ops + returnVal.WriteLatency = newStat.OpLatencies.Writes.Latency + } + if newStat.OpLatencies.Commands != nil { + returnVal.CommandOpsCnt = newStat.OpLatencies.Commands.Ops + returnVal.CommandLatency = newStat.OpLatencies.Commands.Latency + } + } + if newStat.Metrics != nil && oldStat.Metrics != nil { if newStat.Metrics.TTL != nil && oldStat.Metrics.TTL != nil { returnVal.Passes, returnVal.PassesCnt = diff(newStat.Metrics.TTL.Passes, oldStat.Metrics.TTL.Passes, sampleSecs) diff --git a/plugins/inputs/mongodb/mongostat_test.go b/plugins/inputs/mongodb/mongostat_test.go new file mode 100644 index 000000000..5506602a9 --- /dev/null +++ b/plugins/inputs/mongodb/mongostat_test.go @@ -0,0 +1,205 @@ +package mongodb + +import ( + "testing" + //"time" + + //"github.com/influxdata/telegraf/testutil" + "github.com/stretchr/testify/assert" +) + +func TestLatencyStats(t *testing.T) { + + sl := NewStatLine( + MongoStatus{ + ServerStatus: &ServerStatus{ + Connections: &ConnectionStats{}, + Mem: &MemStats{ + Bits: 0, + Resident: 0, + Virtual: 0, + Supported: false, + Mapped: 0, + MappedWithJournal: 0, + }, + }, + }, + MongoStatus{ + ServerStatus: &ServerStatus{ + Connections: &ConnectionStats{}, + Mem: &MemStats{ + Bits: 0, + Resident: 0, + Virtual: 0, + Supported: false, + Mapped: 0, + MappedWithJournal: 0, + }, + OpLatencies: &OpLatenciesStats{ + Reads: &LatencyStats{ + Ops: 0, + Latency: 0, + }, + Writes: &LatencyStats{ + Ops: 0, + Latency: 0, + }, + Commands: &LatencyStats{ + Ops: 0, + Latency: 0, + }, + }, + }, + }, + "foo", + true, + 60, + ) + + assert.Equal(t, sl.CommandLatency, int64(0)) + assert.Equal(t, sl.ReadLatency, int64(0)) + assert.Equal(t, sl.WriteLatency, int64(0)) + assert.Equal(t, sl.CommandOpsCnt, int64(0)) + assert.Equal(t, sl.ReadOpsCnt, int64(0)) + assert.Equal(t, sl.WriteOpsCnt, int64(0)) +} + +func TestLatencyStatsDiffZero(t *testing.T) { + + sl := NewStatLine( + MongoStatus{ + ServerStatus: &ServerStatus{ + Connections: &ConnectionStats{}, + Mem: &MemStats{ + Bits: 0, + Resident: 0, + Virtual: 0, + Supported: false, + Mapped: 0, + MappedWithJournal: 0, + }, + OpLatencies: &OpLatenciesStats{ + Reads: &LatencyStats{ + Ops: 0, + Latency: 0, + }, + Writes: &LatencyStats{ + Ops: 0, + Latency: 0, + }, + Commands: &LatencyStats{ + Ops: 0, + Latency: 0, + }, + }, + }, + }, + MongoStatus{ + ServerStatus: &ServerStatus{ + Connections: &ConnectionStats{}, + Mem: &MemStats{ + Bits: 0, + Resident: 0, + Virtual: 0, + Supported: false, + Mapped: 0, + MappedWithJournal: 0, + }, + OpLatencies: &OpLatenciesStats{ + Reads: &LatencyStats{ + Ops: 0, + Latency: 0, + }, + Writes: &LatencyStats{ + Ops: 0, + Latency: 0, + }, + Commands: &LatencyStats{ + Ops: 0, + Latency: 0, + }, + }, + }, + }, + "foo", + true, + 60, + ) + + assert.Equal(t, sl.CommandLatency, int64(0)) + assert.Equal(t, sl.ReadLatency, int64(0)) + assert.Equal(t, sl.WriteLatency, int64(0)) + assert.Equal(t, sl.CommandOpsCnt, int64(0)) + assert.Equal(t, sl.ReadOpsCnt, int64(0)) + assert.Equal(t, sl.WriteOpsCnt, int64(0)) +} + +func TestLatencyStatsDiff(t *testing.T) { + + sl := NewStatLine( + MongoStatus{ + ServerStatus: &ServerStatus{ + Connections: &ConnectionStats{}, + Mem: &MemStats{ + Bits: 0, + Resident: 0, + Virtual: 0, + Supported: false, + Mapped: 0, + MappedWithJournal: 0, + }, + OpLatencies: &OpLatenciesStats{ + Reads: &LatencyStats{ + Ops: 4189041956, + Latency: 2255922322753, + }, + Writes: &LatencyStats{ + Ops: 1691019457, + Latency: 494478256915, + }, + Commands: &LatencyStats{ + Ops: 1019150402, + Latency: 59177710371, + }, + }, + }, + }, + MongoStatus{ + ServerStatus: &ServerStatus{ + Connections: &ConnectionStats{}, + Mem: &MemStats{ + Bits: 0, + Resident: 0, + Virtual: 0, + Supported: false, + Mapped: 0, + MappedWithJournal: 0, + }, + OpLatencies: &OpLatenciesStats{ + Reads: &LatencyStats{ + Ops: 4189049884, + Latency: 2255946760057, + }, + Writes: &LatencyStats{ + Ops: 1691021287, + Latency: 494479456987, + }, + Commands: &LatencyStats{ + Ops: 1019152861, + Latency: 59177981552, + }, + }, + }, + }, + "foo", + true, + 60, + ) + + assert.Equal(t, sl.CommandLatency, int64(59177981552)) + assert.Equal(t, sl.ReadLatency, int64(2255946760057)) + assert.Equal(t, sl.WriteLatency, int64(494479456987)) + assert.Equal(t, sl.CommandOpsCnt, int64(1019152861)) + assert.Equal(t, sl.ReadOpsCnt, int64(4189049884)) + assert.Equal(t, sl.WriteOpsCnt, int64(1691021287)) +}