Add latency stats to mongo input (#6733)

plugins/inputs/mongodb/README.md

@@ -77,6 +77,12 @@ by running Telegraf with the `--debug` argument.
     - getmores (integer)
     - inserts (integer)
     - jumbo_chunks (integer)
+    - latency_commands_count (integer)
+    - latency_commands (integer)
+    - latency_reads_count (integer)
+    - latency_reads (integer)
+    - latency_writes_count (integer)
+    - latency_writes (integer)
     - member_status (string)
     - net_in_bytes_count (integer)
     - net_out_bytes_count (integer)
@@ -185,6 +191,7 @@ by running Telegraf with the `--debug` argument.
 
 ### Example Output:
 ```
+mongodb,hostname=127.0.0.1:27017 active_reads=0i,active_writes=0i,commands=1335i,commands_per_sec=7i,connections_available=814i,connections_current=5i,connections_total_created=0i,cursor_no_timeout=0i,cursor_no_timeout_count=0i,cursor_pinned=0i,cursor_pinned_count=1i,cursor_timed_out=0i,cursor_timed_out_count=0i,cursor_total=0i,cursor_total_count=1i,deletes=0i,deletes_per_sec=0i,document_deleted=0i,document_inserted=0i,document_returned=13i,document_updated=0i,flushes=5i,flushes_per_sec=0i,getmores=269i,getmores_per_sec=0i,inserts=0i,inserts_per_sec=0i,jumbo_chunks=0i,latency_commands_count=0i,latency_commands=0i,latency_reads_count=0i,latency_reads=0i,latency_writes_count=0i,latency_writes=0i,member_status="PRI",net_in_bytes=986i,net_in_bytes_count=358006i,net_out_bytes=23906i,net_out_bytes_count=661507i,open_connections=5i,percent_cache_dirty=0,percent_cache_used=0,queries=18i,queries_per_sec=3i,queued_reads=0i,queued_writes=0i,repl_commands=0i,repl_commands_per_sec=0i,repl_deletes=0i,repl_deletes_per_sec=0i,repl_getmores=0i,repl_getmores_per_sec=0i,repl_inserts=0i,repl_inserts_per_sec=0i,repl_lag=0i,repl_oplog_window_sec=24355215i,repl_queries=0i,repl_queries_per_sec=0i,repl_updates=0i,repl_updates_per_sec=0i,resident_megabytes=62i,state="PRIMARY",total_available=0i,total_created=0i,total_in_use=0i,total_refreshing=0i,ttl_deletes=0i,ttl_deletes_per_sec=0i,ttl_passes=23i,ttl_passes_per_sec=0i,updates=0i,updates_per_sec=0i,vsize_megabytes=713i,wtcache_app_threads_page_read_count=13i,wtcache_app_threads_page_read_time=74i,wtcache_app_threads_page_write_count=0i,wtcache_bytes_read_into=55271i,wtcache_bytes_written_from=125402i,wtcache_current_bytes=117050i,wtcache_max_bytes_configured=1073741824i,wtcache_pages_evicted_by_app_thread=0i,wtcache_pages_queued_for_eviction=0i,wtcache_server_evicting_pages=0i,wtcache_tracked_dirty_bytes=0i,wtcache_worker_thread_evictingpages=0i 1547159491000000000
 mongodb,hostname=127.0.0.1:27017,node_type=PRI active_reads=0i,active_writes=0i,commands=1335i,commands_per_sec=7i,connections_available=814i,connections_current=5i,connections_total_created=0i,cursor_no_timeout=0i,cursor_no_timeout_count=0i,cursor_pinned=0i,cursor_pinned_count=1i,cursor_timed_out=0i,cursor_timed_out_count=0i,cursor_total=0i,cursor_total_count=1i,deletes=0i,deletes_per_sec=0i,document_deleted=0i,document_inserted=0i,document_returned=13i,document_updated=0i,flushes=5i,flushes_per_sec=0i,getmores=269i,getmores_per_sec=0i,inserts=0i,inserts_per_sec=0i,jumbo_chunks=0i,member_status="PRI",net_in_bytes=986i,net_in_bytes_count=358006i,net_out_bytes=23906i,net_out_bytes_count=661507i,open_connections=5i,percent_cache_dirty=0,percent_cache_used=0,queries=18i,queries_per_sec=3i,queued_reads=0i,queued_writes=0i,repl_commands=0i,repl_commands_per_sec=0i,repl_deletes=0i,repl_deletes_per_sec=0i,repl_getmores=0i,repl_getmores_per_sec=0i,repl_inserts=0i,repl_inserts_per_sec=0i,repl_lag=0i,repl_oplog_window_sec=24355215i,repl_queries=0i,repl_queries_per_sec=0i,repl_updates=0i,repl_updates_per_sec=0i,resident_megabytes=62i,state="PRIMARY",total_available=0i,total_created=0i,total_in_use=0i,total_refreshing=0i,ttl_deletes=0i,ttl_deletes_per_sec=0i,ttl_passes=23i,ttl_passes_per_sec=0i,updates=0i,updates_per_sec=0i,vsize_megabytes=713i,wtcache_app_threads_page_read_count=13i,wtcache_app_threads_page_read_time=74i,wtcache_app_threads_page_write_count=0i,wtcache_bytes_read_into=55271i,wtcache_bytes_written_from=125402i,wtcache_current_bytes=117050i,wtcache_max_bytes_configured=1073741824i,wtcache_pages_evicted_by_app_thread=0i,wtcache_pages_queued_for_eviction=0i,wtcache_server_evicting_pages=0i,wtcache_tracked_dirty_bytes=0i,wtcache_worker_thread_evictingpages=0i 1547159491000000000
 mongodb_db_stats,db_name=admin,hostname=127.0.0.1:27017 avg_obj_size=241,collections=2i,data_size=723i,index_size=49152i,indexes=3i,num_extents=0i,objects=3i,ok=1i,storage_size=53248i,type="db_stat" 1547159491000000000
 mongodb_db_stats,db_name=local,hostname=127.0.0.1:27017 avg_obj_size=813.9705882352941,collections=6i,data_size=55350i,index_size=102400i,indexes=5i,num_extents=0i,objects=68i,ok=1i,storage_size=204800i,type="db_stat" 1547159491000000000

plugins/inputs/mongodb/mongodb_data.go

@@ -86,6 +86,15 @@ var DefaultStats = map[string]string{
 	"connections_total_created": "TotalCreatedC",
 }
 
+var DefaultLatencyStats = map[string]string{
+	"latency_writes_count":   "WriteOpsCnt",
+	"latency_writes":         "WriteLatency",
+	"latency_reads_count":    "ReadOpsCnt",
+	"latency_reads":          "ReadLatency",
+	"latency_commands_count": "CommandOpsCnt",
+	"latency_commands":       "CommandLatency",
+}
+
 var DefaultReplStats = map[string]string{
 	"repl_inserts":          "InsertRCnt",
 	"repl_inserts_per_sec":  "InsertR",
@@ -232,6 +241,10 @@ func (d *MongodbData) AddDefaultStats() {
 		d.Tags["node_type"] = d.StatLine.NodeType
 	}
 
+	if d.StatLine.ReadLatency > 0 {
+		d.addStat(statLine, DefaultLatencyStats)
+	}
+
 	if d.StatLine.OplogStats != nil {
 		d.add("repl_oplog_window_sec", d.StatLine.OplogStats.TimeDiff)
 	}

plugins/inputs/mongodb/mongodb_data_test.go

@@ -142,6 +142,29 @@ func TestAddShardStats(t *testing.T) {
 	}
 }
 
+func TestAddLatencyStats(t *testing.T) {
+	d := NewMongodbData(
+		&StatLine{
+			CommandOpsCnt:  73,
+			CommandLatency: 364,
+			ReadOpsCnt:     113,
+			ReadLatency:    201,
+			WriteOpsCnt:    7,
+			WriteLatency:   55,
+		},
+		tags,
+	)
+
+	var acc testutil.Accumulator
+
+	d.AddDefaultStats()
+	d.flush(&acc)
+
+	for key := range DefaultLatencyStats {
+		assert.True(t, acc.HasInt64Field("mongodb", key))
+	}
+}
+
 func TestAddShardHostStats(t *testing.T) {
 	expectedHosts := []string{"hostA", "hostB"}
 	hostStatLines := map[string]ShardHostStatLine{}

plugins/inputs/mongodb/mongostat.go

@@ -58,6 +58,7 @@ type ServerStatus struct {
 	Network            *NetworkStats          `bson:"network"`
 	Opcounters         *OpcountStats          `bson:"opcounters"`
 	OpcountersRepl     *OpcountStats          `bson:"opcountersRepl"`
+	OpLatencies        *OpLatenciesStats      `bson:"opLatencies"`
 	RecordStats        *DBRecordStats         `bson:"recordStats"`
 	Mem                *MemStats              `bson:"mem"`
 	Repl               *ReplStatus            `bson:"repl"`
@@ -314,6 +315,19 @@ type OpcountStats struct {
 	Command int64 `bson:"command"`
 }
 
+// OpLatenciesStats stores information related to operation latencies for the database as a whole
+type OpLatenciesStats struct {
+	Reads    *LatencyStats `bson:"reads"`
+	Writes   *LatencyStats `bson:"writes"`
+	Commands *LatencyStats `bson:"commands"`
+}
+
+// LatencyStats lists total latency in microseconds and count of operations, enabling you to obtain an average
+type LatencyStats struct {
+	Latency int64 `bson:"latency"`
+	Ops     int64 `bson:"ops"`
+}
+
 // MetricsStats stores information related to metrics
 type MetricsStats struct {
 	TTL      *TTLStats      `bson:"ttl"`
@@ -493,6 +507,14 @@ type StatLine struct {
 	GetMore, GetMoreCnt int64
 	Command, CommandCnt int64
 
+	// OpLatency fields
+	WriteOpsCnt    int64
+	WriteLatency   int64
+	ReadOpsCnt     int64
+	ReadLatency    int64
+	CommandOpsCnt  int64
+	CommandLatency int64
+
 	// TTL fields
 	Passes, PassesCnt                     int64
 	DeletedDocuments, DeletedDocumentsCnt int64
@@ -684,6 +706,21 @@ func NewStatLine(oldMongo, newMongo MongoStatus, key string, all bool, sampleSec
 		returnVal.Command, returnVal.CommandCnt = diff(newStat.Opcounters.Command, oldStat.Opcounters.Command, sampleSecs)
 	}
 
+	if newStat.OpLatencies != nil {
+		if newStat.OpLatencies.Reads != nil {
+			returnVal.ReadOpsCnt = newStat.OpLatencies.Reads.Ops
+			returnVal.ReadLatency = newStat.OpLatencies.Reads.Latency
+		}
+		if newStat.OpLatencies.Writes != nil {
+			returnVal.WriteOpsCnt = newStat.OpLatencies.Writes.Ops
+			returnVal.WriteLatency = newStat.OpLatencies.Writes.Latency
+		}
+		if newStat.OpLatencies.Commands != nil {
+			returnVal.CommandOpsCnt = newStat.OpLatencies.Commands.Ops
+			returnVal.CommandLatency = newStat.OpLatencies.Commands.Latency
+		}
+	}
+
 	if newStat.Metrics != nil && oldStat.Metrics != nil {
 		if newStat.Metrics.TTL != nil && oldStat.Metrics.TTL != nil {
 			returnVal.Passes, returnVal.PassesCnt = diff(newStat.Metrics.TTL.Passes, oldStat.Metrics.TTL.Passes, sampleSecs)

plugins/inputs/mongodb/mongostat_test.go

@@ -0,0 +1,205 @@
+package mongodb
+
+import (
+	"testing"
+	//"time"
+
+	//"github.com/influxdata/telegraf/testutil"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestLatencyStats(t *testing.T) {
+
+	sl := NewStatLine(
+		MongoStatus{
+			ServerStatus: &ServerStatus{
+				Connections: &ConnectionStats{},
+				Mem: &MemStats{
+					Bits:              0,
+					Resident:          0,
+					Virtual:           0,
+					Supported:         false,
+					Mapped:            0,
+					MappedWithJournal: 0,
+				},
+			},
+		},
+		MongoStatus{
+			ServerStatus: &ServerStatus{
+				Connections: &ConnectionStats{},
+				Mem: &MemStats{
+					Bits:              0,
+					Resident:          0,
+					Virtual:           0,
+					Supported:         false,
+					Mapped:            0,
+					MappedWithJournal: 0,
+				},
+				OpLatencies: &OpLatenciesStats{
+					Reads: &LatencyStats{
+						Ops:     0,
+						Latency: 0,
+					},
+					Writes: &LatencyStats{
+						Ops:     0,
+						Latency: 0,
+					},
+					Commands: &LatencyStats{
+						Ops:     0,
+						Latency: 0,
+					},
+				},
+			},
+		},
+		"foo",
+		true,
+		60,
+	)
+
+	assert.Equal(t, sl.CommandLatency, int64(0))
+	assert.Equal(t, sl.ReadLatency, int64(0))
+	assert.Equal(t, sl.WriteLatency, int64(0))
+	assert.Equal(t, sl.CommandOpsCnt, int64(0))
+	assert.Equal(t, sl.ReadOpsCnt, int64(0))
+	assert.Equal(t, sl.WriteOpsCnt, int64(0))
+}
+
+func TestLatencyStatsDiffZero(t *testing.T) {
+
+	sl := NewStatLine(
+		MongoStatus{
+			ServerStatus: &ServerStatus{
+				Connections: &ConnectionStats{},
+				Mem: &MemStats{
+					Bits:              0,
+					Resident:          0,
+					Virtual:           0,
+					Supported:         false,
+					Mapped:            0,
+					MappedWithJournal: 0,
+				},
+				OpLatencies: &OpLatenciesStats{
+					Reads: &LatencyStats{
+						Ops:     0,
+						Latency: 0,
+					},
+					Writes: &LatencyStats{
+						Ops:     0,
+						Latency: 0,
+					},
+					Commands: &LatencyStats{
+						Ops:     0,
+						Latency: 0,
+					},
+				},
+			},
+		},
+		MongoStatus{
+			ServerStatus: &ServerStatus{
+				Connections: &ConnectionStats{},
+				Mem: &MemStats{
+					Bits:              0,
+					Resident:          0,
+					Virtual:           0,
+					Supported:         false,
+					Mapped:            0,
+					MappedWithJournal: 0,
+				},
+				OpLatencies: &OpLatenciesStats{
+					Reads: &LatencyStats{
+						Ops:     0,
+						Latency: 0,
+					},
+					Writes: &LatencyStats{
+						Ops:     0,
+						Latency: 0,
+					},
+					Commands: &LatencyStats{
+						Ops:     0,
+						Latency: 0,
+					},
+				},
+			},
+		},
+		"foo",
+		true,
+		60,
+	)
+
+	assert.Equal(t, sl.CommandLatency, int64(0))
+	assert.Equal(t, sl.ReadLatency, int64(0))
+	assert.Equal(t, sl.WriteLatency, int64(0))
+	assert.Equal(t, sl.CommandOpsCnt, int64(0))
+	assert.Equal(t, sl.ReadOpsCnt, int64(0))
+	assert.Equal(t, sl.WriteOpsCnt, int64(0))
+}
+
+func TestLatencyStatsDiff(t *testing.T) {
+
+	sl := NewStatLine(
+		MongoStatus{
+			ServerStatus: &ServerStatus{
+				Connections: &ConnectionStats{},
+				Mem: &MemStats{
+					Bits:              0,
+					Resident:          0,
+					Virtual:           0,
+					Supported:         false,
+					Mapped:            0,
+					MappedWithJournal: 0,
+				},
+				OpLatencies: &OpLatenciesStats{
+					Reads: &LatencyStats{
+						Ops:     4189041956,
+						Latency: 2255922322753,
+					},
+					Writes: &LatencyStats{
+						Ops:     1691019457,
+						Latency: 494478256915,
+					},
+					Commands: &LatencyStats{
+						Ops:     1019150402,
+						Latency: 59177710371,
+					},
+				},
+			},
+		},
+		MongoStatus{
+			ServerStatus: &ServerStatus{
+				Connections: &ConnectionStats{},
+				Mem: &MemStats{
+					Bits:              0,
+					Resident:          0,
+					Virtual:           0,
+					Supported:         false,
+					Mapped:            0,
+					MappedWithJournal: 0,
+				},
+				OpLatencies: &OpLatenciesStats{
+					Reads: &LatencyStats{
+						Ops:     4189049884,
+						Latency: 2255946760057,
+					},
+					Writes: &LatencyStats{
+						Ops:     1691021287,
+						Latency: 494479456987,
+					},
+					Commands: &LatencyStats{
+						Ops:     1019152861,
+						Latency: 59177981552,
+					},
+				},
+			},
+		},
+		"foo",
+		true,
+		60,
+	)
+
+	assert.Equal(t, sl.CommandLatency, int64(59177981552))
+	assert.Equal(t, sl.ReadLatency, int64(2255946760057))
+	assert.Equal(t, sl.WriteLatency, int64(494479456987))
+	assert.Equal(t, sl.CommandOpsCnt, int64(1019152861))
+	assert.Equal(t, sl.ReadOpsCnt, int64(4189049884))
+	assert.Equal(t, sl.WriteOpsCnt, int64(1691021287))
+}