From 4b3580cceb89d7ddb9eeb9c608067fa58f0ea948 Mon Sep 17 00:00:00 2001 From: "Artem V. Navrotskiy" Date: Thu, 10 Jan 2019 02:48:45 +0300 Subject: [PATCH] Add raw counters to mongodb input (#5264) --- plugins/inputs/mongodb/mongodb_data.go | 22 +++++ plugins/inputs/mongodb/mongodb_data_test.go | 23 +++++ plugins/inputs/mongodb/mongostat.go | 100 +++++++++++--------- 3 files changed, 102 insertions(+), 43 deletions(-) diff --git a/plugins/inputs/mongodb/mongodb_data.go b/plugins/inputs/mongodb/mongodb_data.go index 0c69670d5..733018374 100644 --- a/plugins/inputs/mongodb/mongodb_data.go +++ b/plugins/inputs/mongodb/mongodb_data.go @@ -31,12 +31,19 @@ func NewMongodbData(statLine *StatLine, tags map[string]string) *MongodbData { } var DefaultStats = map[string]string{ + "inserts": "InsertCnt", "inserts_per_sec": "Insert", + "queries": "QueryCnt", "queries_per_sec": "Query", + "updates": "UpdateCnt", "updates_per_sec": "Update", + "deletes": "DeleteCnt", "deletes_per_sec": "Delete", + "getmores": "GetMoreCnt", "getmores_per_sec": "GetMore", + "commands": "CommandCnt", "commands_per_sec": "Command", + "flushes": "FlushesCnt", "flushes_per_sec": "Flushes", "vsize_megabytes": "Virtual", "resident_megabytes": "Resident", @@ -44,15 +51,23 @@ var DefaultStats = map[string]string{ "queued_writes": "QueuedWriters", "active_reads": "ActiveReaders", "active_writes": "ActiveWriters", + "net_in_bytes_count": "NetInCnt", "net_in_bytes": "NetIn", + "net_out_bytes_count": "NetOutCnt", "net_out_bytes": "NetOut", "open_connections": "NumConnections", + "ttl_deletes": "DeletedDocumentsCnt", "ttl_deletes_per_sec": "DeletedDocuments", + "ttl_passes": "PassesCnt", "ttl_passes_per_sec": "Passes", "cursor_timed_out": "TimedOutC", + "cursor_timed_out_count": "TimedOutCCnt", "cursor_no_timeout": "NoTimeoutC", + "cursor_no_timeout_count": "NoTimeoutCCnt", "cursor_pinned": "PinnedC", + "cursor_pinned_count": "PinnedCCnt", "cursor_total": "TotalC", + "cursor_total_count": "TotalCCnt", "document_deleted": "DeletedD", "document_inserted": "InsertedD", "document_returned": "ReturnedD", @@ -63,11 +78,17 @@ var DefaultStats = map[string]string{ } var DefaultReplStats = map[string]string{ + "repl_inserts": "InsertRCnt", "repl_inserts_per_sec": "InsertR", + "repl_queries": "QueryRCnt", "repl_queries_per_sec": "QueryR", + "repl_updates": "UpdateRCnt", "repl_updates_per_sec": "UpdateR", + "repl_deletes": "DeleteRCnt", "repl_deletes_per_sec": "DeleteR", + "repl_getmores": "GetMoreRCnt", "repl_getmores_per_sec": "GetMoreR", + "repl_commands": "CommandRCnt", "repl_commands_per_sec": "CommandR", "member_status": "NodeType", "state": "NodeState", @@ -96,6 +117,7 @@ var ShardHostStats = map[string]string{ var MmapStats = map[string]string{ "mapped_megabytes": "Mapped", "non-mapped_megabytes": "NonMapped", + "page_faults": "FaultsCnt", "page_faults_per_sec": "Faults", } diff --git a/plugins/inputs/mongodb/mongodb_data_test.go b/plugins/inputs/mongodb/mongodb_data_test.go index de75ed7e1..ca15ff977 100644 --- a/plugins/inputs/mongodb/mongodb_data_test.go +++ b/plugins/inputs/mongodb/mongodb_data_test.go @@ -19,10 +19,12 @@ func TestAddNonReplStats(t *testing.T) { Insert: 0, Query: 0, Update: 0, + UpdateCnt: 0, Delete: 0, GetMore: 0, Command: 0, Flushes: 0, + FlushesCnt: 0, Virtual: 0, Resident: 0, QueuedReaders: 0, @@ -191,31 +193,48 @@ func TestStateTag(t *testing.T) { fields := map[string]interface{}{ "active_reads": int64(0), "active_writes": int64(0), + "commands": int64(0), "commands_per_sec": int64(0), + "deletes": int64(0), "deletes_per_sec": int64(0), + "flushes": int64(0), "flushes_per_sec": int64(0), + "getmores": int64(0), "getmores_per_sec": int64(0), + "inserts": int64(0), "inserts_per_sec": int64(0), "member_status": "PRI", "state": "PRIMARY", + "net_in_bytes_count": int64(0), "net_in_bytes": int64(0), + "net_out_bytes_count": int64(0), "net_out_bytes": int64(0), "open_connections": int64(0), + "queries": int64(0), "queries_per_sec": int64(0), "queued_reads": int64(0), "queued_writes": int64(0), + "repl_commands": int64(0), "repl_commands_per_sec": int64(0), + "repl_deletes": int64(0), "repl_deletes_per_sec": int64(0), + "repl_getmores": int64(0), "repl_getmores_per_sec": int64(0), + "repl_inserts": int64(0), "repl_inserts_per_sec": int64(0), + "repl_queries": int64(0), "repl_queries_per_sec": int64(0), + "repl_updates": int64(0), "repl_updates_per_sec": int64(0), "repl_lag": int64(0), "repl_oplog_window_sec": int64(0), "resident_megabytes": int64(0), + "updates": int64(0), "updates_per_sec": int64(0), "vsize_megabytes": int64(0), + "ttl_deletes": int64(0), "ttl_deletes_per_sec": int64(0), + "ttl_passes": int64(0), "ttl_passes_per_sec": int64(0), "jumbo_chunks": int64(0), "total_in_use": int64(0), @@ -223,9 +242,13 @@ func TestStateTag(t *testing.T) { "total_created": int64(0), "total_refreshing": int64(0), "cursor_timed_out": int64(0), + "cursor_timed_out_count": int64(0), "cursor_no_timeout": int64(0), + "cursor_no_timeout_count": int64(0), "cursor_pinned": int64(0), + "cursor_pinned_count": int64(0), "cursor_total": int64(0), + "cursor_total_count": int64(0), "document_deleted": int64(0), "document_inserted": int64(0), "document_returned": int64(0), diff --git a/plugins/inputs/mongodb/mongostat.go b/plugins/inputs/mongodb/mongostat.go index e32596deb..1320c32e9 100644 --- a/plugins/inputs/mongodb/mongostat.go +++ b/plugins/inputs/mongodb/mongostat.go @@ -457,14 +457,22 @@ type StatLine struct { LastPrinted time.Time // Opcounter fields - Insert, Query, Update, Delete, GetMore, Command int64 + Insert, InsertCnt int64 + Query, QueryCnt int64 + Update, UpdateCnt int64 + Delete, DeleteCnt int64 + GetMore, GetMoreCnt int64 + Command, CommandCnt int64 // TTL fields - Passes, DeletedDocuments int64 + Passes, PassesCnt int64 + DeletedDocuments, DeletedDocumentsCnt int64 // Cursor fields - TimedOutC int64 - NoTimeoutC, PinnedC, TotalC int64 + TimedOutC, TimedOutCCnt int64 + NoTimeoutC, NoTimeoutCCnt int64 + PinnedC, PinnedCCnt int64 + TotalC, TotalCCnt int64 // Document fields DeletedD, InsertedD, ReturnedD, UpdatedD int64 @@ -494,20 +502,26 @@ type StatLine struct { WorkerThreadEvictingPages int64 // Replicated Opcounter fields - InsertR, QueryR, UpdateR, DeleteR, GetMoreR, CommandR int64 - ReplLag int64 - OplogTimeDiff int64 - Flushes int64 - Mapped, Virtual, Resident, NonMapped int64 - Faults int64 - HighestLocked *LockStatus - QueuedReaders, QueuedWriters int64 - ActiveReaders, ActiveWriters int64 - NetIn, NetOut int64 - NumConnections int64 - ReplSetName string - NodeType string - NodeState string + InsertR, InsertRCnt int64 + QueryR, QueryRCnt int64 + UpdateR, UpdateRCnt int64 + DeleteR, DeleteRCnt int64 + GetMoreR, GetMoreRCnt int64 + CommandR, CommandRCnt int64 + ReplLag int64 + OplogTimeDiff int64 + Flushes, FlushesCnt int64 + Mapped, Virtual, Resident, NonMapped int64 + Faults, FaultsCnt int64 + HighestLocked *LockStatus + QueuedReaders, QueuedWriters int64 + ActiveReaders, ActiveWriters int64 + NetIn, NetInCnt int64 + NetOut, NetOutCnt int64 + NumConnections int64 + ReplSetName string + NodeType string + NodeState string // Cluster fields JumboChunksCount int64 @@ -576,12 +590,12 @@ func computeLockDiffs(prevLocks, curLocks map[string]LockUsage) []LockUsage { return lockUsages } -func diff(newVal, oldVal, sampleTime int64) int64 { +func diff(newVal, oldVal, sampleTime int64) (int64, int64) { d := newVal - oldVal if d < 0 { d = newVal } - return d / sampleTime + return d / sampleTime, newVal } // NewStatLine constructs a StatLine object from two MongoStatus objects. @@ -612,25 +626,25 @@ func NewStatLine(oldMongo, newMongo MongoStatus, key string, all bool, sampleSec } if newStat.Opcounters != nil && oldStat.Opcounters != nil { - returnVal.Insert = diff(newStat.Opcounters.Insert, oldStat.Opcounters.Insert, sampleSecs) - returnVal.Query = diff(newStat.Opcounters.Query, oldStat.Opcounters.Query, sampleSecs) - returnVal.Update = diff(newStat.Opcounters.Update, oldStat.Opcounters.Update, sampleSecs) - returnVal.Delete = diff(newStat.Opcounters.Delete, oldStat.Opcounters.Delete, sampleSecs) - returnVal.GetMore = diff(newStat.Opcounters.GetMore, oldStat.Opcounters.GetMore, sampleSecs) - returnVal.Command = diff(newStat.Opcounters.Command, oldStat.Opcounters.Command, sampleSecs) + returnVal.Insert, returnVal.InsertCnt = diff(newStat.Opcounters.Insert, oldStat.Opcounters.Insert, sampleSecs) + returnVal.Query, returnVal.QueryCnt = diff(newStat.Opcounters.Query, oldStat.Opcounters.Query, sampleSecs) + returnVal.Update, returnVal.UpdateCnt = diff(newStat.Opcounters.Update, oldStat.Opcounters.Update, sampleSecs) + returnVal.Delete, returnVal.DeleteCnt = diff(newStat.Opcounters.Delete, oldStat.Opcounters.Delete, sampleSecs) + returnVal.GetMore, returnVal.GetMoreCnt = diff(newStat.Opcounters.GetMore, oldStat.Opcounters.GetMore, sampleSecs) + returnVal.Command, returnVal.CommandCnt = diff(newStat.Opcounters.Command, oldStat.Opcounters.Command, sampleSecs) } if newStat.Metrics != nil && oldStat.Metrics != nil { if newStat.Metrics.TTL != nil && oldStat.Metrics.TTL != nil { - returnVal.Passes = diff(newStat.Metrics.TTL.Passes, oldStat.Metrics.TTL.Passes, sampleSecs) - returnVal.DeletedDocuments = diff(newStat.Metrics.TTL.DeletedDocuments, oldStat.Metrics.TTL.DeletedDocuments, sampleSecs) + returnVal.Passes, returnVal.PassesCnt = diff(newStat.Metrics.TTL.Passes, oldStat.Metrics.TTL.Passes, sampleSecs) + returnVal.DeletedDocuments, returnVal.DeletedDocumentsCnt = diff(newStat.Metrics.TTL.DeletedDocuments, oldStat.Metrics.TTL.DeletedDocuments, sampleSecs) } if newStat.Metrics.Cursor != nil && oldStat.Metrics.Cursor != nil { - returnVal.TimedOutC = diff(newStat.Metrics.Cursor.TimedOut, oldStat.Metrics.Cursor.TimedOut, sampleSecs) + returnVal.TimedOutC, returnVal.TimedOutCCnt = diff(newStat.Metrics.Cursor.TimedOut, oldStat.Metrics.Cursor.TimedOut, sampleSecs) if newStat.Metrics.Cursor.Open != nil && oldStat.Metrics.Cursor.Open != nil { - returnVal.NoTimeoutC = diff(newStat.Metrics.Cursor.Open.NoTimeout, oldStat.Metrics.Cursor.Open.NoTimeout, sampleSecs) - returnVal.PinnedC = diff(newStat.Metrics.Cursor.Open.Pinned, oldStat.Metrics.Cursor.Open.Pinned, sampleSecs) - returnVal.TotalC = diff(newStat.Metrics.Cursor.Open.Total, oldStat.Metrics.Cursor.Open.Total, sampleSecs) + returnVal.NoTimeoutC, returnVal.NoTimeoutCCnt = diff(newStat.Metrics.Cursor.Open.NoTimeout, oldStat.Metrics.Cursor.Open.NoTimeout, sampleSecs) + returnVal.PinnedC, returnVal.PinnedCCnt = diff(newStat.Metrics.Cursor.Open.Pinned, oldStat.Metrics.Cursor.Open.Pinned, sampleSecs) + returnVal.TotalC, returnVal.TotalCCnt = diff(newStat.Metrics.Cursor.Open.Total, oldStat.Metrics.Cursor.Open.Total, sampleSecs) } } if newStat.Metrics.Document != nil { @@ -642,18 +656,18 @@ func NewStatLine(oldMongo, newMongo MongoStatus, key string, all bool, sampleSec } if newStat.OpcountersRepl != nil && oldStat.OpcountersRepl != nil { - returnVal.InsertR = diff(newStat.OpcountersRepl.Insert, oldStat.OpcountersRepl.Insert, sampleSecs) - returnVal.QueryR = diff(newStat.OpcountersRepl.Query, oldStat.OpcountersRepl.Query, sampleSecs) - returnVal.UpdateR = diff(newStat.OpcountersRepl.Update, oldStat.OpcountersRepl.Update, sampleSecs) - returnVal.DeleteR = diff(newStat.OpcountersRepl.Delete, oldStat.OpcountersRepl.Delete, sampleSecs) - returnVal.GetMoreR = diff(newStat.OpcountersRepl.GetMore, oldStat.OpcountersRepl.GetMore, sampleSecs) - returnVal.CommandR = diff(newStat.OpcountersRepl.Command, oldStat.OpcountersRepl.Command, sampleSecs) + returnVal.InsertR, returnVal.InsertRCnt = diff(newStat.OpcountersRepl.Insert, oldStat.OpcountersRepl.Insert, sampleSecs) + returnVal.QueryR, returnVal.QueryRCnt = diff(newStat.OpcountersRepl.Query, oldStat.OpcountersRepl.Query, sampleSecs) + returnVal.UpdateR, returnVal.UpdateRCnt = diff(newStat.OpcountersRepl.Update, oldStat.OpcountersRepl.Update, sampleSecs) + returnVal.DeleteR, returnVal.DeleteRCnt = diff(newStat.OpcountersRepl.Delete, oldStat.OpcountersRepl.Delete, sampleSecs) + returnVal.GetMoreR, returnVal.GetMoreRCnt = diff(newStat.OpcountersRepl.GetMore, oldStat.OpcountersRepl.GetMore, sampleSecs) + returnVal.CommandR, returnVal.CommandRCnt = diff(newStat.OpcountersRepl.Command, oldStat.OpcountersRepl.Command, sampleSecs) } returnVal.CacheDirtyPercent = -1 returnVal.CacheUsedPercent = -1 if newStat.WiredTiger != nil && oldStat.WiredTiger != nil { - returnVal.Flushes = newStat.WiredTiger.Transaction.TransCheckpoints - oldStat.WiredTiger.Transaction.TransCheckpoints + returnVal.Flushes, returnVal.FlushesCnt = diff(newStat.WiredTiger.Transaction.TransCheckpoints, oldStat.WiredTiger.Transaction.TransCheckpoints, sampleSecs) returnVal.CacheDirtyPercent = float64(newStat.WiredTiger.Cache.TrackedDirtyBytes) / float64(newStat.WiredTiger.Cache.MaxBytesConfigured) returnVal.CacheUsedPercent = float64(newStat.WiredTiger.Cache.CurrentCachedBytes) / float64(newStat.WiredTiger.Cache.MaxBytesConfigured) @@ -670,7 +684,7 @@ func NewStatLine(oldMongo, newMongo MongoStatus, key string, all bool, sampleSec returnVal.ServerEvictingPages = newStat.WiredTiger.Cache.ServerEvictingPages returnVal.WorkerThreadEvictingPages = newStat.WiredTiger.Cache.WorkerThreadEvictingPages } else if newStat.BackgroundFlushing != nil && oldStat.BackgroundFlushing != nil { - returnVal.Flushes = newStat.BackgroundFlushing.Flushes - oldStat.BackgroundFlushing.Flushes + returnVal.Flushes, returnVal.FlushesCnt = diff(newStat.BackgroundFlushing.Flushes, oldStat.BackgroundFlushing.Flushes, sampleSecs) } returnVal.Time = newMongo.SampleTime @@ -713,7 +727,7 @@ func NewStatLine(oldMongo, newMongo MongoStatus, key string, all bool, sampleSec if oldStat.ExtraInfo != nil && newStat.ExtraInfo != nil && oldStat.ExtraInfo.PageFaults != nil && newStat.ExtraInfo.PageFaults != nil { - returnVal.Faults = diff(*(newStat.ExtraInfo.PageFaults), *(oldStat.ExtraInfo.PageFaults), sampleSecs) + returnVal.Faults, returnVal.FaultsCnt = diff(*(newStat.ExtraInfo.PageFaults), *(oldStat.ExtraInfo.PageFaults), sampleSecs) } if !returnVal.IsMongos && oldStat.Locks != nil { globalCheck, hasGlobal := oldStat.Locks["Global"] @@ -812,8 +826,8 @@ func NewStatLine(oldMongo, newMongo MongoStatus, key string, all bool, sampleSec } if oldStat.Network != nil && newStat.Network != nil { - returnVal.NetIn = diff(newStat.Network.BytesIn, oldStat.Network.BytesIn, sampleSecs) - returnVal.NetOut = diff(newStat.Network.BytesOut, oldStat.Network.BytesOut, sampleSecs) + returnVal.NetIn, returnVal.NetInCnt = diff(newStat.Network.BytesIn, oldStat.Network.BytesIn, sampleSecs) + returnVal.NetOut, returnVal.NetOutCnt = diff(newStat.Network.BytesOut, oldStat.Network.BytesOut, sampleSecs) } if newStat.Connections != nil {