From df337597113016e9c9ffc4bed9854ac1b7eb98d1 Mon Sep 17 00:00:00 2001 From: "Artem V. Navrotskiy" Date: Tue, 15 Jan 2019 22:31:52 +0300 Subject: [PATCH] Add flush_total_time_ns and additional wired tiger fields to mongodb input (#5273) --- plugins/inputs/mongodb/README.md | 6 +++++ plugins/inputs/mongodb/mongodb_data.go | 6 +++++ plugins/inputs/mongodb/mongodb_data_test.go | 11 +++++--- plugins/inputs/mongodb/mongostat.go | 28 ++++++++++++++++++--- 4 files changed, 45 insertions(+), 6 deletions(-) diff --git a/plugins/inputs/mongodb/README.md b/plugins/inputs/mongodb/README.md index 3751d51a4..982936811 100644 --- a/plugins/inputs/mongodb/README.md +++ b/plugins/inputs/mongodb/README.md @@ -61,6 +61,7 @@ Error in input [mongodb]: not authorized on admin to execute command { serverSta - document_returned (integer) - document_updated (integer) - flushes (integer) + - flushes_total_time_ns (integer) - getmores (integer) - inserts (integer - jumbo_chunks (integer) @@ -96,8 +97,13 @@ Error in input [mongodb]: not authorized on admin to execute command { serverSta - wtcache_app_threads_page_write_count (integer) - wtcache_bytes_read_into (integer) - wtcache_bytes_written_from (integer) + - wtcache_pages_read_info (integer) + - wtcache_pages_requested_from (integer) - wtcache_current_bytes (integer) - wtcache_max_bytes_configured (integer) + - wtcache_internal_pages_evicted (integer) + - wtcache_modified_pages_evicted (integer) + - wtcache_unmodified_pages_evicted (integer) - wtcache_pages_evicted_by_app_thread (integer) - wtcache_pages_queued_for_eviction (integer) - wtcache_server_evicting_pages (integer) diff --git a/plugins/inputs/mongodb/mongodb_data.go b/plugins/inputs/mongodb/mongodb_data.go index 733018374..c0e7baf65 100644 --- a/plugins/inputs/mongodb/mongodb_data.go +++ b/plugins/inputs/mongodb/mongodb_data.go @@ -45,6 +45,7 @@ var DefaultStats = map[string]string{ "commands_per_sec": "Command", "flushes": "FlushesCnt", "flushes_per_sec": "Flushes", + "flushes_total_time_ns": "FlushesTotalTime", "vsize_megabytes": "Virtual", "resident_megabytes": "Resident", "queued_reads": "QueuedReaders", @@ -137,8 +138,13 @@ var WiredTigerExtStats = map[string]string{ "wtcache_bytes_read_into": "BytesReadInto", "wtcache_pages_evicted_by_app_thread": "PagesEvictedByAppThread", "wtcache_pages_queued_for_eviction": "PagesQueuedForEviction", + "wtcache_pages_read_info": "PagesReadIntoCache", + "wtcache_pages_requested_from": "PagesRequestedFromCache", "wtcache_server_evicting_pages": "ServerEvictingPages", "wtcache_worker_thread_evictingpages": "WorkerThreadEvictingPages", + "wtcache_internal_pages_evicted": "InternalPagesEvicted", + "wtcache_modified_pages_evicted": "ModifiedPagesEvicted", + "wtcache_unmodified_pages_evicted": "UnmodifiedPagesEvicted", } var DbDataStats = map[string]string{ diff --git a/plugins/inputs/mongodb/mongodb_data_test.go b/plugins/inputs/mongodb/mongodb_data_test.go index ca15ff977..da50bdc9e 100644 --- a/plugins/inputs/mongodb/mongodb_data_test.go +++ b/plugins/inputs/mongodb/mongodb_data_test.go @@ -56,7 +56,7 @@ func TestAddNonReplStats(t *testing.T) { d.flush(&acc) for key := range DefaultStats { - assert.True(t, acc.HasInt64Field("mongodb", key)) + assert.True(t, acc.HasFloatField("mongodb", key) || acc.HasInt64Field("mongodb", key), key) } } @@ -77,7 +77,7 @@ func TestAddReplStats(t *testing.T) { d.flush(&acc) for key := range MmapStats { - assert.True(t, acc.HasInt64Field("mongodb", key)) + assert.True(t, acc.HasInt64Field("mongodb", key), key) } } @@ -109,7 +109,11 @@ func TestAddWiredTigerStats(t *testing.T) { d.flush(&acc) for key := range WiredTigerStats { - assert.True(t, acc.HasFloatField("mongodb", key)) + assert.True(t, acc.HasFloatField("mongodb", key), key) + } + + for key := range WiredTigerExtStats { + assert.True(t, acc.HasFloatField("mongodb", key) || acc.HasInt64Field("mongodb", key), key) } } @@ -199,6 +203,7 @@ func TestStateTag(t *testing.T) { "deletes_per_sec": int64(0), "flushes": int64(0), "flushes_per_sec": int64(0), + "flushes_total_time_ns": int64(0), "getmores": int64(0), "getmores_per_sec": int64(0), "inserts": int64(0), diff --git a/plugins/inputs/mongodb/mongostat.go b/plugins/inputs/mongodb/mongostat.go index 1320c32e9..b763631ca 100644 --- a/plugins/inputs/mongodb/mongostat.go +++ b/plugins/inputs/mongodb/mongostat.go @@ -168,13 +168,19 @@ type CacheStats struct { BytesReadInto int64 `bson:"bytes read into cache"` PagesEvictedByAppThread int64 `bson:"pages evicted by application threads"` PagesQueuedForEviction int64 `bson:"pages queued for eviction"` + PagesReadIntoCache int64 `bson:"pages read into cache"` + PagesRequestedFromCache int64 `bson:"pages requested from the cache"` ServerEvictingPages int64 `bson:"eviction server evicting pages"` WorkerThreadEvictingPages int64 `bson:"eviction worker thread evicting pages"` + InternalPagesEvicted int64 `bson:"internal pages evicted"` + ModifiedPagesEvicted int64 `bson:"modified pages evicted"` + UnmodifiedPagesEvicted int64 `bson:"unmodified pages evicted"` } // TransactionStats stores transaction checkpoints in WiredTiger. type TransactionStats struct { - TransCheckpoints int64 `bson:"transaction checkpoints"` + TransCheckpointsTotalTimeMsecs int64 `bson:"transaction checkpoint total time (msecs)"` + TransCheckpoints int64 `bson:"transaction checkpoints"` } // ReplStatus stores data related to replica sets. @@ -498,8 +504,13 @@ type StatLine struct { BytesReadInto int64 PagesEvictedByAppThread int64 PagesQueuedForEviction int64 + PagesReadIntoCache int64 + PagesRequestedFromCache int64 ServerEvictingPages int64 WorkerThreadEvictingPages int64 + InternalPagesEvicted int64 + ModifiedPagesEvicted int64 + UnmodifiedPagesEvicted int64 // Replicated Opcounter fields InsertR, InsertRCnt int64 @@ -511,6 +522,7 @@ type StatLine struct { ReplLag int64 OplogTimeDiff int64 Flushes, FlushesCnt int64 + FlushesTotalTime int64 Mapped, Virtual, Resident, NonMapped int64 Faults, FaultsCnt int64 HighestLocked *LockStatus @@ -666,8 +678,7 @@ func NewStatLine(oldMongo, newMongo MongoStatus, key string, all bool, sampleSec returnVal.CacheDirtyPercent = -1 returnVal.CacheUsedPercent = -1 - if newStat.WiredTiger != nil && oldStat.WiredTiger != nil { - returnVal.Flushes, returnVal.FlushesCnt = diff(newStat.WiredTiger.Transaction.TransCheckpoints, oldStat.WiredTiger.Transaction.TransCheckpoints, sampleSecs) + if newStat.WiredTiger != nil { returnVal.CacheDirtyPercent = float64(newStat.WiredTiger.Cache.TrackedDirtyBytes) / float64(newStat.WiredTiger.Cache.MaxBytesConfigured) returnVal.CacheUsedPercent = float64(newStat.WiredTiger.Cache.CurrentCachedBytes) / float64(newStat.WiredTiger.Cache.MaxBytesConfigured) @@ -681,8 +692,19 @@ func NewStatLine(oldMongo, newMongo MongoStatus, key string, all bool, sampleSec returnVal.BytesReadInto = newStat.WiredTiger.Cache.BytesReadInto returnVal.PagesEvictedByAppThread = newStat.WiredTiger.Cache.PagesEvictedByAppThread returnVal.PagesQueuedForEviction = newStat.WiredTiger.Cache.PagesQueuedForEviction + returnVal.PagesReadIntoCache = newStat.WiredTiger.Cache.PagesReadIntoCache + returnVal.PagesRequestedFromCache = newStat.WiredTiger.Cache.PagesRequestedFromCache returnVal.ServerEvictingPages = newStat.WiredTiger.Cache.ServerEvictingPages returnVal.WorkerThreadEvictingPages = newStat.WiredTiger.Cache.WorkerThreadEvictingPages + + returnVal.InternalPagesEvicted = newStat.WiredTiger.Cache.InternalPagesEvicted + returnVal.ModifiedPagesEvicted = newStat.WiredTiger.Cache.ModifiedPagesEvicted + returnVal.UnmodifiedPagesEvicted = newStat.WiredTiger.Cache.UnmodifiedPagesEvicted + + returnVal.FlushesTotalTime = newStat.WiredTiger.Transaction.TransCheckpointsTotalTimeMsecs * int64(time.Millisecond) + } + if newStat.WiredTiger != nil && oldStat.WiredTiger != nil { + returnVal.Flushes, returnVal.FlushesCnt = diff(newStat.WiredTiger.Transaction.TransCheckpoints, oldStat.WiredTiger.Transaction.TransCheckpoints, sampleSecs) } else if newStat.BackgroundFlushing != nil && oldStat.BackgroundFlushing != nil { returnVal.Flushes, returnVal.FlushesCnt = diff(newStat.BackgroundFlushing.Flushes, oldStat.BackgroundFlushing.Flushes, sampleSecs) }