Fix master check and move cluster health indices to separate measurement (#6004)

plugins/inputs/elasticsearch/elasticsearch.go

@@ -40,31 +40,32 @@ type nodeStat struct {
 }
 
 type clusterHealth struct {
-	ClusterName                 string                 `json:"cluster_name"`
-	Status                      string                 `json:"status"`
-	TimedOut                    bool                   `json:"timed_out"`
-	NumberOfNodes               int                    `json:"number_of_nodes"`
-	NumberOfDataNodes           int                    `json:"number_of_data_nodes"`
 	ActivePrimaryShards         int                    `json:"active_primary_shards"`
 	ActiveShards                int                    `json:"active_shards"`
-	RelocatingShards            int                    `json:"relocating_shards"`
-	InitializingShards          int                    `json:"initializing_shards"`
-	UnassignedShards            int                    `json:"unassigned_shards"`
-	DelayedUnassignedShards     int                    `json:"delayed_unassigned_shards"`
-	NumberOfPendingTasks        int                    `json:"number_of_pending_tasks"`
-	TaskMaxWaitingInQueueMillis int                    `json:"task_max_waiting_in_queue_millis"`
 	ActiveShardsPercentAsNumber float64                `json:"active_shards_percent_as_number"`
+	ClusterName                 string                 `json:"cluster_name"`
+	DelayedUnassignedShards     int                    `json:"delayed_unassigned_shards"`
+	InitializingShards          int                    `json:"initializing_shards"`
+	NumberOfDataNodes           int                    `json:"number_of_data_nodes"`
+	NumberOfInFlightFetch       int                    `json:"number_of_in_flight_fetch"`
+	NumberOfNodes               int                    `json:"number_of_nodes"`
+	NumberOfPendingTasks        int                    `json:"number_of_pending_tasks"`
+	RelocatingShards            int                    `json:"relocating_shards"`
+	Status                      string                 `json:"status"`
+	TaskMaxWaitingInQueueMillis int                    `json:"task_max_waiting_in_queue_millis"`
+	TimedOut                    bool                   `json:"timed_out"`
+	UnassignedShards            int                    `json:"unassigned_shards"`
 	Indices                     map[string]indexHealth `json:"indices"`
 }
 
 type indexHealth struct {
-	Status              string `json:"status"`
-	NumberOfShards      int    `json:"number_of_shards"`
-	NumberOfReplicas    int    `json:"number_of_replicas"`
 	ActivePrimaryShards int    `json:"active_primary_shards"`
 	ActiveShards        int    `json:"active_shards"`
-	RelocatingShards    int    `json:"relocating_shards"`
 	InitializingShards  int    `json:"initializing_shards"`
+	NumberOfReplicas    int    `json:"number_of_replicas"`
+	NumberOfShards      int    `json:"number_of_shards"`
+	RelocatingShards    int    `json:"relocating_shards"`
+	Status              string `json:"status"`
 	UnassignedShards    int    `json:"unassigned_shards"`
 }
 
@@ -137,9 +138,17 @@ type Elasticsearch struct {
 	NodeStats                  []string
 	tls.ClientConfig
 
-	client                  *http.Client
+	client          *http.Client
-	catMasterResponseTokens []string
+	serverInfo      map[string]serverInfo
-	isMaster                bool
+	serverInfoMutex sync.Mutex
+}
+type serverInfo struct {
+	nodeID   string
+	masterID string
+}
+
+func (i serverInfo) isMaster() bool {
+	return i.nodeID == i.masterID
 }
 
 // NewElasticsearch return a new instance of Elasticsearch
@@ -186,6 +195,40 @@ func (e *Elasticsearch) Gather(acc telegraf.Accumulator) error {
 		e.client = client
 	}
 
+	if e.ClusterStats {
+		var wgC sync.WaitGroup
+		wgC.Add(len(e.Servers))
+
+		e.serverInfo = make(map[string]serverInfo)
+		for _, serv := range e.Servers {
+			go func(s string, acc telegraf.Accumulator) {
+				defer wgC.Done()
+				info := serverInfo{}
+
+				var err error
+
+				// Gather node ID
+				if info.nodeID, err = e.gatherNodeID(s + "/_nodes/_local/name"); err != nil {
+					acc.AddError(fmt.Errorf(mask.ReplaceAllString(err.Error(), "http(s)://XXX:XXX@")))
+					return
+				}
+
+				// get cat/master information here so NodeStats can determine
+				// whether this node is the Master
+				if info.masterID, err = e.getCatMaster(s + "/_cat/master"); err != nil {
+					acc.AddError(fmt.Errorf(mask.ReplaceAllString(err.Error(), "http(s)://XXX:XXX@")))
+					return
+				}
+
+				e.serverInfoMutex.Lock()
+				e.serverInfo[s] = info
+				e.serverInfoMutex.Unlock()
+
+			}(serv, acc)
+		}
+		wgC.Wait()
+	}
+
 	var wg sync.WaitGroup
 	wg.Add(len(e.Servers))
 
@@ -193,18 +236,8 @@ func (e *Elasticsearch) Gather(acc telegraf.Accumulator) error {
 		go func(s string, acc telegraf.Accumulator) {
 			defer wg.Done()
 			url := e.nodeStatsUrl(s)
-			e.isMaster = false
 
-			if e.ClusterStats {
+			// Always gather node stats
-				// get cat/master information here so NodeStats can determine
-				// whether this node is the Master
-				if err := e.setCatMaster(s + "/_cat/master"); err != nil {
-					acc.AddError(fmt.Errorf(mask.ReplaceAllString(err.Error(), "http(s)://XXX:XXX@")))
-					return
-				}
-			}
-
-			// Always gather node states
 			if err := e.gatherNodeStats(url, acc); err != nil {
 				acc.AddError(fmt.Errorf(mask.ReplaceAllString(err.Error(), "http(s)://XXX:XXX@")))
 				return
@@ -221,7 +254,7 @@ func (e *Elasticsearch) Gather(acc telegraf.Accumulator) error {
 				}
 			}
 
-			if e.ClusterStats && (e.isMaster || !e.ClusterStatsOnlyFromMaster || !e.Local) {
+			if e.ClusterStats && (e.serverInfo[s].isMaster() || !e.ClusterStatsOnlyFromMaster || !e.Local) {
 				if err := e.gatherClusterStats(s+"/_cluster/stats", acc); err != nil {
 					acc.AddError(fmt.Errorf(mask.ReplaceAllString(err.Error(), "http(s)://XXX:XXX@")))
 					return
@@ -267,6 +300,22 @@ func (e *Elasticsearch) nodeStatsUrl(baseUrl string) string {
 	return fmt.Sprintf("%s/%s", url, strings.Join(e.NodeStats, ","))
 }
 
+func (e *Elasticsearch) gatherNodeID(url string) (string, error) {
+	nodeStats := &struct {
+		ClusterName string               `json:"cluster_name"`
+		Nodes       map[string]*nodeStat `json:"nodes"`
+	}{}
+	if err := e.gatherJsonData(url, nodeStats); err != nil {
+		return "", err
+	}
+
+	// Only 1 should be returned
+	for id := range nodeStats.Nodes {
+		return id, nil
+	}
+	return "", nil
+}
+
 func (e *Elasticsearch) gatherNodeStats(url string, acc telegraf.Accumulator) error {
 	nodeStats := &struct {
 		ClusterName string               `json:"cluster_name"`
@@ -284,11 +333,6 @@ func (e *Elasticsearch) gatherNodeStats(url string, acc telegraf.Accumulator) er
 			"cluster_name": nodeStats.ClusterName,
 		}
 
-		if e.ClusterStats {
-			// check for master
-			e.isMaster = (id == e.catMasterResponseTokens[0])
-		}
-
 		for k, v := range n.Attributes {
 			tags["node_attribute_"+k] = v
 		}
@@ -331,20 +375,21 @@ func (e *Elasticsearch) gatherClusterHealth(url string, acc telegraf.Accumulator
 	}
 	measurementTime := time.Now()
 	clusterFields := map[string]interface{}{
-		"status":                           healthStats.Status,
-		"status_code":                      mapHealthStatusToCode(healthStats.Status),
-		"timed_out":                        healthStats.TimedOut,
-		"number_of_nodes":                  healthStats.NumberOfNodes,
-		"number_of_data_nodes":             healthStats.NumberOfDataNodes,
 		"active_primary_shards":            healthStats.ActivePrimaryShards,
 		"active_shards":                    healthStats.ActiveShards,
-		"relocating_shards":                healthStats.RelocatingShards,
-		"initializing_shards":              healthStats.InitializingShards,
-		"unassigned_shards":                healthStats.UnassignedShards,
-		"delayed_unassigned_shards":        healthStats.DelayedUnassignedShards,
-		"number_of_pending_tasks":          healthStats.NumberOfPendingTasks,
-		"task_max_waiting_in_queue_millis": healthStats.TaskMaxWaitingInQueueMillis,
 		"active_shards_percent_as_number":  healthStats.ActiveShardsPercentAsNumber,
+		"delayed_unassigned_shards":        healthStats.DelayedUnassignedShards,
+		"initializing_shards":              healthStats.InitializingShards,
+		"number_of_data_nodes":             healthStats.NumberOfDataNodes,
+		"number_of_in_flight_fetch":        healthStats.NumberOfInFlightFetch,
+		"number_of_nodes":                  healthStats.NumberOfNodes,
+		"number_of_pending_tasks":          healthStats.NumberOfPendingTasks,
+		"relocating_shards":                healthStats.RelocatingShards,
+		"status":                           healthStats.Status,
+		"status_code":                      mapHealthStatusToCode(healthStats.Status),
+		"task_max_waiting_in_queue_millis": healthStats.TaskMaxWaitingInQueueMillis,
+		"timed_out":                        healthStats.TimedOut,
+		"unassigned_shards":                healthStats.UnassignedShards,
 	}
 	acc.AddFields(
 		"elasticsearch_cluster_health",
@@ -355,18 +400,18 @@ func (e *Elasticsearch) gatherClusterHealth(url string, acc telegraf.Accumulator
 
 	for name, health := range healthStats.Indices {
 		indexFields := map[string]interface{}{
-			"status":                health.Status,
-			"status_code":           mapHealthStatusToCode(health.Status),
-			"number_of_shards":      health.NumberOfShards,
-			"number_of_replicas":    health.NumberOfReplicas,
 			"active_primary_shards": health.ActivePrimaryShards,
 			"active_shards":         health.ActiveShards,
-			"relocating_shards":     health.RelocatingShards,
 			"initializing_shards":   health.InitializingShards,
+			"number_of_replicas":    health.NumberOfReplicas,
+			"number_of_shards":      health.NumberOfShards,
+			"relocating_shards":     health.RelocatingShards,
+			"status":                health.Status,
+			"status_code":           mapHealthStatusToCode(health.Status),
 			"unassigned_shards":     health.UnassignedShards,
 		}
 		acc.AddFields(
-			"elasticsearch_indices",
+			"elasticsearch_cluster_health_indices",
 			indexFields,
 			map[string]string{"index": name, "name": healthStats.ClusterName},
 			measurementTime,
@@ -405,27 +450,27 @@ func (e *Elasticsearch) gatherClusterStats(url string, acc telegraf.Accumulator)
 	return nil
 }
 
-func (e *Elasticsearch) setCatMaster(url string) error {
+func (e *Elasticsearch) getCatMaster(url string) (string, error) {
 	r, err := e.client.Get(url)
 	if err != nil {
-		return err
+		return "", err
 	}
 	defer r.Body.Close()
 	if r.StatusCode != http.StatusOK {
 		// NOTE: we are not going to read/discard r.Body under the assumption we'd prefer
 		// to let the underlying transport close the connection and re-establish a new one for
 		// future calls.
-		return fmt.Errorf("elasticsearch: Unable to retrieve master node information. API responded with status-code %d, expected %d", r.StatusCode, http.StatusOK)
+		return "", fmt.Errorf("elasticsearch: Unable to retrieve master node information. API responded with status-code %d, expected %d", r.StatusCode, http.StatusOK)
 	}
 	response, err := ioutil.ReadAll(r.Body)
 
 	if err != nil {
-		return err
+		return "", err
 	}
 
-	e.catMasterResponseTokens = strings.Split(string(response), " ")
+	masterID := strings.Split(string(response), " ")[0]
 
-	return nil
+	return masterID, nil
 }
 
 func (e *Elasticsearch) gatherJsonData(url string, v interface{}) error {

plugins/inputs/elasticsearch/elasticsearch_test.go

@@ -9,6 +9,7 @@ import (
 	"github.com/influxdata/telegraf/testutil"
 
 	"fmt"
+
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -22,6 +23,9 @@ func defaultTags() map[string]string {
 		"node_host":             "test",
 	}
 }
+func defaultServerInfo() serverInfo {
+	return serverInfo{nodeID: "", masterID: "SDFsfSDFsdfFSDSDfSFDSDF"}
+}
 
 type transportMock struct {
 	statusCode int
@@ -49,8 +53,8 @@ func (t *transportMock) RoundTrip(r *http.Request) (*http.Response, error) {
 func (t *transportMock) CancelRequest(_ *http.Request) {
 }
 
-func checkIsMaster(es *Elasticsearch, expected bool, t *testing.T) {
+func checkIsMaster(es *Elasticsearch, server string, expected bool, t *testing.T) {
-	if es.isMaster != expected {
+	if es.serverInfo[server].isMaster() != expected {
 		msg := fmt.Sprintf("IsMaster set incorrectly")
 		assert.Fail(t, msg)
 	}
@@ -73,13 +77,15 @@ func TestGather(t *testing.T) {
 	es := newElasticsearchWithClient()
 	es.Servers = []string{"http://example.com:9200"}
 	es.client.Transport = newTransportMock(http.StatusOK, nodeStatsResponse)
+	es.serverInfo = make(map[string]serverInfo)
+	es.serverInfo["http://example.com:9200"] = defaultServerInfo()
 
 	var acc testutil.Accumulator
 	if err := acc.GatherError(es.Gather); err != nil {
 		t.Fatal(err)
 	}
 
-	checkIsMaster(es, false, t)
+	checkIsMaster(es, es.Servers[0], false, t)
 	checkNodeStatsResult(t, &acc)
 }
 
@@ -88,13 +94,15 @@ func TestGatherIndividualStats(t *testing.T) {
 	es.Servers = []string{"http://example.com:9200"}
 	es.NodeStats = []string{"jvm", "process"}
 	es.client.Transport = newTransportMock(http.StatusOK, nodeStatsResponseJVMProcess)
+	es.serverInfo = make(map[string]serverInfo)
+	es.serverInfo["http://example.com:9200"] = defaultServerInfo()
 
 	var acc testutil.Accumulator
 	if err := acc.GatherError(es.Gather); err != nil {
 		t.Fatal(err)
 	}
 
-	checkIsMaster(es, false, t)
+	checkIsMaster(es, es.Servers[0], false, t)
 
 	tags := defaultTags()
 	acc.AssertDoesNotContainsTaggedFields(t, "elasticsearch_indices", nodestatsIndicesExpected, tags)
@@ -112,13 +120,15 @@ func TestGatherNodeStats(t *testing.T) {
 	es := newElasticsearchWithClient()
 	es.Servers = []string{"http://example.com:9200"}
 	es.client.Transport = newTransportMock(http.StatusOK, nodeStatsResponse)
+	es.serverInfo = make(map[string]serverInfo)
+	es.serverInfo["http://example.com:9200"] = defaultServerInfo()
 
 	var acc testutil.Accumulator
 	if err := es.gatherNodeStats("junk", &acc); err != nil {
 		t.Fatal(err)
 	}
 
-	checkIsMaster(es, false, t)
+	checkIsMaster(es, es.Servers[0], false, t)
 	checkNodeStatsResult(t, &acc)
 }
 
@@ -128,21 +138,23 @@ func TestGatherClusterHealthEmptyClusterHealth(t *testing.T) {
 	es.ClusterHealth = true
 	es.ClusterHealthLevel = ""
 	es.client.Transport = newTransportMock(http.StatusOK, clusterHealthResponse)
+	es.serverInfo = make(map[string]serverInfo)
+	es.serverInfo["http://example.com:9200"] = defaultServerInfo()
 
 	var acc testutil.Accumulator
 	require.NoError(t, es.gatherClusterHealth("junk", &acc))
 
-	checkIsMaster(es, false, t)
+	checkIsMaster(es, es.Servers[0], false, t)
 
 	acc.AssertContainsTaggedFields(t, "elasticsearch_cluster_health",
 		clusterHealthExpected,
 		map[string]string{"name": "elasticsearch_telegraf"})
 
-	acc.AssertDoesNotContainsTaggedFields(t, "elasticsearch_indices",
+	acc.AssertDoesNotContainsTaggedFields(t, "elasticsearch_cluster_health_indices",
 		v1IndexExpected,
 		map[string]string{"index": "v1"})
 
-	acc.AssertDoesNotContainsTaggedFields(t, "elasticsearch_indices",
+	acc.AssertDoesNotContainsTaggedFields(t, "elasticsearch_cluster_health_indices",
 		v2IndexExpected,
 		map[string]string{"index": "v2"})
 }
@@ -153,21 +165,23 @@ func TestGatherClusterHealthSpecificClusterHealth(t *testing.T) {
 	es.ClusterHealth = true
 	es.ClusterHealthLevel = "cluster"
 	es.client.Transport = newTransportMock(http.StatusOK, clusterHealthResponse)
+	es.serverInfo = make(map[string]serverInfo)
+	es.serverInfo["http://example.com:9200"] = defaultServerInfo()
 
 	var acc testutil.Accumulator
 	require.NoError(t, es.gatherClusterHealth("junk", &acc))
 
-	checkIsMaster(es, false, t)
+	checkIsMaster(es, es.Servers[0], false, t)
 
 	acc.AssertContainsTaggedFields(t, "elasticsearch_cluster_health",
 		clusterHealthExpected,
 		map[string]string{"name": "elasticsearch_telegraf"})
 
-	acc.AssertDoesNotContainsTaggedFields(t, "elasticsearch_indices",
+	acc.AssertDoesNotContainsTaggedFields(t, "elasticsearch_cluster_health_indices",
 		v1IndexExpected,
 		map[string]string{"index": "v1"})
 
-	acc.AssertDoesNotContainsTaggedFields(t, "elasticsearch_indices",
+	acc.AssertDoesNotContainsTaggedFields(t, "elasticsearch_cluster_health_indices",
 		v2IndexExpected,
 		map[string]string{"index": "v2"})
 }
@@ -178,21 +192,23 @@ func TestGatherClusterHealthAlsoIndicesHealth(t *testing.T) {
 	es.ClusterHealth = true
 	es.ClusterHealthLevel = "indices"
 	es.client.Transport = newTransportMock(http.StatusOK, clusterHealthResponseWithIndices)
+	es.serverInfo = make(map[string]serverInfo)
+	es.serverInfo["http://example.com:9200"] = defaultServerInfo()
 
 	var acc testutil.Accumulator
 	require.NoError(t, es.gatherClusterHealth("junk", &acc))
 
-	checkIsMaster(es, false, t)
+	checkIsMaster(es, es.Servers[0], false, t)
 
 	acc.AssertContainsTaggedFields(t, "elasticsearch_cluster_health",
 		clusterHealthExpected,
 		map[string]string{"name": "elasticsearch_telegraf"})
 
-	acc.AssertContainsTaggedFields(t, "elasticsearch_indices",
+	acc.AssertContainsTaggedFields(t, "elasticsearch_cluster_health_indices",
 		v1IndexExpected,
 		map[string]string{"index": "v1", "name": "elasticsearch_telegraf"})
 
-	acc.AssertContainsTaggedFields(t, "elasticsearch_indices",
+	acc.AssertContainsTaggedFields(t, "elasticsearch_cluster_health_indices",
 		v2IndexExpected,
 		map[string]string{"index": "v2", "name": "elasticsearch_telegraf"})
 }
@@ -202,13 +218,18 @@ func TestGatherClusterStatsMaster(t *testing.T) {
 	es := newElasticsearchWithClient()
 	es.ClusterStats = true
 	es.Servers = []string{"http://example.com:9200"}
+	es.serverInfo = make(map[string]serverInfo)
+	info := serverInfo{nodeID: "SDFsfSDFsdfFSDSDfSFDSDF", masterID: ""}
 
 	// first get catMaster
 	es.client.Transport = newTransportMock(http.StatusOK, IsMasterResult)
-	require.NoError(t, es.setCatMaster("junk"))
+	masterID, err := es.getCatMaster("junk")
+	require.NoError(t, err)
+	info.masterID = masterID
+	es.serverInfo["http://example.com:9200"] = info
 
 	IsMasterResultTokens := strings.Split(string(IsMasterResult), " ")
-	if es.catMasterResponseTokens[0] != IsMasterResultTokens[0] {
+	if masterID != IsMasterResultTokens[0] {
 		msg := fmt.Sprintf("catmaster is incorrect")
 		assert.Fail(t, msg)
 	}
@@ -221,7 +242,7 @@ func TestGatherClusterStatsMaster(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	checkIsMaster(es, true, t)
+	checkIsMaster(es, es.Servers[0], true, t)
 	checkNodeStatsResult(t, &acc)
 
 	// now test the clusterstats method
@@ -243,13 +264,16 @@ func TestGatherClusterStatsNonMaster(t *testing.T) {
 	es := newElasticsearchWithClient()
 	es.ClusterStats = true
 	es.Servers = []string{"http://example.com:9200"}
+	es.serverInfo = make(map[string]serverInfo)
+	es.serverInfo["http://example.com:9200"] = serverInfo{nodeID: "SDFsfSDFsdfFSDSDfSFDSDF", masterID: ""}
 
 	// first get catMaster
 	es.client.Transport = newTransportMock(http.StatusOK, IsNotMasterResult)
-	require.NoError(t, es.setCatMaster("junk"))
+	masterID, err := es.getCatMaster("junk")
+	require.NoError(t, err)
 
 	IsNotMasterResultTokens := strings.Split(string(IsNotMasterResult), " ")
-	if es.catMasterResponseTokens[0] != IsNotMasterResultTokens[0] {
+	if masterID != IsNotMasterResultTokens[0] {
 		msg := fmt.Sprintf("catmaster is incorrect")
 		assert.Fail(t, msg)
 	}
@@ -263,7 +287,7 @@ func TestGatherClusterStatsNonMaster(t *testing.T) {
 	}
 
 	// ensure flag is clear so Cluster Stats would not be done
-	checkIsMaster(es, false, t)
+	checkIsMaster(es, es.Servers[0], false, t)
 	checkNodeStatsResult(t, &acc)
 }
 

plugins/inputs/elasticsearch/testdata_test.go

@@ -7,6 +7,7 @@ const clusterHealthResponse = `
    "timed_out": false,
    "number_of_nodes": 3,
    "number_of_data_nodes": 3,
+   "number_of_in_flight_fetch": 0,
    "active_primary_shards": 5,
    "active_shards": 15,
    "relocating_shards": 0,
@@ -26,6 +27,7 @@ const clusterHealthResponseWithIndices = `
    "timed_out": false,
    "number_of_nodes": 3,
    "number_of_data_nodes": 3,
+   "number_of_in_flight_fetch": 0,
    "active_primary_shards": 5,
    "active_shards": 15,
    "relocating_shards": 0,
@@ -66,6 +68,7 @@ var clusterHealthExpected = map[string]interface{}{
 	"timed_out":                        false,
 	"number_of_nodes":                  3,
 	"number_of_data_nodes":             3,
+	"number_of_in_flight_fetch":        0,
 	"active_primary_shards":            5,
 	"active_shards":                    15,
 	"relocating_shards":                0,
@@ -514,6 +517,416 @@ const nodeStatsResponse = `
           "tripped": 0
         }
       }
+    },
+    "SDFsfSDFsdfFSDSDfSPOJUY": {
+      "timestamp": 1436365550135,
+      "name": "test.host.com",
+      "transport_address": "inet[/127.0.0.1:9300]",
+      "host": "test",
+      "ip": [
+        "inet[/127.0.0.1:9300]",
+        "NONE"
+      ],
+      "attributes": {
+        "master": "true"
+      },
+      "indices": {
+        "docs": {
+          "count": 29652,
+          "deleted": 5229
+        },
+        "store": {
+          "size_in_bytes": 37715234,
+          "throttle_time_in_millis": 215
+        },
+        "indexing": {
+          "index_total": 84790,
+          "index_time_in_millis": 29680,
+          "index_current": 0,
+          "delete_total": 13879,
+          "delete_time_in_millis": 1139,
+          "delete_current": 0,
+          "noop_update_total": 0,
+          "is_throttled": false,
+          "throttle_time_in_millis": 0
+        },
+        "get": {
+          "total": 1,
+          "time_in_millis": 2,
+          "exists_total": 0,
+          "exists_time_in_millis": 0,
+          "missing_total": 1,
+          "missing_time_in_millis": 2,
+          "current": 0
+        },
+        "search": {
+          "open_contexts": 0,
+          "query_total": 1452,
+          "query_time_in_millis": 5695,
+          "query_current": 0,
+          "fetch_total": 414,
+          "fetch_time_in_millis": 146,
+          "fetch_current": 0
+        },
+        "merges": {
+          "current": 0,
+          "current_docs": 0,
+          "current_size_in_bytes": 0,
+          "total": 133,
+          "total_time_in_millis": 21060,
+          "total_docs": 203672,
+          "total_size_in_bytes": 142900226
+        },
+        "refresh": {
+          "total": 1076,
+          "total_time_in_millis": 20078
+        },
+        "flush": {
+          "total": 115,
+          "total_time_in_millis": 2401
+        },
+        "warmer": {
+          "current": 0,
+          "total": 2319,
+          "total_time_in_millis": 448
+        },
+        "filter_cache": {
+          "memory_size_in_bytes": 7384,
+          "evictions": 0
+        },
+        "id_cache": {
+          "memory_size_in_bytes": 0
+        },
+        "fielddata": {
+          "memory_size_in_bytes": 12996,
+          "evictions": 0
+        },
+        "percolate": {
+          "total": 0,
+          "time_in_millis": 0,
+          "current": 0,
+          "memory_size_in_bytes": -1,
+          "memory_size": "-1b",
+          "queries": 0
+        },
+        "completion": {
+          "size_in_bytes": 0
+        },
+        "segments": {
+          "count": 134,
+          "memory_in_bytes": 1285212,
+          "index_writer_memory_in_bytes": 0,
+          "index_writer_max_memory_in_bytes": 172368955,
+          "version_map_memory_in_bytes": 611844,
+          "fixed_bit_set_memory_in_bytes": 0
+        },
+        "translog": {
+          "operations": 17702,
+          "size_in_bytes": 17
+        },
+        "suggest": {
+          "total": 0,
+          "time_in_millis": 0,
+          "current": 0
+        },
+        "query_cache": {
+          "memory_size_in_bytes": 0,
+          "evictions": 0,
+          "hit_count": 0,
+          "miss_count": 0
+        },
+        "recovery": {
+          "current_as_source": 0,
+          "current_as_target": 0,
+          "throttle_time_in_millis": 0
+        }
+      },
+      "os": {
+        "timestamp": 1436460392944,
+        "load_average": [
+          0.01,
+          0.04,
+          0.05
+        ],
+        "mem": {
+          "free_in_bytes": 477761536,
+          "used_in_bytes": 1621868544,
+          "free_percent": 74,
+          "used_percent": 25,
+          "actual_free_in_bytes": 1565470720,
+          "actual_used_in_bytes": 534159360
+        },
+        "swap": {
+          "used_in_bytes": 0,
+          "free_in_bytes": 487997440
+        }
+      },
+      "process": {
+        "timestamp": 1436460392945,
+        "open_file_descriptors": 160,
+        "cpu": {
+          "percent": 2,
+          "sys_in_millis": 1870,
+          "user_in_millis": 13610,
+          "total_in_millis": 15480
+        },
+        "mem": {
+          "total_virtual_in_bytes": 4747890688
+        }
+      },
+      "jvm": {
+        "timestamp": 1436460392945,
+        "uptime_in_millis": 202245,
+        "mem": {
+          "heap_used_in_bytes": 52709568,
+          "heap_used_percent": 5,
+          "heap_committed_in_bytes": 259522560,
+          "heap_max_in_bytes": 1038876672,
+          "non_heap_used_in_bytes": 39634576,
+          "non_heap_committed_in_bytes": 40841216,
+          "pools": {
+            "young": {
+              "used_in_bytes": 32685760,
+              "max_in_bytes": 279183360,
+              "peak_used_in_bytes": 71630848,
+              "peak_max_in_bytes": 279183360
+            },
+            "survivor": {
+              "used_in_bytes": 8912880,
+              "max_in_bytes": 34865152,
+              "peak_used_in_bytes": 8912888,
+              "peak_max_in_bytes": 34865152
+            },
+            "old": {
+              "used_in_bytes": 11110928,
+              "max_in_bytes": 724828160,
+              "peak_used_in_bytes": 14354608,
+              "peak_max_in_bytes": 724828160
+            }
+          }
+        },
+        "threads": {
+          "count": 44,
+          "peak_count": 45
+        },
+        "gc": {
+          "collectors": {
+            "young": {
+              "collection_count": 2,
+              "collection_time_in_millis": 98
+            },
+            "old": {
+              "collection_count": 1,
+              "collection_time_in_millis": 24
+            }
+          }
+        },
+        "buffer_pools": {
+          "direct": {
+            "count": 40,
+            "used_in_bytes": 6304239,
+            "total_capacity_in_bytes": 6304239
+          },
+          "mapped": {
+            "count": 0,
+            "used_in_bytes": 0,
+            "total_capacity_in_bytes": 0
+          }
+        }
+      },
+      "thread_pool": {
+        "percolate": {
+          "threads": 123,
+          "queue": 23,
+          "active": 13,
+          "rejected": 235,
+          "largest": 23,
+          "completed": 33
+        },
+        "fetch_shard_started": {
+          "threads": 3,
+          "queue": 1,
+          "active": 5,
+          "rejected": 6,
+          "largest": 4,
+          "completed": 54
+        },
+        "listener": {
+          "threads": 1,
+          "queue": 2,
+          "active": 4,
+          "rejected": 8,
+          "largest": 1,
+          "completed": 1
+        },
+        "index": {
+          "threads": 6,
+          "queue": 8,
+          "active": 4,
+          "rejected": 2,
+          "largest": 3,
+          "completed": 6
+        },
+        "refresh": {
+          "threads": 23,
+          "queue": 7,
+          "active": 3,
+          "rejected": 4,
+          "largest": 8,
+          "completed": 3
+        },
+        "suggest": {
+          "threads": 2,
+          "queue": 7,
+          "active": 2,
+          "rejected": 1,
+          "largest": 8,
+          "completed": 3
+        },
+        "generic": {
+          "threads": 1,
+          "queue": 4,
+          "active": 6,
+          "rejected": 3,
+          "largest": 2,
+          "completed": 27
+        },
+        "warmer": {
+          "threads": 2,
+          "queue": 7,
+          "active": 3,
+          "rejected": 2,
+          "largest": 3,
+          "completed": 1
+        },
+        "search": {
+          "threads": 5,
+          "queue": 7,
+          "active": 2,
+          "rejected": 7,
+          "largest": 2,
+          "completed": 4
+        },
+        "flush": {
+          "threads": 3,
+          "queue": 8,
+          "active": 0,
+          "rejected": 1,
+          "largest": 5,
+          "completed": 3
+        },
+        "optimize": {
+          "threads": 3,
+          "queue": 4,
+          "active": 1,
+          "rejected": 2,
+          "largest": 7,
+          "completed": 3
+        },
+        "fetch_shard_store": {
+          "threads": 1,
+          "queue": 7,
+          "active": 4,
+          "rejected": 2,
+          "largest": 4,
+          "completed": 1
+        },
+        "management": {
+          "threads": 2,
+          "queue": 3,
+          "active": 1,
+          "rejected": 6,
+          "largest": 2,
+          "completed": 22
+        },
+        "get": {
+          "threads": 1,
+          "queue": 8,
+          "active": 4,
+          "rejected": 3,
+          "largest": 2,
+          "completed": 1
+        },
+        "merge": {
+          "threads": 6,
+          "queue": 4,
+          "active": 5,
+          "rejected": 2,
+          "largest": 5,
+          "completed": 1
+        },
+        "bulk": {
+          "threads": 4,
+          "queue": 5,
+          "active": 7,
+          "rejected": 3,
+          "largest": 1,
+          "completed": 4
+        },
+        "snapshot": {
+          "threads": 8,
+          "queue": 5,
+          "active": 6,
+          "rejected": 2,
+          "largest": 1,
+          "completed": 0
+        }
+      },
+      "fs": {
+        "timestamp": 1436460392946,
+        "total": {
+          "total_in_bytes": 19507089408,
+          "free_in_bytes": 16909316096,
+          "available_in_bytes": 15894814720
+        },
+        "data": [
+          {
+            "path": "/usr/share/elasticsearch/data/elasticsearch/nodes/0",
+            "mount": "/usr/share/elasticsearch/data",
+            "type": "ext4",
+            "total_in_bytes": 19507089408,
+            "free_in_bytes": 16909316096,
+            "available_in_bytes": 15894814720
+          }
+        ]
+      },
+      "transport": {
+        "server_open": 13,
+        "rx_count": 6,
+        "rx_size_in_bytes": 1380,
+        "tx_count": 6,
+        "tx_size_in_bytes": 1380
+      },
+      "http": {
+        "current_open": 3,
+        "total_opened": 3
+      },
+      "breakers": {
+        "fielddata": {
+          "limit_size_in_bytes": 623326003,
+          "limit_size": "594.4mb",
+          "estimated_size_in_bytes": 0,
+          "estimated_size": "0b",
+          "overhead": 1.03,
+          "tripped": 0
+        },
+        "request": {
+          "limit_size_in_bytes": 415550668,
+          "limit_size": "396.2mb",
+          "estimated_size_in_bytes": 0,
+          "estimated_size": "0b",
+          "overhead": 1.0,
+          "tripped": 0
+        },
+        "parent": {
+          "limit_size_in_bytes": 727213670,
+          "limit_size": "693.5mb",
+          "estimated_size_in_bytes": 0,
+          "estimated_size": "0b",
+          "overhead": 1.0,
+          "tripped": 0
+        }
+      }
     }
   }
 }