Implemented ceph collector (#1172)
- records metrics from the output of mon and osd socket perf dumps.
This commit is contained in:
robinpercy-xm
Cameron Sparr
parent
f32916a5bd
commit
69dfe63809
|
|
@ -369,6 +369,24 @@
|
|||
# INPUT PLUGINS #
|
||||
###############################################################################
|
||||
|
||||
# # Collects performance metrics from the MON and OSD nodes in a Ceph storage cluster.
|
||||
# [[inputs.ceph]]
|
||||
# ## All configuration values are optional, defaults are shown below
|
||||
#
|
||||
# ## location of ceph binary
|
||||
# ceph_binary = "/usr/bin/ceph"
|
||||
#
|
||||
# ## directory in which to look for socket files
|
||||
# socket_dir = "/var/run/ceph"
|
||||
#
|
||||
# ## prefix of MON and OSD socket files, used to determine socket type
|
||||
# mon_prefix = "ceph-mon"
|
||||
# osd_prefix = "ceph-osd"
|
||||
#
|
||||
# ## suffix used to identify socket files
|
||||
# socket_suffix = "asok"
|
||||
|
||||
|
||||
# Read metrics about cpu usage
|
||||
[[inputs.cpu]]
|
||||
## Whether to report per-cpu stats or not
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ import (
|
|||
_ "github.com/influxdata/telegraf/plugins/inputs/apache"
|
||||
_ "github.com/influxdata/telegraf/plugins/inputs/bcache"
|
||||
_ "github.com/influxdata/telegraf/plugins/inputs/cassandra"
|
||||
_ "github.com/influxdata/telegraf/plugins/inputs/ceph"
|
||||
_ "github.com/influxdata/telegraf/plugins/inputs/cloudwatch"
|
||||
_ "github.com/influxdata/telegraf/plugins/inputs/couchbase"
|
||||
_ "github.com/influxdata/telegraf/plugins/inputs/couchdb"
|
||||
|
|
|
|||
|
|
@ -0,0 +1,109 @@
|
|||
# Ceph Storage Input Plugin
|
||||
|
||||
Collects performance metrics from the MON and OSD nodes in a Ceph storage cluster.
|
||||
|
||||
The plugin works by scanning the configured SocketDir for OSD and MON socket files. When it finds
|
||||
a MON socket, it runs **ceph --admin-daemon $file perfcounters_dump**. For OSDs it runs **ceph --admin-daemon $file perf dump**
|
||||
|
||||
The resulting JSON is parsed and grouped into collections, based on top-level key. Top-level keys are
|
||||
used as collection tags, and all sub-keys are flattened. For example:
|
||||
|
||||
```
|
||||
{
|
||||
"paxos": {
|
||||
"refresh": 9363435,
|
||||
"refresh_latency": {
|
||||
"avgcount": 9363435,
|
||||
"sum": 5378.794002000
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Would be parsed into the following metrics, all of which would be tagged with collection=paxos:
|
||||
|
||||
- refresh = 9363435
|
||||
- refresh_latency.avgcount: 9363435
|
||||
- refresh_latency.sum: 5378.794002000
|
||||
|
||||
|
||||
### Configuration:
|
||||
|
||||
```
|
||||
# Collects performance metrics from the MON and OSD nodes in a Ceph storage cluster.
|
||||
[[inputs.ceph]]
|
||||
## All configuration values are optional, defaults are shown below
|
||||
|
||||
## location of ceph binary
|
||||
ceph_binary = "/usr/bin/ceph"
|
||||
|
||||
## directory in which to look for socket files
|
||||
socket_dir = "/var/run/ceph"
|
||||
|
||||
## prefix of MON and OSD socket files, used to determine socket type
|
||||
mon_prefix = "ceph-mon"
|
||||
osd_prefix = "ceph-osd"
|
||||
|
||||
## suffix used to identify socket files
|
||||
socket_suffix = "asok"
|
||||
```
|
||||
|
||||
### Measurements & Fields:
|
||||
|
||||
All fields are collected under the **ceph** measurement and stored as float64s. For a full list of fields, see the sample perf dumps in ceph_test.go.
|
||||
|
||||
|
||||
### Tags:
|
||||
|
||||
All measurements will have the following tags:
|
||||
|
||||
- type: either 'osd' or 'mon' to indicate which type of node was queried
|
||||
- id: a unique string identifier, parsed from the socket file name for the node
|
||||
- collection: the top-level key under which these fields were reported. Possible values are:
|
||||
- for MON nodes:
|
||||
- cluster
|
||||
- leveldb
|
||||
- mon
|
||||
- paxos
|
||||
- throttle-mon_client_bytes
|
||||
- throttle-mon_daemon_bytes
|
||||
- throttle-msgr_dispatch_throttler-mon
|
||||
- for OSD nodes:
|
||||
- WBThrottle
|
||||
- filestore
|
||||
- leveldb
|
||||
- mutex-FileJournal::completions_lock
|
||||
- mutex-FileJournal::finisher_lock
|
||||
- mutex-FileJournal::write_lock
|
||||
- mutex-FileJournal::writeq_lock
|
||||
- mutex-JOS::ApplyManager::apply_lock
|
||||
- mutex-JOS::ApplyManager::com_lock
|
||||
- mutex-JOS::SubmitManager::lock
|
||||
- mutex-WBThrottle::lock
|
||||
- objecter
|
||||
- osd
|
||||
- recoverystate_perf
|
||||
- throttle-filestore_bytes
|
||||
- throttle-filestore_ops
|
||||
- throttle-msgr_dispatch_throttler-client
|
||||
- throttle-msgr_dispatch_throttler-cluster
|
||||
- throttle-msgr_dispatch_throttler-hb_back_server
|
||||
- throttle-msgr_dispatch_throttler-hb_front_serve
|
||||
- throttle-msgr_dispatch_throttler-hbclient
|
||||
- throttle-msgr_dispatch_throttler-ms_objecter
|
||||
- throttle-objecter_bytes
|
||||
- throttle-objecter_ops
|
||||
- throttle-osd_client_bytes
|
||||
- throttle-osd_client_messages
|
||||
|
||||
|
||||
### Example Output:
|
||||
|
||||
<pre>
|
||||
telegraf -test -config /etc/telegraf/telegraf.conf -config-directory /etc/telegraf/telegraf.d -input-filter ceph
|
||||
* Plugin: ceph, Collection 1
|
||||
> ceph,collection=paxos, id=node-2,role=openstack,type=mon accept_timeout=0,begin=14931264,begin_bytes.avgcount=14931264,begin_bytes.sum=180309683362,begin_keys.avgcount=0,begin_keys.sum=0,begin_latency.avgcount=14931264,begin_latency.sum=9293.29589,collect=1,collect_bytes.avgcount=1,collect_bytes.sum=24,collect_keys.avgcount=1,collect_keys.sum=1,collect_latency.avgcount=1,collect_latency.sum=0.00028,collect_timeout=0,collect_uncommitted=0,commit=14931264,commit_bytes.avgcount=0,commit_bytes.sum=0,commit_keys.avgcount=0,commit_keys.sum=0,commit_latency.avgcount=0,commit_latency.sum=0,lease_ack_timeout=0,lease_timeout=0,new_pn=0,new_pn_latency.avgcount=0,new_pn_latency.sum=0,refresh=14931264,refresh_latency.avgcount=14931264,refresh_latency.sum=8706.98498,restart=4,share_state=0,share_state_bytes.avgcount=0,share_state_bytes.sum=0,share_state_keys.avgcount=0,share_state_keys.sum=0,start_leader=0,start_peon=1,store_state=14931264,store_state_bytes.avgcount=14931264,store_state_bytes.sum=353119959211,store_state_keys.avgcount=14931264,store_state_keys.sum=289807523,store_state_latency.avgcount=14931264,store_state_latency.sum=10952.835724 1462821234814535148
|
||||
> ceph,collection=throttle-mon_client_bytes,id=node-2,type=mon get=1413017,get_or_fail_fail=0,get_or_fail_success=0,get_sum=71211705,max=104857600,put=1413013,put_sum=71211459,take=0,take_sum=0,val=246,wait.avgcount=0,wait.sum=0 1462821234814737219
|
||||
> ceph,collection=throttle-mon_daemon_bytes,id=node-2,type=mon get=4058121,get_or_fail_fail=0,get_or_fail_success=0,get_sum=6027348117,max=419430400,put=4058121,put_sum=6027348117,take=0,take_sum=0,val=0,wait.avgcount=0,wait.sum=0 1462821234814815661
|
||||
> ceph,collection=throttle-msgr_dispatch_throttler-mon,id=node-2,type=mon get=54276277,get_or_fail_fail=0,get_or_fail_success=0,get_sum=370232877040,max=104857600,put=54276277,put_sum=370232877040,take=0,take_sum=0,val=0,wait.avgcount=0,wait.sum=0 1462821234814872064
|
||||
</pre>
|
||||
|
|
@ -0,0 +1,249 @@
|
|||
package ceph
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"github.com/influxdata/telegraf"
|
||||
"github.com/influxdata/telegraf/plugins/inputs"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const (
|
||||
measurement = "ceph"
|
||||
typeMon = "monitor"
|
||||
typeOsd = "osd"
|
||||
osdPrefix = "ceph-osd"
|
||||
monPrefix = "ceph-mon"
|
||||
sockSuffix = "asok"
|
||||
)
|
||||
|
||||
type Ceph struct {
|
||||
CephBinary string
|
||||
OsdPrefix string
|
||||
MonPrefix string
|
||||
SocketDir string
|
||||
SocketSuffix string
|
||||
}
|
||||
|
||||
func (c *Ceph) setDefaults() {
|
||||
if c.CephBinary == "" {
|
||||
c.CephBinary = "/usr/bin/ceph"
|
||||
}
|
||||
|
||||
if c.OsdPrefix == "" {
|
||||
c.OsdPrefix = osdPrefix
|
||||
}
|
||||
|
||||
if c.MonPrefix == "" {
|
||||
c.MonPrefix = monPrefix
|
||||
}
|
||||
|
||||
if c.SocketDir == "" {
|
||||
c.SocketDir = "/var/run/ceph"
|
||||
}
|
||||
|
||||
if c.SocketSuffix == "" {
|
||||
c.SocketSuffix = sockSuffix
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Ceph) Description() string {
|
||||
return "Collects performance metrics from the MON and OSD nodes in a Ceph storage cluster."
|
||||
}
|
||||
|
||||
var sampleConfig = `
|
||||
## All configuration values are optional, defaults are shown below
|
||||
|
||||
## location of ceph binary
|
||||
ceph_binary = "/usr/bin/ceph"
|
||||
|
||||
## directory in which to look for socket files
|
||||
socket_dir = "/var/run/ceph"
|
||||
|
||||
## prefix of MON and OSD socket files, used to determine socket type
|
||||
mon_prefix = "ceph-mon"
|
||||
osd_prefix = "ceph-osd"
|
||||
|
||||
## suffix used to identify socket files
|
||||
socket_suffix = "asok"
|
||||
`
|
||||
|
||||
func (c *Ceph) SampleConfig() string {
|
||||
return sampleConfig
|
||||
}
|
||||
|
||||
func (c *Ceph) Gather(acc telegraf.Accumulator) error {
|
||||
c.setDefaults()
|
||||
sockets, err := findSockets(c)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to find sockets at path '%s': %v", c.SocketDir, err)
|
||||
}
|
||||
|
||||
for _, s := range sockets {
|
||||
dump, err := perfDump(c.CephBinary, s)
|
||||
if err != nil {
|
||||
log.Printf("error reading from socket '%s': %v", s.socket, err)
|
||||
continue
|
||||
}
|
||||
data, err := parseDump(dump)
|
||||
if err != nil {
|
||||
log.Printf("error parsing dump from socket '%s': %v", s.socket, err)
|
||||
continue
|
||||
}
|
||||
for tag, metrics := range *data {
|
||||
acc.AddFields(measurement,
|
||||
map[string]interface{}(metrics),
|
||||
map[string]string{"type": s.sockType, "id": s.sockId, "collection": tag})
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func init() {
|
||||
inputs.Add(measurement, func() telegraf.Input { return &Ceph{} })
|
||||
}
|
||||
|
||||
var perfDump = func(binary string, socket *socket) (string, error) {
|
||||
cmdArgs := []string{"--admin-daemon", socket.socket}
|
||||
if socket.sockType == typeOsd {
|
||||
cmdArgs = append(cmdArgs, "perf", "dump")
|
||||
} else if socket.sockType == typeMon {
|
||||
cmdArgs = append(cmdArgs, "perfcounters_dump")
|
||||
} else {
|
||||
return "", fmt.Errorf("ignoring unknown socket type: %s", socket.sockType)
|
||||
}
|
||||
|
||||
cmd := exec.Command(binary, cmdArgs...)
|
||||
var out bytes.Buffer
|
||||
cmd.Stdout = &out
|
||||
err := cmd.Run()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("error running ceph dump: %s", err)
|
||||
}
|
||||
|
||||
return out.String(), nil
|
||||
}
|
||||
|
||||
var findSockets = func(c *Ceph) ([]*socket, error) {
|
||||
listing, err := ioutil.ReadDir(c.SocketDir)
|
||||
if err != nil {
|
||||
return []*socket{}, fmt.Errorf("Failed to read socket directory '%s': %v", c.SocketDir, err)
|
||||
}
|
||||
sockets := make([]*socket, 0, len(listing))
|
||||
for _, info := range listing {
|
||||
f := info.Name()
|
||||
var sockType string
|
||||
var sockPrefix string
|
||||
if strings.HasPrefix(f, c.MonPrefix) {
|
||||
sockType = typeMon
|
||||
sockPrefix = monPrefix
|
||||
}
|
||||
if strings.HasPrefix(f, c.OsdPrefix) {
|
||||
sockType = typeOsd
|
||||
sockPrefix = osdPrefix
|
||||
|
||||
}
|
||||
if sockType == typeOsd || sockType == typeMon {
|
||||
path := filepath.Join(c.SocketDir, f)
|
||||
sockets = append(sockets, &socket{parseSockId(f, sockPrefix, c.SocketSuffix), sockType, path})
|
||||
}
|
||||
}
|
||||
return sockets, nil
|
||||
}
|
||||
|
||||
func parseSockId(fname, prefix, suffix string) string {
|
||||
s := fname
|
||||
s = strings.TrimPrefix(s, prefix)
|
||||
s = strings.TrimSuffix(s, suffix)
|
||||
s = strings.Trim(s, ".-_")
|
||||
return s
|
||||
}
|
||||
|
||||
type socket struct {
|
||||
sockId string
|
||||
sockType string
|
||||
socket string
|
||||
}
|
||||
|
||||
type metric struct {
|
||||
pathStack []string // lifo stack of name components
|
||||
value float64
|
||||
}
|
||||
|
||||
// Pops names of pathStack to build the flattened name for a metric
|
||||
func (m *metric) name() string {
|
||||
buf := bytes.Buffer{}
|
||||
for i := len(m.pathStack) - 1; i >= 0; i-- {
|
||||
if buf.Len() > 0 {
|
||||
buf.WriteString(".")
|
||||
}
|
||||
buf.WriteString(m.pathStack[i])
|
||||
}
|
||||
return buf.String()
|
||||
}
|
||||
|
||||
type metricMap map[string]interface{}
|
||||
|
||||
type taggedMetricMap map[string]metricMap
|
||||
|
||||
// Parses a raw JSON string into a taggedMetricMap
|
||||
// Delegates the actual parsing to newTaggedMetricMap(..)
|
||||
func parseDump(dump string) (*taggedMetricMap, error) {
|
||||
data := make(map[string]interface{})
|
||||
err := json.Unmarshal([]byte(dump), &data)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to parse json: '%s': %v", dump, err)
|
||||
}
|
||||
|
||||
tmm := newTaggedMetricMap(data)
|
||||
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to tag dataset: '%v': %v", tmm, err)
|
||||
}
|
||||
|
||||
return tmm, nil
|
||||
}
|
||||
|
||||
// Builds a TaggedMetricMap out of a generic string map.
|
||||
// The top-level key is used as a tag and all sub-keys are flattened into metrics
|
||||
func newTaggedMetricMap(data map[string]interface{}) *taggedMetricMap {
|
||||
tmm := make(taggedMetricMap)
|
||||
for tag, datapoints := range data {
|
||||
mm := make(metricMap)
|
||||
for _, m := range flatten(datapoints) {
|
||||
mm[m.name()] = m.value
|
||||
}
|
||||
tmm[tag] = mm
|
||||
}
|
||||
return &tmm
|
||||
}
|
||||
|
||||
// Recursively flattens any k-v hierarchy present in data.
|
||||
// Nested keys are flattened into ordered slices associated with a metric value.
|
||||
// The key slices are treated as stacks, and are expected to be reversed and concatenated
|
||||
// when passed as metrics to the accumulator. (see (*metric).name())
|
||||
func flatten(data interface{}) []*metric {
|
||||
var metrics []*metric
|
||||
|
||||
switch val := data.(type) {
|
||||
case float64:
|
||||
metrics = []*metric{&metric{make([]string, 0, 1), val}}
|
||||
case map[string]interface{}:
|
||||
metrics = make([]*metric, 0, len(val))
|
||||
for k, v := range val {
|
||||
for _, m := range flatten(v) {
|
||||
m.pathStack = append(m.pathStack, k)
|
||||
metrics = append(metrics, m)
|
||||
}
|
||||
}
|
||||
default:
|
||||
log.Printf("Ignoring unexpected type '%T' for value %v", val, val)
|
||||
}
|
||||
|
||||
return metrics
|
||||
}
|
||||
|
|
@ -0,0 +1,682 @@
|
|||
package ceph
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/influxdata/telegraf/testutil"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"path"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
const (
|
||||
epsilon = float64(0.00000001)
|
||||
)
|
||||
|
||||
func TestParseSockId(t *testing.T) {
|
||||
s := parseSockId(sockFile(osdPrefix, 1), osdPrefix, sockSuffix)
|
||||
assert.Equal(t, s, "1")
|
||||
}
|
||||
|
||||
func TestParseMonDump(t *testing.T) {
|
||||
dump, err := parseDump(monPerfDump)
|
||||
assert.NoError(t, err)
|
||||
assert.InEpsilon(t, 5678670180, (*dump)["cluster"]["osd_kb_used"], epsilon)
|
||||
assert.InEpsilon(t, 6866.540527000, (*dump)["paxos"]["store_state_latency.sum"], epsilon)
|
||||
}
|
||||
|
||||
func TestParseOsdDump(t *testing.T) {
|
||||
dump, err := parseDump(osdPerfDump)
|
||||
assert.NoError(t, err)
|
||||
assert.InEpsilon(t, 552132.109360000, (*dump)["filestore"]["commitcycle_interval.sum"], epsilon)
|
||||
assert.Equal(t, float64(0), (*dump)["mutex-FileJournal::finisher_lock"]["wait.avgcount"])
|
||||
}
|
||||
|
||||
func TestGather(t *testing.T) {
|
||||
saveFind := findSockets
|
||||
saveDump := perfDump
|
||||
defer func() {
|
||||
findSockets = saveFind
|
||||
perfDump = saveDump
|
||||
}()
|
||||
|
||||
findSockets = func(c *Ceph) ([]*socket, error) {
|
||||
return []*socket{&socket{"osd.1", typeOsd, ""}}, nil
|
||||
}
|
||||
|
||||
perfDump = func(binary string, s *socket) (string, error) {
|
||||
return osdPerfDump, nil
|
||||
}
|
||||
|
||||
acc := &testutil.Accumulator{}
|
||||
c := &Ceph{}
|
||||
c.Gather(acc)
|
||||
|
||||
}
|
||||
|
||||
func TestFindSockets(t *testing.T) {
|
||||
tmpdir, err := ioutil.TempDir("", "socktest")
|
||||
assert.NoError(t, err)
|
||||
defer func() {
|
||||
err := os.Remove(tmpdir)
|
||||
assert.NoError(t, err)
|
||||
}()
|
||||
c := &Ceph{
|
||||
CephBinary: "foo",
|
||||
SocketDir: tmpdir,
|
||||
}
|
||||
|
||||
c.setDefaults()
|
||||
|
||||
for _, st := range sockTestParams {
|
||||
createTestFiles(tmpdir, st)
|
||||
|
||||
sockets, err := findSockets(c)
|
||||
assert.NoError(t, err)
|
||||
|
||||
for i := 1; i <= st.osds; i++ {
|
||||
assertFoundSocket(t, tmpdir, typeOsd, i, sockets)
|
||||
}
|
||||
|
||||
for i := 1; i <= st.mons; i++ {
|
||||
assertFoundSocket(t, tmpdir, typeMon, i, sockets)
|
||||
}
|
||||
cleanupTestFiles(tmpdir, st)
|
||||
}
|
||||
}
|
||||
|
||||
func assertFoundSocket(t *testing.T, dir, sockType string, i int, sockets []*socket) {
|
||||
var prefix string
|
||||
if sockType == typeOsd {
|
||||
prefix = osdPrefix
|
||||
} else {
|
||||
prefix = monPrefix
|
||||
}
|
||||
expected := path.Join(dir, sockFile(prefix, i))
|
||||
found := false
|
||||
for _, s := range sockets {
|
||||
fmt.Printf("Checking %s\n", s.socket)
|
||||
if s.socket == expected {
|
||||
found = true
|
||||
assert.Equal(t, s.sockType, sockType, "Unexpected socket type for '%s'", s)
|
||||
assert.Equal(t, s.sockId, strconv.Itoa(i))
|
||||
}
|
||||
}
|
||||
assert.True(t, found, "Did not find socket: %s", expected)
|
||||
}
|
||||
|
||||
func sockFile(prefix string, i int) string {
|
||||
return strings.Join([]string{prefix, strconv.Itoa(i), sockSuffix}, ".")
|
||||
}
|
||||
|
||||
func createTestFiles(dir string, st *SockTest) {
|
||||
writeFile := func(prefix string, i int) {
|
||||
f := sockFile(prefix, i)
|
||||
fpath := path.Join(dir, f)
|
||||
ioutil.WriteFile(fpath, []byte(""), 0777)
|
||||
}
|
||||
tstFileApply(st, writeFile)
|
||||
}
|
||||
|
||||
func cleanupTestFiles(dir string, st *SockTest) {
|
||||
rmFile := func(prefix string, i int) {
|
||||
f := sockFile(prefix, i)
|
||||
fpath := path.Join(dir, f)
|
||||
err := os.Remove(fpath)
|
||||
if err != nil {
|
||||
fmt.Printf("Error removing test file %s: %v\n", fpath, err)
|
||||
}
|
||||
}
|
||||
tstFileApply(st, rmFile)
|
||||
}
|
||||
|
||||
func tstFileApply(st *SockTest, fn func(prefix string, i int)) {
|
||||
for i := 1; i <= st.osds; i++ {
|
||||
fn(osdPrefix, i)
|
||||
}
|
||||
for i := 1; i <= st.mons; i++ {
|
||||
fn(monPrefix, i)
|
||||
}
|
||||
}
|
||||
|
||||
type SockTest struct {
|
||||
osds int
|
||||
mons int
|
||||
}
|
||||
|
||||
var sockTestParams = []*SockTest{
|
||||
&SockTest{
|
||||
osds: 2,
|
||||
mons: 2,
|
||||
},
|
||||
&SockTest{
|
||||
mons: 1,
|
||||
},
|
||||
&SockTest{
|
||||
osds: 1,
|
||||
},
|
||||
&SockTest{},
|
||||
}
|
||||
|
||||
var monPerfDump = `
|
||||
{ "cluster": { "num_mon": 2,
|
||||
"num_mon_quorum": 2,
|
||||
"num_osd": 26,
|
||||
"num_osd_up": 26,
|
||||
"num_osd_in": 26,
|
||||
"osd_epoch": 3306,
|
||||
"osd_kb": 11487846448,
|
||||
"osd_kb_used": 5678670180,
|
||||
"osd_kb_avail": 5809176268,
|
||||
"num_pool": 12,
|
||||
"num_pg": 768,
|
||||
"num_pg_active_clean": 768,
|
||||
"num_pg_active": 768,
|
||||
"num_pg_peering": 0,
|
||||
"num_object": 397616,
|
||||
"num_object_degraded": 0,
|
||||
"num_object_unfound": 0,
|
||||
"num_bytes": 2917848227467,
|
||||
"num_mds_up": 0,
|
||||
"num_mds_in": 0,
|
||||
"num_mds_failed": 0,
|
||||
"mds_epoch": 1},
|
||||
"leveldb": { "leveldb_get": 321950312,
|
||||
"leveldb_transaction": 18729922,
|
||||
"leveldb_compact": 0,
|
||||
"leveldb_compact_range": 74141,
|
||||
"leveldb_compact_queue_merge": 0,
|
||||
"leveldb_compact_queue_len": 0},
|
||||
"mon": {},
|
||||
"paxos": { "start_leader": 0,
|
||||
"start_peon": 1,
|
||||
"restart": 4,
|
||||
"refresh": 9363435,
|
||||
"refresh_latency": { "avgcount": 9363435,
|
||||
"sum": 5378.794002000},
|
||||
"begin": 9363435,
|
||||
"begin_keys": { "avgcount": 0,
|
||||
"sum": 0},
|
||||
"begin_bytes": { "avgcount": 9363435,
|
||||
"sum": 110468605489},
|
||||
"begin_latency": { "avgcount": 9363435,
|
||||
"sum": 5850.060682000},
|
||||
"commit": 9363435,
|
||||
"commit_keys": { "avgcount": 0,
|
||||
"sum": 0},
|
||||
"commit_bytes": { "avgcount": 0,
|
||||
"sum": 0},
|
||||
"commit_latency": { "avgcount": 0,
|
||||
"sum": 0.000000000},
|
||||
"collect": 1,
|
||||
"collect_keys": { "avgcount": 1,
|
||||
"sum": 1},
|
||||
"collect_bytes": { "avgcount": 1,
|
||||
"sum": 24},
|
||||
"collect_latency": { "avgcount": 1,
|
||||
"sum": 0.000280000},
|
||||
"collect_uncommitted": 0,
|
||||
"collect_timeout": 0,
|
||||
"accept_timeout": 0,
|
||||
"lease_ack_timeout": 0,
|
||||
"lease_timeout": 0,
|
||||
"store_state": 9363435,
|
||||
"store_state_keys": { "avgcount": 9363435,
|
||||
"sum": 176572789},
|
||||
"store_state_bytes": { "avgcount": 9363435,
|
||||
"sum": 216355887217},
|
||||
"store_state_latency": { "avgcount": 9363435,
|
||||
"sum": 6866.540527000},
|
||||
"share_state": 0,
|
||||
"share_state_keys": { "avgcount": 0,
|
||||
"sum": 0},
|
||||
"share_state_bytes": { "avgcount": 0,
|
||||
"sum": 0},
|
||||
"new_pn": 0,
|
||||
"new_pn_latency": { "avgcount": 0,
|
||||
"sum": 0.000000000}},
|
||||
"throttle-mon_client_bytes": { "val": 246,
|
||||
"max": 104857600,
|
||||
"get": 896030,
|
||||
"get_sum": 45854374,
|
||||
"get_or_fail_fail": 0,
|
||||
"get_or_fail_success": 0,
|
||||
"take": 0,
|
||||
"take_sum": 0,
|
||||
"put": 896026,
|
||||
"put_sum": 45854128,
|
||||
"wait": { "avgcount": 0,
|
||||
"sum": 0.000000000}},
|
||||
"throttle-mon_daemon_bytes": { "val": 0,
|
||||
"max": 419430400,
|
||||
"get": 2773768,
|
||||
"get_sum": 3627676976,
|
||||
"get_or_fail_fail": 0,
|
||||
"get_or_fail_success": 0,
|
||||
"take": 0,
|
||||
"take_sum": 0,
|
||||
"put": 2773768,
|
||||
"put_sum": 3627676976,
|
||||
"wait": { "avgcount": 0,
|
||||
"sum": 0.000000000}},
|
||||
"throttle-msgr_dispatch_throttler-mon": { "val": 0,
|
||||
"max": 104857600,
|
||||
"get": 34504949,
|
||||
"get_sum": 226860281124,
|
||||
"get_or_fail_fail": 0,
|
||||
"get_or_fail_success": 0,
|
||||
"take": 0,
|
||||
"take_sum": 0,
|
||||
"put": 34504949,
|
||||
"put_sum": 226860281124,
|
||||
"wait": { "avgcount": 0,
|
||||
"sum": 0.000000000}}}
|
||||
`
|
||||
|
||||
var osdPerfDump = `
|
||||
{ "WBThrottle": { "bytes_dirtied": 28405539,
|
||||
"bytes_wb": 0,
|
||||
"ios_dirtied": 93,
|
||||
"ios_wb": 0,
|
||||
"inodes_dirtied": 86,
|
||||
"inodes_wb": 0},
|
||||
"filestore": { "journal_queue_max_ops": 0,
|
||||
"journal_queue_ops": 0,
|
||||
"journal_ops": 1108008,
|
||||
"journal_queue_max_bytes": 0,
|
||||
"journal_queue_bytes": 0,
|
||||
"journal_bytes": 73233416196,
|
||||
"journal_latency": { "avgcount": 1108008,
|
||||
"sum": 290.981036000},
|
||||
"journal_wr": 1091866,
|
||||
"journal_wr_bytes": { "avgcount": 1091866,
|
||||
"sum": 74925682688},
|
||||
"journal_full": 0,
|
||||
"committing": 0,
|
||||
"commitcycle": 110389,
|
||||
"commitcycle_interval": { "avgcount": 110389,
|
||||
"sum": 552132.109360000},
|
||||
"commitcycle_latency": { "avgcount": 110389,
|
||||
"sum": 178.657804000},
|
||||
"op_queue_max_ops": 50,
|
||||
"op_queue_ops": 0,
|
||||
"ops": 1108008,
|
||||
"op_queue_max_bytes": 104857600,
|
||||
"op_queue_bytes": 0,
|
||||
"bytes": 73226768148,
|
||||
"apply_latency": { "avgcount": 1108008,
|
||||
"sum": 947.742722000},
|
||||
"queue_transaction_latency_avg": { "avgcount": 1108008,
|
||||
"sum": 0.511327000}},
|
||||
"leveldb": { "leveldb_get": 4361221,
|
||||
"leveldb_transaction": 4351276,
|
||||
"leveldb_compact": 0,
|
||||
"leveldb_compact_range": 0,
|
||||
"leveldb_compact_queue_merge": 0,
|
||||
"leveldb_compact_queue_len": 0},
|
||||
"mutex-FileJournal::completions_lock": { "wait": { "avgcount": 0,
|
||||
"sum": 0.000000000}},
|
||||
"mutex-FileJournal::finisher_lock": { "wait": { "avgcount": 0,
|
||||
"sum": 0.000000000}},
|
||||
"mutex-FileJournal::write_lock": { "wait": { "avgcount": 0,
|
||||
"sum": 0.000000000}},
|
||||
"mutex-FileJournal::writeq_lock": { "wait": { "avgcount": 0,
|
||||
"sum": 0.000000000}},
|
||||
"mutex-JOS::ApplyManager::apply_lock": { "wait": { "avgcount": 0,
|
||||
"sum": 0.000000000}},
|
||||
"mutex-JOS::ApplyManager::com_lock": { "wait": { "avgcount": 0,
|
||||
"sum": 0.000000000}},
|
||||
"mutex-JOS::SubmitManager::lock": { "wait": { "avgcount": 0,
|
||||
"sum": 0.000000000}},
|
||||
"mutex-WBThrottle::lock": { "wait": { "avgcount": 0,
|
||||
"sum": 0.000000000}},
|
||||
"objecter": { "op_active": 0,
|
||||
"op_laggy": 0,
|
||||
"op_send": 0,
|
||||
"op_send_bytes": 0,
|
||||
"op_resend": 0,
|
||||
"op_ack": 0,
|
||||
"op_commit": 0,
|
||||
"op": 0,
|
||||
"op_r": 0,
|
||||
"op_w": 0,
|
||||
"op_rmw": 0,
|
||||
"op_pg": 0,
|
||||
"osdop_stat": 0,
|
||||
"osdop_create": 0,
|
||||
"osdop_read": 0,
|
||||
"osdop_write": 0,
|
||||
"osdop_writefull": 0,
|
||||
"osdop_append": 0,
|
||||
"osdop_zero": 0,
|
||||
"osdop_truncate": 0,
|
||||
"osdop_delete": 0,
|
||||
"osdop_mapext": 0,
|
||||
"osdop_sparse_read": 0,
|
||||
"osdop_clonerange": 0,
|
||||
"osdop_getxattr": 0,
|
||||
"osdop_setxattr": 0,
|
||||
"osdop_cmpxattr": 0,
|
||||
"osdop_rmxattr": 0,
|
||||
"osdop_resetxattrs": 0,
|
||||
"osdop_tmap_up": 0,
|
||||
"osdop_tmap_put": 0,
|
||||
"osdop_tmap_get": 0,
|
||||
"osdop_call": 0,
|
||||
"osdop_watch": 0,
|
||||
"osdop_notify": 0,
|
||||
"osdop_src_cmpxattr": 0,
|
||||
"osdop_pgls": 0,
|
||||
"osdop_pgls_filter": 0,
|
||||
"osdop_other": 0,
|
||||
"linger_active": 0,
|
||||
"linger_send": 0,
|
||||
"linger_resend": 0,
|
||||
"poolop_active": 0,
|
||||
"poolop_send": 0,
|
||||
"poolop_resend": 0,
|
||||
"poolstat_active": 0,
|
||||
"poolstat_send": 0,
|
||||
"poolstat_resend": 0,
|
||||
"statfs_active": 0,
|
||||
"statfs_send": 0,
|
||||
"statfs_resend": 0,
|
||||
"command_active": 0,
|
||||
"command_send": 0,
|
||||
"command_resend": 0,
|
||||
"map_epoch": 3300,
|
||||
"map_full": 0,
|
||||
"map_inc": 3293,
|
||||
"osd_sessions": 0,
|
||||
"osd_session_open": 0,
|
||||
"osd_session_close": 0,
|
||||
"osd_laggy": 0},
|
||||
"osd": { "opq": 0,
|
||||
"op_wip": 0,
|
||||
"op": 23939,
|
||||
"op_in_bytes": 1245903961,
|
||||
"op_out_bytes": 29103083856,
|
||||
"op_latency": { "avgcount": 23939,
|
||||
"sum": 440.192015000},
|
||||
"op_process_latency": { "avgcount": 23939,
|
||||
"sum": 30.170685000},
|
||||
"op_r": 23112,
|
||||
"op_r_out_bytes": 29103056146,
|
||||
"op_r_latency": { "avgcount": 23112,
|
||||
"sum": 19.373526000},
|
||||
"op_r_process_latency": { "avgcount": 23112,
|
||||
"sum": 14.625928000},
|
||||
"op_w": 549,
|
||||
"op_w_in_bytes": 1245804358,
|
||||
"op_w_rlat": { "avgcount": 549,
|
||||
"sum": 17.022299000},
|
||||
"op_w_latency": { "avgcount": 549,
|
||||
"sum": 418.494610000},
|
||||
"op_w_process_latency": { "avgcount": 549,
|
||||
"sum": 13.316555000},
|
||||
"op_rw": 278,
|
||||
"op_rw_in_bytes": 99603,
|
||||
"op_rw_out_bytes": 27710,
|
||||
"op_rw_rlat": { "avgcount": 278,
|
||||
"sum": 2.213785000},
|
||||
"op_rw_latency": { "avgcount": 278,
|
||||
"sum": 2.323879000},
|
||||
"op_rw_process_latency": { "avgcount": 278,
|
||||
"sum": 2.228202000},
|
||||
"subop": 1074774,
|
||||
"subop_in_bytes": 26841811636,
|
||||
"subop_latency": { "avgcount": 1074774,
|
||||
"sum": 745.509160000},
|
||||
"subop_w": 0,
|
||||
"subop_w_in_bytes": 26841811636,
|
||||
"subop_w_latency": { "avgcount": 1074774,
|
||||
"sum": 745.509160000},
|
||||
"subop_pull": 0,
|
||||
"subop_pull_latency": { "avgcount": 0,
|
||||
"sum": 0.000000000},
|
||||
"subop_push": 0,
|
||||
"subop_push_in_bytes": 0,
|
||||
"subop_push_latency": { "avgcount": 0,
|
||||
"sum": 0.000000000},
|
||||
"pull": 0,
|
||||
"push": 28,
|
||||
"push_out_bytes": 103483392,
|
||||
"push_in": 0,
|
||||
"push_in_bytes": 0,
|
||||
"recovery_ops": 15,
|
||||
"loadavg": 202,
|
||||
"buffer_bytes": 0,
|
||||
"numpg": 18,
|
||||
"numpg_primary": 8,
|
||||
"numpg_replica": 10,
|
||||
"numpg_stray": 0,
|
||||
"heartbeat_to_peers": 10,
|
||||
"heartbeat_from_peers": 0,
|
||||
"map_messages": 7413,
|
||||
"map_message_epochs": 9792,
|
||||
"map_message_epoch_dups": 10105,
|
||||
"messages_delayed_for_map": 83,
|
||||
"stat_bytes": 102123175936,
|
||||
"stat_bytes_used": 49961820160,
|
||||
"stat_bytes_avail": 52161355776,
|
||||
"copyfrom": 0,
|
||||
"tier_promote": 0,
|
||||
"tier_flush": 0,
|
||||
"tier_flush_fail": 0,
|
||||
"tier_try_flush": 0,
|
||||
"tier_try_flush_fail": 0,
|
||||
"tier_evict": 0,
|
||||
"tier_whiteout": 0,
|
||||
"tier_dirty": 230,
|
||||
"tier_clean": 0,
|
||||
"tier_delay": 0,
|
||||
"agent_wake": 0,
|
||||
"agent_skip": 0,
|
||||
"agent_flush": 0,
|
||||
"agent_evict": 0},
|
||||
"recoverystate_perf": { "initial_latency": { "avgcount": 473,
|
||||
"sum": 0.027207000},
|
||||
"started_latency": { "avgcount": 1480,
|
||||
"sum": 9854902.397648000},
|
||||
"reset_latency": { "avgcount": 1953,
|
||||
"sum": 0.096206000},
|
||||
"start_latency": { "avgcount": 1953,
|
||||
"sum": 0.059947000},
|
||||
"primary_latency": { "avgcount": 765,
|
||||
"sum": 4688922.186935000},
|
||||
"peering_latency": { "avgcount": 704,
|
||||
"sum": 1668.652135000},
|
||||
"backfilling_latency": { "avgcount": 0,
|
||||
"sum": 0.000000000},
|
||||
"waitremotebackfillreserved_latency": { "avgcount": 0,
|
||||
"sum": 0.000000000},
|
||||
"waitlocalbackfillreserved_latency": { "avgcount": 0,
|
||||
"sum": 0.000000000},
|
||||
"notbackfilling_latency": { "avgcount": 0,
|
||||
"sum": 0.000000000},
|
||||
"repnotrecovering_latency": { "avgcount": 462,
|
||||
"sum": 5158922.114600000},
|
||||
"repwaitrecoveryreserved_latency": { "avgcount": 15,
|
||||
"sum": 0.008275000},
|
||||
"repwaitbackfillreserved_latency": { "avgcount": 1,
|
||||
"sum": 0.000095000},
|
||||
"RepRecovering_latency": { "avgcount": 16,
|
||||
"sum": 2274.944727000},
|
||||
"activating_latency": { "avgcount": 514,
|
||||
"sum": 261.008520000},
|
||||
"waitlocalrecoveryreserved_latency": { "avgcount": 20,
|
||||
"sum": 0.175422000},
|
||||
"waitremoterecoveryreserved_latency": { "avgcount": 20,
|
||||
"sum": 0.682778000},
|
||||
"recovering_latency": { "avgcount": 20,
|
||||
"sum": 0.697551000},
|
||||
"recovered_latency": { "avgcount": 511,
|
||||
"sum": 0.011038000},
|
||||
"clean_latency": { "avgcount": 503,
|
||||
"sum": 4686961.154278000},
|
||||
"active_latency": { "avgcount": 506,
|
||||
"sum": 4687223.640464000},
|
||||
"replicaactive_latency": { "avgcount": 446,
|
||||
"sum": 5161197.078966000},
|
||||
"stray_latency": { "avgcount": 794,
|
||||
"sum": 4805.105128000},
|
||||
"getinfo_latency": { "avgcount": 704,
|
||||
"sum": 1138.477937000},
|
||||
"getlog_latency": { "avgcount": 678,
|
||||
"sum": 0.036393000},
|
||||
"waitactingchange_latency": { "avgcount": 69,
|
||||
"sum": 59.172893000},
|
||||
"incomplete_latency": { "avgcount": 0,
|
||||
"sum": 0.000000000},
|
||||
"getmissing_latency": { "avgcount": 609,
|
||||
"sum": 0.012288000},
|
||||
"waitupthru_latency": { "avgcount": 576,
|
||||
"sum": 530.106999000}},
|
||||
"throttle-filestore_bytes": { "val": 0,
|
||||
"max": 0,
|
||||
"get": 0,
|
||||
"get_sum": 0,
|
||||
"get_or_fail_fail": 0,
|
||||
"get_or_fail_success": 0,
|
||||
"take": 0,
|
||||
"take_sum": 0,
|
||||
"put": 0,
|
||||
"put_sum": 0,
|
||||
"wait": { "avgcount": 0,
|
||||
"sum": 0.000000000}},
|
||||
"throttle-filestore_ops": { "val": 0,
|
||||
"max": 0,
|
||||
"get": 0,
|
||||
"get_sum": 0,
|
||||
"get_or_fail_fail": 0,
|
||||
"get_or_fail_success": 0,
|
||||
"take": 0,
|
||||
"take_sum": 0,
|
||||
"put": 0,
|
||||
"put_sum": 0,
|
||||
"wait": { "avgcount": 0,
|
||||
"sum": 0.000000000}},
|
||||
"throttle-msgr_dispatch_throttler-client": { "val": 0,
|
||||
"max": 104857600,
|
||||
"get": 130730,
|
||||
"get_sum": 1246039872,
|
||||
"get_or_fail_fail": 0,
|
||||
"get_or_fail_success": 0,
|
||||
"take": 0,
|
||||
"take_sum": 0,
|
||||
"put": 130730,
|
||||
"put_sum": 1246039872,
|
||||
"wait": { "avgcount": 0,
|
||||
"sum": 0.000000000}},
|
||||
"throttle-msgr_dispatch_throttler-cluster": { "val": 0,
|
||||
"max": 104857600,
|
||||
"get": 1108033,
|
||||
"get_sum": 71277949992,
|
||||
"get_or_fail_fail": 0,
|
||||
"get_or_fail_success": 0,
|
||||
"take": 0,
|
||||
"take_sum": 0,
|
||||
"put": 1108033,
|
||||
"put_sum": 71277949992,
|
||||
"wait": { "avgcount": 0,
|
||||
"sum": 0.000000000}},
|
||||
"throttle-msgr_dispatch_throttler-hb_back_server": { "val": 0,
|
||||
"max": 104857600,
|
||||
"get": 18320575,
|
||||
"get_sum": 861067025,
|
||||
"get_or_fail_fail": 0,
|
||||
"get_or_fail_success": 0,
|
||||
"take": 0,
|
||||
"take_sum": 0,
|
||||
"put": 18320575,
|
||||
"put_sum": 861067025,
|
||||
"wait": { "avgcount": 0,
|
||||
"sum": 0.000000000}},
|
||||
"throttle-msgr_dispatch_throttler-hb_front_server": { "val": 0,
|
||||
"max": 104857600,
|
||||
"get": 18320575,
|
||||
"get_sum": 861067025,
|
||||
"get_or_fail_fail": 0,
|
||||
"get_or_fail_success": 0,
|
||||
"take": 0,
|
||||
"take_sum": 0,
|
||||
"put": 18320575,
|
||||
"put_sum": 861067025,
|
||||
"wait": { "avgcount": 0,
|
||||
"sum": 0.000000000}},
|
||||
"throttle-msgr_dispatch_throttler-hbclient": { "val": 0,
|
||||
"max": 104857600,
|
||||
"get": 40479394,
|
||||
"get_sum": 1902531518,
|
||||
"get_or_fail_fail": 0,
|
||||
"get_or_fail_success": 0,
|
||||
"take": 0,
|
||||
"take_sum": 0,
|
||||
"put": 40479394,
|
||||
"put_sum": 1902531518,
|
||||
"wait": { "avgcount": 0,
|
||||
"sum": 0.000000000}},
|
||||
"throttle-msgr_dispatch_throttler-ms_objecter": { "val": 0,
|
||||
"max": 104857600,
|
||||
"get": 0,
|
||||
"get_sum": 0,
|
||||
"get_or_fail_fail": 0,
|
||||
"get_or_fail_success": 0,
|
||||
"take": 0,
|
||||
"take_sum": 0,
|
||||
"put": 0,
|
||||
"put_sum": 0,
|
||||
"wait": { "avgcount": 0,
|
||||
"sum": 0.000000000}},
|
||||
"throttle-objecter_bytes": { "val": 0,
|
||||
"max": 104857600,
|
||||
"get": 0,
|
||||
"get_sum": 0,
|
||||
"get_or_fail_fail": 0,
|
||||
"get_or_fail_success": 0,
|
||||
"take": 0,
|
||||
"take_sum": 0,
|
||||
"put": 0,
|
||||
"put_sum": 0,
|
||||
"wait": { "avgcount": 0,
|
||||
"sum": 0.000000000}},
|
||||
"throttle-objecter_ops": { "val": 0,
|
||||
"max": 1024,
|
||||
"get": 0,
|
||||
"get_sum": 0,
|
||||
"get_or_fail_fail": 0,
|
||||
"get_or_fail_success": 0,
|
||||
"take": 0,
|
||||
"take_sum": 0,
|
||||
"put": 0,
|
||||
"put_sum": 0,
|
||||
"wait": { "avgcount": 0,
|
||||
"sum": 0.000000000}},
|
||||
"throttle-osd_client_bytes": { "val": 0,
|
||||
"max": 524288000,
|
||||
"get": 24241,
|
||||
"get_sum": 1241992581,
|
||||
"get_or_fail_fail": 0,
|
||||
"get_or_fail_success": 0,
|
||||
"take": 0,
|
||||
"take_sum": 0,
|
||||
"put": 25958,
|
||||
"put_sum": 1241992581,
|
||||
"wait": { "avgcount": 0,
|
||||
"sum": 0.000000000}},
|
||||
"throttle-osd_client_messages": { "val": 0,
|
||||
"max": 100,
|
||||
"get": 49214,
|
||||
"get_sum": 49214,
|
||||
"get_or_fail_fail": 0,
|
||||
"get_or_fail_success": 0,
|
||||
"take": 0,
|
||||
"take_sum": 0,
|
||||
"put": 49214,
|
||||
"put_sum": 49214,
|
||||
"wait": { "avgcount": 0,
|
||||
"sum": 0.000000000}}}
|
||||
`
|
||||
Loading…
Reference in New Issue