dom
/
telegraf
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Capture all cores by default

This commit is contained in:
Lukasz Jagiello 2016-09-09 09:18:40 +00:00
parent 245306a623
commit e46e9ab79c
4 changed files with 209 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
plugins/inputs/solr/README.md
View File

@ -12,6 +12,6 @@ More about [performance statistics](https://cwiki.apache.org/confluence/display/
## specify a list of one or more Solr servers
servers = ["http://localhost:8983"]
## specify a list of one or more Solr cores
cores = ["main"]
## specify a list of one or more Solr cores (default - all)
# cores = ["main"]
```

64
plugins/inputs/solr/solr.go
View File

@ -17,6 +17,7 @@ import (
)
const mbeansPath = "/admin/mbeans?stats=true&wt=json"
const adminCoresPath = "/solr/admin/cores?action=STATUS&wt=json"
type node struct {
Host string `json:"host"`
@ -26,8 +27,8 @@ const sampleConfig = `
## specify a list of one or more Solr servers
servers = ["http://localhost:8983"]
## specify a list of one or more Solr cores
cores = ["main"]
## specify a list of one or more Solr cores (default - all)
# cores = ["main"]
`
// Solr is a plugin to read stats from one or many Solr servers
@ -39,6 +40,12 @@ type Solr struct {
client *http.Client
}
// AdminCores is an exported type that
// contains a response with information about Solr cores.
type AdminCores struct {
Status map[string]json.RawMessage `json:"status"`
}
// Metrics is an exported type that
// contains a response from Solr with metrics
type Metrics struct {
@ -148,20 +155,49 @@ func (s *Solr) Description() string {
return "Read stats from one or more Solr servers or cores"
}
// Default settings
func (s *Solr) setDefaults() ([][]string, int, error) {
var max int
cores := [][]string{}
if len(s.Cores) == 0 {
for n, server := range s.Servers {
adminCores := &AdminCores{}
if err := s.gatherData(fmt.Sprintf("%s%s", server, adminCoresPath), adminCores); err != nil {
return nil, 0, err
}
serverCores := []string{}
for coreName := range adminCores.Status {
serverCores = append(serverCores, coreName)
}
cores = append(cores, serverCores)
if len(cores[n]) > max {
max = len(cores[n])
}
}
} else {
cores = append(cores, s.Cores)
max = len(s.Cores)
}
return cores, max, nil
}
// Gather reads the stats from Solr and writes it to the
// Accumulator.
func (s *Solr) Gather(acc telegraf.Accumulator) error {
if s.client == nil {
client := s.createHTTPClient()
s.client = client
}
cores, max, err := s.setDefaults()
if err != nil {
return err
}
errChan := errchan.New(len(s.Servers) * len(s.Cores))
errChan := errchan.New(len(s.Servers) * max)
var wg sync.WaitGroup
for _, serv := range s.Servers {
for _, core := range s.Cores {
for n, serv := range s.Servers {
for _, core := range cores[n] {
wg.Add(1)
go func(serv string, core string, acc telegraf.Accumulator) {
defer wg.Done()
@ -188,11 +224,11 @@ func (s *Solr) createHTTPClient() *http.Client {
return client
}
func gatherCoreMetrics(mbeansJson json.RawMessage, core, category string, acc telegraf.Accumulator) error {
func gatherCoreMetrics(mbeansJSON json.RawMessage, core, category string, acc telegraf.Accumulator) error {
var coreMetrics map[string]Core
measurementTime := time.Now()
if err := json.Unmarshal(mbeansJson, &coreMetrics); err != nil {
if err := json.Unmarshal(mbeansJSON, &coreMetrics); err != nil {
return err
}
for name, metrics := range coreMetrics {
@ -217,11 +253,11 @@ func gatherCoreMetrics(mbeansJson json.RawMessage, core, category string, acc te
return nil
}
func gatherQueryHandlerMetrics(mbeansJson json.RawMessage, core, category string, acc telegraf.Accumulator) error {
func gatherQueryHandlerMetrics(mbeansJSON json.RawMessage, core, category string, acc telegraf.Accumulator) error {
var coreMetrics map[string]QueryHandler
measurementTime := time.Now()
if err := json.Unmarshal(mbeansJson, &coreMetrics); err != nil {
if err := json.Unmarshal(mbeansJSON, &coreMetrics); err != nil {
return err
}
for name, metrics := range coreMetrics {
@ -254,11 +290,11 @@ func gatherQueryHandlerMetrics(mbeansJson json.RawMessage, core, category string
return nil
}
func gatherUpdateHandlerMetrics(mbeansJson json.RawMessage, core, category string, acc telegraf.Accumulator) error {
func gatherUpdateHandlerMetrics(mbeansJSON json.RawMessage, core, category string, acc telegraf.Accumulator) error {
var coreMetrics map[string]UpdateHandler
measurementTime := time.Now()
if err := json.Unmarshal(mbeansJson, &coreMetrics); err != nil {
if err := json.Unmarshal(mbeansJSON, &coreMetrics); err != nil {
return err
}
for name, metrics := range coreMetrics {
@ -298,11 +334,11 @@ func gatherUpdateHandlerMetrics(mbeansJson json.RawMessage, core, category strin
return nil
}
func gatherCacheMetrics(mbeansJson json.RawMessage, core, category string, acc telegraf.Accumulator) error {
func gatherCacheMetrics(mbeansJSON json.RawMessage, core, category string, acc telegraf.Accumulator) error {
var coreMetrics map[string]Cache
measurementTime := time.Now()
if err := json.Unmarshal(mbeansJson, &coreMetrics); err != nil {
if err := json.Unmarshal(mbeansJSON, &coreMetrics); err != nil {
return err
}
for name, metrics := range coreMetrics {

14
plugins/inputs/solr/solr_test.go
View File

@ -1,6 +1,7 @@
package solr
import (
"fmt"
"io/ioutil"
"net/http"
"strings"
@ -37,6 +38,19 @@ func (t *transportMock) RoundTrip(r *http.Request) (*http.Response, error) {
func (t *transportMock) CancelRequest(_ *http.Request) {
}
func TestSetDefaults(t *testing.T) {
solr := newSolrWithClient()
solr.Servers = []string{"http://example.com:8983"}
solr.client.Transport = newTransportMock(http.StatusOK, adminCoresResponse)
cores, max, err := solr.setDefaults()
if max != 5 {
err = fmt.Errorf("Received unexpected error: max number of cores: %v, expected 5", max)
}
require.NoError(t, err)
}
func TestGatherClusterStats(t *testing.T) {
solr := newSolrWithClient()
solr.Servers = []string{"http://example.com:8983"}

143
plugins/inputs/solr/testdata_test.go
View File

@ -877,6 +877,149 @@ const cacheStatsResponse = `
}
`
const adminCoresResponse = `
{
"responseHeader": {
"status": 0,
"QTime": 10
},
"defaultCoreName": "main",
"initFailures": {},
"status": {
"core0": {
"name": "core0",
"isDefaultCore": false,
"instanceDir": "solr/core0/",
"dataDir": "/srv/solr-core0/",
"config": "solrconfig.xml",
"schema": "schema.xml",
"startTime": "2016-08-05T00:18:20.871Z",
"uptime": 3054210877,
"index": {
"numDocs": 38732,
"maxDoc": 38732,
"deletedDocs": 0,
"version": 256479,
"segmentCount": 3,
"current": true,
"hasDeletions": false,
"directory": "org.apache.lucene.store.MMapDirectory:org.apache.lucene.store.MMapDirectory@/srv/solr-core0/index.20160606181000006 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@2c9bc1b0",
"userData": {
"commitTimeMSec": "1473397332781"
},
"lastModified": "2016-09-09T05:02:12.781Z",
"sizeInBytes": 5304845,
"size": "5.06 MB"
}
},
"core1": {
"name": "core1",
"isDefaultCore": false,
"instanceDir": "solr/core1/",
"dataDir": "/srv/solr-core1/",
"config": "solrconfig.xml",
"schema": "schema.xml",
"startTime": "2016-08-05T00:18:17.689Z",
"uptime": 3054214059,
"index": {
"numDocs": 1823284,
"maxDoc": 2840737,
"deletedDocs": 1017453,
"version": 4916340,
"segmentCount": 23,
"current": true,
"hasDeletions": true,
"directory": "org.apache.lucene.store.MMapDirectory:org.apache.lucene.store.MMapDirectory@/srv/solr-core1/index.20160606235000107 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@798c8d0e",
"userData": {
"commitTimeMSec": "1473328888638"
},
"lastModified": "2016-09-08T10:01:28.638Z",
"sizeInBytes": 3208083615,
"size": "2.99 GB"
}
},
"main": {
"name": "main",
"isDefaultCore": true,
"instanceDir": "solr/main/",
"dataDir": "/srv/solr/",
"config": "solrconfig.xml",
"schema": "schema.xml",
"startTime": "2016-08-05T00:18:19.517Z",
"uptime": 3054212233,
"index": {
"numDocs": 238785023,
"maxDoc": 250822790,
"deletedDocs": 12037767,
"version": 67802912,
"segmentCount": 43,
"current": true,
"hasDeletions": true,
"directory": "org.apache.lucene.store.MMapDirectory:org.apache.lucene.store.MMapDirectory@/srv/solr/index lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@3e260545",
"userData": {
"commitTimeMSec": "1473400796980"
},
"lastModified": "2016-09-09T05:59:56.98Z",
"sizeInBytes": 372377512357,
"size": "346.8 GB"
}
},
"core2": {
"name": "core2",
"isDefaultCore": false,
"instanceDir": "solr/core2/",
"dataDir": "/srv/solr-core2/",
"config": "solrconfig.xml",
"schema": "schema.xml",
"startTime": "2016-08-05T00:18:20.867Z",
"uptime": 3054210887,
"index": {
"numDocs": 7517469,
"maxDoc": 7538430,
"deletedDocs": 20961,
"version": 266096,
"segmentCount": 22,
"current": true,
"hasDeletions": true,
"directory": "org.apache.lucene.store.MMapDirectory:org.apache.lucene.store.MMapDirectory@/srv/solr-core2/index.20160607000000270 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@52f4b09b",
"userData": {
"commitTimeMSec": "1472812205664"
},
"lastModified": "2016-09-02T10:30:05.664Z",
"sizeInBytes": 1762126735,
"size": "1.64 GB"
}
},
"core3": {
"name": "core3",
"isDefaultCore": false,
"instanceDir": "solr/core3/",
"dataDir": "/srv/solr-core3/",
"config": "solrconfig.xml",
"schema": "schema.xml",
"startTime": "2016-08-05T00:18:19.262Z",
"uptime": 3054212494,
"index": {
"numDocs": 415176,
"maxDoc": 485825,
"deletedDocs": 70649,
"version": 282990385,
"segmentCount": 18,
"current": true,
"hasDeletions": true,
"directory": "org.apache.lucene.store.MMapDirectory:org.apache.lucene.store.MMapDirectory@/srv/solr-core3/index.20160606235000106 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@20107d34",
"userData": {
"commitTimeMSec": "1473410395901"
},
"lastModified": "2016-09-09T08:39:55.901Z",
"sizeInBytes": 939779160,
"size": "896.24 MB"
}
}
}
}
`
var solrCoreExpected = map[string]interface{}{
"class_name": string("org.apache.solr.search.SolrIndexSearcher"),
"num_docs": int(1822549),