Add Marklogic Input Plugin (#6193)

This commit is contained in:
Craig Hobbs 2019-08-21 18:22:00 -07:00 committed by Daniel Nelson
parent 93b41457df
commit f0c26dbd99
4 changed files with 1607 additions and 0 deletions

View File

@ -81,6 +81,7 @@ import (
_ "github.com/influxdata/telegraf/plugins/inputs/logstash"
_ "github.com/influxdata/telegraf/plugins/inputs/lustre2"
_ "github.com/influxdata/telegraf/plugins/inputs/mailchimp"
_ "github.com/influxdata/telegraf/plugins/inputs/marklogic"
_ "github.com/influxdata/telegraf/plugins/inputs/mcrouter"
_ "github.com/influxdata/telegraf/plugins/inputs/mem"
_ "github.com/influxdata/telegraf/plugins/inputs/memcached"

View File

@ -0,0 +1,64 @@
# MarkLogic Plugin
The MarkLogic Telegraf plugin gathers health status metrics from one or more host.
### Configuration:
```toml
[[inputs.marklogic]]
## Base URL of the MarkLogic HTTP Server.
url = "http://localhost:8002"
## List of specific hostnames to retrieve information. At least (1) required.
# hosts = ["hostname1", "hostname2"]
## Using HTTP Basic Authentication. Management API requires 'manage-user' role privileges
# username = "myuser"
# password = "mypassword"
## Optional TLS Config
# tls_ca = "/etc/telegraf/ca.pem"
# tls_cert = "/etc/telegraf/cert.pem"
# tls_key = "/etc/telegraf/key.pem"
## Use TLS but skip chain & host verification
# insecure_skip_verify = false
```
### Metrics
- marklogic
- tags:
- source (the hostname of the server address, ex. `ml1.local`)
- id (the host node unique id ex. `2592913110757471141`)
- fields:
- online
- total_load
- total_rate
- ncpus
- ncores
- total_cpu_stat_user
- total_cpu_stat_system
- total_cpu_stat_idle
- total_cpu_stat_iowait
- memory_process_size
- memory_process_rss
- memory_system_total
- memory_system_free
- memory_process_swap_size
- memory_size
- host_size
- log_device_space
- data_dir_space
- query_read_bytes
- query_read_load
- merge_read_bytes
- merge_write_load
- http_server_receive_bytes
- http_server_send_bytes
### Example Output:
```
$> marklogic,host=localhost,id=2592913110757471141,source=ml1.local total_cpu_stat_iowait=0.0125649003311992,memory_process_swap_size=0i,host_size=380i,data_dir_space=28216i,query_read_load=0i,ncpus=1i,log_device_space=28216i,query_read_bytes=13947332i,merge_write_load=0i,http_server_receive_bytes=225893i,online=true,ncores=4i,total_cpu_stat_user=0.150778993964195,total_cpu_stat_system=0.598927974700928,total_cpu_stat_idle=99.2210006713867,memory_system_total=3947i,memory_system_free=2669i,memory_size=4096i,total_rate=14.7697010040283,http_server_send_bytes=0i,memory_process_size=903i,memory_process_rss=486i,merge_read_load=0i,total_load=0.00502600101754069 1566373000000000000
```

View File

@ -0,0 +1,260 @@
package marklogic
import (
"encoding/json"
"fmt"
"net/http"
"net/url"
"path"
"sync"
"time"
"github.com/influxdata/telegraf"
"github.com/influxdata/telegraf/internal/tls"
"github.com/influxdata/telegraf/plugins/inputs"
)
// Marklogic configuration toml
type Marklogic struct {
URL string `toml:"url"`
Hosts []string `toml:"hosts"`
Username string `toml:"username"`
Password string `toml:"password"`
Sources []string
tls.ClientConfig
client *http.Client
}
type MlPointInt struct {
Value int `json:"value"`
}
type MlPointFloat struct {
Value float64 `json:"value"`
}
type MlPointBool struct {
Value bool `json:"value"`
}
// MarkLogic v2 management api endpoints for hosts status
const statsPath = "/manage/v2/hosts/"
const viewFormat = "view=status&format=json"
type MlHost struct {
HostStatus struct {
ID string `json:"id"`
Name string `json:"name"`
StatusProperties struct {
Online MlPointBool `json:"online"`
LoadProperties struct {
TotalLoad MlPointFloat `json:"total-load"`
} `json:"load-properties"`
RateProperties struct {
TotalRate MlPointFloat `json:"total-rate"`
} `json:"rate-properties"`
StatusDetail struct {
Cpus MlPointInt `json:"cpus"`
Cores MlPointInt `json:"cores"`
TotalCPUStatUser float64 `json:"total-cpu-stat-user"`
TotalCPUStatSystem float64 `json:"total-cpu-stat-system"`
TotalCPUStatIdle float64 `json:"total-cpu-stat-idle"`
TotalCPUStatIowait float64 `json:"total-cpu-stat-iowait"`
MemoryProcessSize MlPointInt `json:"memory-process-size"`
MemoryProcessRss MlPointInt `json:"memory-process-rss"`
MemorySystemTotal MlPointInt `json:"memory-system-total"`
MemorySystemFree MlPointInt `json:"memory-system-free"`
MemoryProcessSwapSize MlPointInt `json:"memory-process-swap-size"`
MemorySize MlPointInt `json:"memory-size"`
HostSize MlPointInt `json:"host-size"`
LogDeviceSpace MlPointInt `json:"log-device-space"`
DataDirSpace MlPointInt `json:"data-dir-space"`
QueryReadBytes MlPointInt `json:"query-read-bytes"`
QueryReadLoad MlPointInt `json:"query-read-load"`
MergeReadLoad MlPointInt `json:"merge-read-load"`
MergeWriteLoad MlPointInt `json:"merge-write-load"`
HTTPServerReceiveBytes MlPointInt `json:"http-server-receive-bytes"`
HTTPServerSendBytes MlPointInt `json:"http-server-send-bytes"`
} `json:"status-detail"`
} `json:"status-properties"`
} `json:"host-status"`
}
// Description of plugin returned
func (c *Marklogic) Description() string {
return "Retrives information on a specific host in a MarkLogic Cluster"
}
var sampleConfig = `
## Base URL of the MarkLogic HTTP Server.
url = "http://localhost:8002"
## List of specific hostnames to retrieve information. At least (1) required.
# hosts = ["hostname1", "hostname2"]
## Using HTTP Basic Authentication. Management API requires 'manage-user' role privileges
# username = "myuser"
# password = "mypassword"
## Optional TLS Config
# tls_ca = "/etc/telegraf/ca.pem"
# tls_cert = "/etc/telegraf/cert.pem"
# tls_key = "/etc/telegraf/key.pem"
## Use TLS but skip chain & host verification
# insecure_skip_verify = false
`
// Init parse all source URLs and place on the Marklogic struct
func (c *Marklogic) Init() error {
if len(c.URL) == 0 {
c.URL = "http://localhost:8002/"
}
for _, u := range c.Hosts {
base, err := url.Parse(c.URL)
if err != nil {
return err
}
base.Path = path.Join(base.Path, statsPath, u)
addr := base.ResolveReference(base)
addr.RawQuery = viewFormat
u := addr.String()
c.Sources = append(c.Sources, u)
}
return nil
}
// SampleConfig to gather stats from localhost, default port.
func (c *Marklogic) SampleConfig() string {
return sampleConfig
}
// Gather metrics from HTTP Server.
func (c *Marklogic) Gather(accumulator telegraf.Accumulator) error {
var wg sync.WaitGroup
if c.client == nil {
client, err := c.createHTTPClient()
if err != nil {
return err
}
c.client = client
}
// Range over all source URL's appended to the struct
for _, serv := range c.Sources {
//fmt.Printf("Encoded URL is %q\n", serv)
wg.Add(1)
go func(serv string) {
defer wg.Done()
if err := c.fetchAndInsertData(accumulator, serv); err != nil {
accumulator.AddError(fmt.Errorf("[host=%s]: %s", serv, err))
}
}(serv)
}
wg.Wait()
return nil
}
func (c *Marklogic) fetchAndInsertData(acc telegraf.Accumulator, url string) error {
ml := &MlHost{}
if err := c.gatherJSONData(url, ml); err != nil {
return err
}
// Build a map of tags
tags := map[string]string{
"source": ml.HostStatus.Name,
"id": ml.HostStatus.ID,
}
// Build a map of field values
fields := map[string]interface{}{
"online": ml.HostStatus.StatusProperties.Online.Value,
"total_load": ml.HostStatus.StatusProperties.LoadProperties.TotalLoad.Value,
"total_rate": ml.HostStatus.StatusProperties.RateProperties.TotalRate.Value,
"ncpus": ml.HostStatus.StatusProperties.StatusDetail.Cpus.Value,
"ncores": ml.HostStatus.StatusProperties.StatusDetail.Cores.Value,
"total_cpu_stat_user": ml.HostStatus.StatusProperties.StatusDetail.TotalCPUStatUser,
"total_cpu_stat_system": ml.HostStatus.StatusProperties.StatusDetail.TotalCPUStatSystem,
"total_cpu_stat_idle": ml.HostStatus.StatusProperties.StatusDetail.TotalCPUStatIdle,
"total_cpu_stat_iowait": ml.HostStatus.StatusProperties.StatusDetail.TotalCPUStatIowait,
"memory_process_size": ml.HostStatus.StatusProperties.StatusDetail.MemoryProcessSize.Value,
"memory_process_rss": ml.HostStatus.StatusProperties.StatusDetail.MemoryProcessRss.Value,
"memory_system_total": ml.HostStatus.StatusProperties.StatusDetail.MemorySystemTotal.Value,
"memory_system_free": ml.HostStatus.StatusProperties.StatusDetail.MemorySystemFree.Value,
"memory_process_swap_size": ml.HostStatus.StatusProperties.StatusDetail.MemoryProcessSwapSize.Value,
"memory_size": ml.HostStatus.StatusProperties.StatusDetail.MemorySize.Value,
"host_size": ml.HostStatus.StatusProperties.StatusDetail.HostSize.Value,
"log_device_space": ml.HostStatus.StatusProperties.StatusDetail.LogDeviceSpace.Value,
"data_dir_space": ml.HostStatus.StatusProperties.StatusDetail.DataDirSpace.Value,
"query_read_bytes": ml.HostStatus.StatusProperties.StatusDetail.QueryReadBytes.Value,
"query_read_load": ml.HostStatus.StatusProperties.StatusDetail.QueryReadLoad.Value,
"merge_read_load": ml.HostStatus.StatusProperties.StatusDetail.MergeReadLoad.Value,
"merge_write_load": ml.HostStatus.StatusProperties.StatusDetail.MergeWriteLoad.Value,
"http_server_receive_bytes": ml.HostStatus.StatusProperties.StatusDetail.HTTPServerReceiveBytes.Value,
"http_server_send_bytes": ml.HostStatus.StatusProperties.StatusDetail.HTTPServerSendBytes.Value,
}
// Accumulate the tags and values
acc.AddFields("marklogic", fields, tags)
return nil
}
func (c *Marklogic) createHTTPClient() (*http.Client, error) {
tlsCfg, err := c.ClientConfig.TLSConfig()
if err != nil {
return nil, err
}
client := &http.Client{
Transport: &http.Transport{
TLSClientConfig: tlsCfg,
},
Timeout: time.Duration(5 * time.Second),
}
return client, nil
}
func (c *Marklogic) gatherJSONData(url string, v interface{}) error {
req, err := http.NewRequest("GET", url, nil)
if err != nil {
return err
}
if c.Username != "" || c.Password != "" {
req.SetBasicAuth(c.Username, c.Password)
}
response, err := c.client.Do(req)
if err != nil {
return err
}
defer response.Body.Close()
if response.StatusCode != http.StatusOK {
return fmt.Errorf("marklogic: API responded with status-code %d, expected %d",
response.StatusCode, http.StatusOK)
}
if err = json.NewDecoder(response.Body).Decode(v); err != nil {
return err
}
return nil
}
func init() {
inputs.Add("marklogic", func() telegraf.Input {
return &Marklogic{}
})
}

File diff suppressed because it is too large Load Diff