Add Marklogic Input Plugin (#6193)

plugins/inputs/all/all.go

@@ -81,6 +81,7 @@ import (
 	_ "github.com/influxdata/telegraf/plugins/inputs/logstash"
 	_ "github.com/influxdata/telegraf/plugins/inputs/lustre2"
 	_ "github.com/influxdata/telegraf/plugins/inputs/mailchimp"
+	_ "github.com/influxdata/telegraf/plugins/inputs/marklogic"
 	_ "github.com/influxdata/telegraf/plugins/inputs/mcrouter"
 	_ "github.com/influxdata/telegraf/plugins/inputs/mem"
 	_ "github.com/influxdata/telegraf/plugins/inputs/memcached"

plugins/inputs/marklogic/README.md

@@ -0,0 +1,64 @@
+# MarkLogic Plugin
+
+The MarkLogic Telegraf plugin gathers health status metrics from one or more host.
+
+### Configuration:
+
+```toml
+[[inputs.marklogic]]
+  ## Base URL of the MarkLogic HTTP Server.
+  url = "http://localhost:8002"
+
+  ## List of specific hostnames to retrieve information. At least (1) required.
+  # hosts = ["hostname1", "hostname2"]
+
+  ## Using HTTP Basic Authentication. Management API requires 'manage-user' role privileges
+  # username = "myuser"
+  # password = "mypassword"
+
+  ## Optional TLS Config
+  # tls_ca = "/etc/telegraf/ca.pem"
+  # tls_cert = "/etc/telegraf/cert.pem"
+  # tls_key = "/etc/telegraf/key.pem"
+  ## Use TLS but skip chain & host verification
+  # insecure_skip_verify = false
+```
+
+### Metrics
+
+- marklogic
+  - tags:
+    - source (the hostname of the server address, ex. `ml1.local`)
+    - id (the host node unique id ex. `2592913110757471141`)
+  - fields:
+    - online
+    - total_load
+    - total_rate
+    - ncpus
+    - ncores
+    - total_cpu_stat_user
+    - total_cpu_stat_system
+    - total_cpu_stat_idle
+    - total_cpu_stat_iowait
+    - memory_process_size
+    - memory_process_rss
+    - memory_system_total
+    - memory_system_free
+    - memory_process_swap_size
+    - memory_size
+    - host_size
+    - log_device_space
+    - data_dir_space
+    - query_read_bytes
+    - query_read_load
+    - merge_read_bytes
+    - merge_write_load
+    - http_server_receive_bytes
+    - http_server_send_bytes
+
+### Example Output:
+
+```
+$> marklogic,host=localhost,id=2592913110757471141,source=ml1.local total_cpu_stat_iowait=0.0125649003311992,memory_process_swap_size=0i,host_size=380i,data_dir_space=28216i,query_read_load=0i,ncpus=1i,log_device_space=28216i,query_read_bytes=13947332i,merge_write_load=0i,http_server_receive_bytes=225893i,online=true,ncores=4i,total_cpu_stat_user=0.150778993964195,total_cpu_stat_system=0.598927974700928,total_cpu_stat_idle=99.2210006713867,memory_system_total=3947i,memory_system_free=2669i,memory_size=4096i,total_rate=14.7697010040283,http_server_send_bytes=0i,memory_process_size=903i,memory_process_rss=486i,merge_read_load=0i,total_load=0.00502600101754069 1566373000000000000
+
+```

plugins/inputs/marklogic/marklogic.go

@@ -0,0 +1,260 @@
+package marklogic
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"net/url"
+	"path"
+	"sync"
+	"time"
+
+	"github.com/influxdata/telegraf"
+	"github.com/influxdata/telegraf/internal/tls"
+	"github.com/influxdata/telegraf/plugins/inputs"
+)
+
+// Marklogic configuration toml
+type Marklogic struct {
+	URL      string   `toml:"url"`
+	Hosts    []string `toml:"hosts"`
+	Username string   `toml:"username"`
+	Password string   `toml:"password"`
+	Sources  []string
+
+	tls.ClientConfig
+
+	client *http.Client
+}
+
+type MlPointInt struct {
+	Value int `json:"value"`
+}
+
+type MlPointFloat struct {
+	Value float64 `json:"value"`
+}
+
+type MlPointBool struct {
+	Value bool `json:"value"`
+}
+
+// MarkLogic v2 management api endpoints for hosts status
+const statsPath = "/manage/v2/hosts/"
+const viewFormat = "view=status&format=json"
+
+type MlHost struct {
+	HostStatus struct {
+		ID               string `json:"id"`
+		Name             string `json:"name"`
+		StatusProperties struct {
+			Online         MlPointBool `json:"online"`
+			LoadProperties struct {
+				TotalLoad MlPointFloat `json:"total-load"`
+			} `json:"load-properties"`
+			RateProperties struct {
+				TotalRate MlPointFloat `json:"total-rate"`
+			} `json:"rate-properties"`
+			StatusDetail struct {
+				Cpus                   MlPointInt `json:"cpus"`
+				Cores                  MlPointInt `json:"cores"`
+				TotalCPUStatUser       float64    `json:"total-cpu-stat-user"`
+				TotalCPUStatSystem     float64    `json:"total-cpu-stat-system"`
+				TotalCPUStatIdle       float64    `json:"total-cpu-stat-idle"`
+				TotalCPUStatIowait     float64    `json:"total-cpu-stat-iowait"`
+				MemoryProcessSize      MlPointInt `json:"memory-process-size"`
+				MemoryProcessRss       MlPointInt `json:"memory-process-rss"`
+				MemorySystemTotal      MlPointInt `json:"memory-system-total"`
+				MemorySystemFree       MlPointInt `json:"memory-system-free"`
+				MemoryProcessSwapSize  MlPointInt `json:"memory-process-swap-size"`
+				MemorySize             MlPointInt `json:"memory-size"`
+				HostSize               MlPointInt `json:"host-size"`
+				LogDeviceSpace         MlPointInt `json:"log-device-space"`
+				DataDirSpace           MlPointInt `json:"data-dir-space"`
+				QueryReadBytes         MlPointInt `json:"query-read-bytes"`
+				QueryReadLoad          MlPointInt `json:"query-read-load"`
+				MergeReadLoad          MlPointInt `json:"merge-read-load"`
+				MergeWriteLoad         MlPointInt `json:"merge-write-load"`
+				HTTPServerReceiveBytes MlPointInt `json:"http-server-receive-bytes"`
+				HTTPServerSendBytes    MlPointInt `json:"http-server-send-bytes"`
+			} `json:"status-detail"`
+		} `json:"status-properties"`
+	} `json:"host-status"`
+}
+
+// Description of plugin returned
+func (c *Marklogic) Description() string {
+	return "Retrives information on a specific host in a MarkLogic Cluster"
+}
+
+var sampleConfig = `
+  ## Base URL of the MarkLogic HTTP Server.
+  url = "http://localhost:8002"
+
+  ## List of specific hostnames to retrieve information. At least (1) required.
+  # hosts = ["hostname1", "hostname2"]
+
+  ## Using HTTP Basic Authentication. Management API requires 'manage-user' role privileges
+  # username = "myuser"
+  # password = "mypassword"
+
+  ## Optional TLS Config
+  # tls_ca = "/etc/telegraf/ca.pem"
+  # tls_cert = "/etc/telegraf/cert.pem"
+  # tls_key = "/etc/telegraf/key.pem"
+  ## Use TLS but skip chain & host verification
+  # insecure_skip_verify = false
+`
+
+// Init parse all source URLs and place on the Marklogic struct
+func (c *Marklogic) Init() error {
+
+	if len(c.URL) == 0 {
+		c.URL = "http://localhost:8002/"
+	}
+
+	for _, u := range c.Hosts {
+		base, err := url.Parse(c.URL)
+		if err != nil {
+			return err
+		}
+
+		base.Path = path.Join(base.Path, statsPath, u)
+		addr := base.ResolveReference(base)
+
+		addr.RawQuery = viewFormat
+		u := addr.String()
+		c.Sources = append(c.Sources, u)
+	}
+	return nil
+}
+
+// SampleConfig to gather stats from localhost, default port.
+func (c *Marklogic) SampleConfig() string {
+	return sampleConfig
+}
+
+// Gather metrics from HTTP Server.
+func (c *Marklogic) Gather(accumulator telegraf.Accumulator) error {
+	var wg sync.WaitGroup
+
+	if c.client == nil {
+		client, err := c.createHTTPClient()
+
+		if err != nil {
+			return err
+		}
+		c.client = client
+	}
+
+	// Range over all source URL's appended to the struct
+	for _, serv := range c.Sources {
+		//fmt.Printf("Encoded URL is %q\n", serv)
+		wg.Add(1)
+		go func(serv string) {
+			defer wg.Done()
+			if err := c.fetchAndInsertData(accumulator, serv); err != nil {
+				accumulator.AddError(fmt.Errorf("[host=%s]: %s", serv, err))
+			}
+		}(serv)
+	}
+
+	wg.Wait()
+
+	return nil
+}
+
+func (c *Marklogic) fetchAndInsertData(acc telegraf.Accumulator, url string) error {
+	ml := &MlHost{}
+	if err := c.gatherJSONData(url, ml); err != nil {
+		return err
+	}
+
+	// Build a map of tags
+	tags := map[string]string{
+		"source": ml.HostStatus.Name,
+		"id":     ml.HostStatus.ID,
+	}
+
+	// Build a map of field values
+	fields := map[string]interface{}{
+		"online":                    ml.HostStatus.StatusProperties.Online.Value,
+		"total_load":                ml.HostStatus.StatusProperties.LoadProperties.TotalLoad.Value,
+		"total_rate":                ml.HostStatus.StatusProperties.RateProperties.TotalRate.Value,
+		"ncpus":                     ml.HostStatus.StatusProperties.StatusDetail.Cpus.Value,
+		"ncores":                    ml.HostStatus.StatusProperties.StatusDetail.Cores.Value,
+		"total_cpu_stat_user":       ml.HostStatus.StatusProperties.StatusDetail.TotalCPUStatUser,
+		"total_cpu_stat_system":     ml.HostStatus.StatusProperties.StatusDetail.TotalCPUStatSystem,
+		"total_cpu_stat_idle":       ml.HostStatus.StatusProperties.StatusDetail.TotalCPUStatIdle,
+		"total_cpu_stat_iowait":     ml.HostStatus.StatusProperties.StatusDetail.TotalCPUStatIowait,
+		"memory_process_size":       ml.HostStatus.StatusProperties.StatusDetail.MemoryProcessSize.Value,
+		"memory_process_rss":        ml.HostStatus.StatusProperties.StatusDetail.MemoryProcessRss.Value,
+		"memory_system_total":       ml.HostStatus.StatusProperties.StatusDetail.MemorySystemTotal.Value,
+		"memory_system_free":        ml.HostStatus.StatusProperties.StatusDetail.MemorySystemFree.Value,
+		"memory_process_swap_size":  ml.HostStatus.StatusProperties.StatusDetail.MemoryProcessSwapSize.Value,
+		"memory_size":               ml.HostStatus.StatusProperties.StatusDetail.MemorySize.Value,
+		"host_size":                 ml.HostStatus.StatusProperties.StatusDetail.HostSize.Value,
+		"log_device_space":          ml.HostStatus.StatusProperties.StatusDetail.LogDeviceSpace.Value,
+		"data_dir_space":            ml.HostStatus.StatusProperties.StatusDetail.DataDirSpace.Value,
+		"query_read_bytes":          ml.HostStatus.StatusProperties.StatusDetail.QueryReadBytes.Value,
+		"query_read_load":           ml.HostStatus.StatusProperties.StatusDetail.QueryReadLoad.Value,
+		"merge_read_load":           ml.HostStatus.StatusProperties.StatusDetail.MergeReadLoad.Value,
+		"merge_write_load":          ml.HostStatus.StatusProperties.StatusDetail.MergeWriteLoad.Value,
+		"http_server_receive_bytes": ml.HostStatus.StatusProperties.StatusDetail.HTTPServerReceiveBytes.Value,
+		"http_server_send_bytes":    ml.HostStatus.StatusProperties.StatusDetail.HTTPServerSendBytes.Value,
+	}
+
+	// Accumulate the tags and values
+	acc.AddFields("marklogic", fields, tags)
+
+	return nil
+}
+
+func (c *Marklogic) createHTTPClient() (*http.Client, error) {
+	tlsCfg, err := c.ClientConfig.TLSConfig()
+	if err != nil {
+		return nil, err
+	}
+
+	client := &http.Client{
+		Transport: &http.Transport{
+			TLSClientConfig: tlsCfg,
+		},
+		Timeout: time.Duration(5 * time.Second),
+	}
+
+	return client, nil
+}
+
+func (c *Marklogic) gatherJSONData(url string, v interface{}) error {
+	req, err := http.NewRequest("GET", url, nil)
+	if err != nil {
+		return err
+	}
+
+	if c.Username != "" || c.Password != "" {
+		req.SetBasicAuth(c.Username, c.Password)
+	}
+
+	response, err := c.client.Do(req)
+	if err != nil {
+		return err
+	}
+	defer response.Body.Close()
+	if response.StatusCode != http.StatusOK {
+		return fmt.Errorf("marklogic: API responded with status-code %d, expected %d",
+			response.StatusCode, http.StatusOK)
+	}
+
+	if err = json.NewDecoder(response.Body).Decode(v); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func init() {
+	inputs.Add("marklogic", func() telegraf.Input {
+		return &Marklogic{}
+	})
+}