From dd265ca2697280d41e3e5fe885e87f7c2deb9fcb Mon Sep 17 00:00:00 2001 From: Jonathan Cross Date: Tue, 13 Oct 2015 19:15:39 -0400 Subject: [PATCH 1/4] Zookeeper plugin MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Created a zookeeper plugin that fetches from the ‘mntr’ command will output measurements that are int and string based --- plugins/all/all.go | 1 + plugins/zookeeper/zookeeper.go | 107 ++++++++++++++++++++++++++++ plugins/zookeeper/zookeeper_test.go | 30 ++++++++ 3 files changed, 138 insertions(+) create mode 100644 plugins/zookeeper/zookeeper.go create mode 100644 plugins/zookeeper/zookeeper_test.go diff --git a/plugins/all/all.go b/plugins/all/all.go index 1cb115bfc..364ca3949 100644 --- a/plugins/all/all.go +++ b/plugins/all/all.go @@ -22,4 +22,5 @@ import ( _ "github.com/influxdb/telegraf/plugins/redis" _ "github.com/influxdb/telegraf/plugins/rethinkdb" _ "github.com/influxdb/telegraf/plugins/system" + _ "github.com/influxdb/telegraf/plugins/zookeeper" ) diff --git a/plugins/zookeeper/zookeeper.go b/plugins/zookeeper/zookeeper.go new file mode 100644 index 000000000..8619bc976 --- /dev/null +++ b/plugins/zookeeper/zookeeper.go @@ -0,0 +1,107 @@ +package zookeeper + +import ( + "bufio" + "fmt" + "net" + "os" + "regexp" + "strconv" + "strings" + "time" + + "github.com/influxdb/telegraf/plugins" +) + +// Zookeeper is a zookeeper plugin +type Zookeeper struct { + Servers []string +} + +var sampleConfig = ` + # An array of address to gather stats about. Specify an ip or hostname + # with port. ie localhost:2181, 10.0.0.1:2181, etc. + + # If no servers are specified, then localhost is used as the host. + # If no port is specified, 2181 is used + servers = [":2181"] +` + +var defaultTimeout = time.Second * time.Duration(5) + +// SampleConfig returns sample configuration message +func (z *Zookeeper) SampleConfig() string { + return sampleConfig +} + +// Description returns description of Zookeeper plugin +func (z *Zookeeper) Description() string { + return `Reads 'mntr' stats from one or many zookeeper servers` +} + +// Gather reads stats from all configured servers accumulates stats +func (z *Zookeeper) Gather(acc plugins.Accumulator) error { + if len(z.Servers) == 0 { + return nil + } + + for _, serverAddress := range z.Servers { + if err := z.gatherServer(serverAddress, acc); err != nil { + return err + } + } + return nil +} + +func (z *Zookeeper) gatherServer(address string, acc plugins.Accumulator) error { + _, _, err := net.SplitHostPort(address) + if err != nil { + address = address + ":2181" + } + + c, err := net.DialTimeout("tcp", address, defaultTimeout) + if err != nil { + fmt.Fprintln(os.Stderr, err) + return err + } + defer c.Close() + + fmt.Fprintf(c, "%s\n", "mntr") + + rdr := bufio.NewReader(c) + + scanner := bufio.NewScanner(rdr) + + for scanner.Scan() { + line := scanner.Text() + + re := regexp.MustCompile(`^zk_(\w+)\s+([\w\.\-]+)`) + parts := re.FindStringSubmatch(string(line)) + + service := strings.Split(address, ":") + + if len(parts) != 3 || len(service) != 2 { + return fmt.Errorf("unexpected line in mntr response: %q", line) + } + + tags := map[string]string{"server": service[0], "port": service[1]} + + measurement := strings.TrimPrefix(parts[1], "zk_") + sValue := string(parts[2]) + + iVal, err := strconv.ParseInt(sValue, 10, 64) + if err == nil { + acc.Add(measurement, iVal, tags) + } else { + acc.Add(measurement, sValue, tags) + } + } + + return nil +} + +func init() { + plugins.Add("zookeeper", func() plugins.Plugin { + return &Zookeeper{} + }) +} diff --git a/plugins/zookeeper/zookeeper_test.go b/plugins/zookeeper/zookeeper_test.go new file mode 100644 index 000000000..2f8fba114 --- /dev/null +++ b/plugins/zookeeper/zookeeper_test.go @@ -0,0 +1,30 @@ +package zookeeper + +import ( + "testing" + + "github.com/influxdb/telegraf/testutil" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestMemcachedGeneratesMetrics(t *testing.T) { + if testing.Short() { + t.Skip("Skipping integration test in short mode") + } + + z := &Zookeeper{ + Servers: []string{testutil.GetLocalHost()}, + } + + var acc testutil.Accumulator + + err := z.Gather(&acc) + require.NoError(t, err) + + intMetrics := []string{"zookeeper_avg_latency", "zookeeper_packets_sent", "zookeeper_znode_count", "zookeeper_open_file_descriptor_count", "zookeeper_max_file_descriptor_count"} + + for _, metric := range intMetrics { + assert.True(t, acc.HasIntValue(metric), metric) + } +} From 9fad8abed9e0fdd3f01537109e3eb352f420eb42 Mon Sep 17 00:00:00 2001 From: Jonathan Cross Date: Tue, 13 Oct 2015 22:01:07 -0400 Subject: [PATCH 2/4] fixes based on comments --- plugins/zookeeper/README.md | 62 +++++++++++++++++++++++++++++ plugins/zookeeper/zookeeper_test.go | 16 +++++++- scripts/docker-compose.yml | 5 +++ 3 files changed, 81 insertions(+), 2 deletions(-) create mode 100644 plugins/zookeeper/README.md diff --git a/plugins/zookeeper/README.md b/plugins/zookeeper/README.md new file mode 100644 index 000000000..c2dc6b540 --- /dev/null +++ b/plugins/zookeeper/README.md @@ -0,0 +1,62 @@ +## Telegraf Plugin: Zookeeper + +#### Description + +The zookeeper plugin collects variables outputted from the 'mntr' command +[Zookeeper Admin](https://zookeeper.apache.org/doc/trunk/zookeeperAdmin.html). + +``` +echo mntr | nc localhost 2181 + + zk_version 3.4.0 + zk_avg_latency 0 + zk_max_latency 0 + zk_min_latency 0 + zk_packets_received 70 + zk_packets_sent 69 + zk_outstanding_requests 0 + zk_server_state leader + zk_znode_count 4 + zk_watch_count 0 + zk_ephemerals_count 0 + zk_approximate_data_size 27 + zk_followers 4 - only exposed by the Leader + zk_synced_followers 4 - only exposed by the Leader + zk_pending_syncs 0 - only exposed by the Leader + zk_open_file_descriptor_count 23 - only available on Unix platforms + zk_max_file_descriptor_count 1024 - only available on Unix platforms +``` + +## Measurements: +#### Zookeeper measurements: + +Meta: +- units: int64 +- tags: `server= port=` + +Measurement names: +- avg_latency +- max_latency +- min_latency +- packets_received +- packets_sent +- outstanding_requests +- znode_count +- watch_count +- ephemerals_count +- approximate_data_size +- followers #only exposed by the Leader +- synced_followers #only exposed by the Leader +- pending_syncs #only exposed by the Leader +- open_file_descriptor_count +- max_file_descriptor_count + +#### Zookeeper string measurements: + +Meta: +- units: string +- tags: `server= port=` + +Measurement names: +- zk_version +- server_state \ No newline at end of file diff --git a/plugins/zookeeper/zookeeper_test.go b/plugins/zookeeper/zookeeper_test.go index 2f8fba114..ba4bd9cab 100644 --- a/plugins/zookeeper/zookeeper_test.go +++ b/plugins/zookeeper/zookeeper_test.go @@ -8,7 +8,7 @@ import ( "github.com/stretchr/testify/require" ) -func TestMemcachedGeneratesMetrics(t *testing.T) { +func TestZookeeperGeneratesMetrics(t *testing.T) { if testing.Short() { t.Skip("Skipping integration test in short mode") } @@ -22,7 +22,19 @@ func TestMemcachedGeneratesMetrics(t *testing.T) { err := z.Gather(&acc) require.NoError(t, err) - intMetrics := []string{"zookeeper_avg_latency", "zookeeper_packets_sent", "zookeeper_znode_count", "zookeeper_open_file_descriptor_count", "zookeeper_max_file_descriptor_count"} + intMetrics := []string{"zookeeper_avg_latency", + "zookeeper_max_latency", + "zookeeper_min_latency", + "zookeeper_packets_received", + "zookeeper_packets_sent", + "zookeeper_outstanding_requests", + "zookeeper_znode_count", + "zookeeper_watch_count", + "zookeeper_ephemerals_count", + "zookeeper_approximate_data_size", + "zookeeper_pending_syncs", + "zookeeper_open_file_descriptor_count", + "zookeeper_max_file_descriptor_count"} for _, metric := range intMetrics { assert.True(t, acc.HasIntValue(metric), metric) diff --git a/scripts/docker-compose.yml b/scripts/docker-compose.yml index a04d9ff91..3e72bb814 100644 --- a/scripts/docker-compose.yml +++ b/scripts/docker-compose.yml @@ -42,3 +42,8 @@ redis: image: redis ports: - "6379:6379" + +zookeeper: + image: spotify/kafka + ports: + - "2181:2181" From de43dcbdbc97a1c1d1ee452cf5b8ea7b2506fc44 Mon Sep 17 00:00:00 2001 From: Jonathan Cross Date: Wed, 14 Oct 2015 09:15:56 -0400 Subject: [PATCH 3/4] added measurement prefix --- plugins/zookeeper/README.md | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/plugins/zookeeper/README.md b/plugins/zookeeper/README.md index c2dc6b540..fe7a8a4ad 100644 --- a/plugins/zookeeper/README.md +++ b/plugins/zookeeper/README.md @@ -35,21 +35,21 @@ Meta: - tags: `server= port=` Measurement names: -- avg_latency -- max_latency -- min_latency -- packets_received -- packets_sent -- outstanding_requests -- znode_count -- watch_count -- ephemerals_count -- approximate_data_size -- followers #only exposed by the Leader -- synced_followers #only exposed by the Leader -- pending_syncs #only exposed by the Leader -- open_file_descriptor_count -- max_file_descriptor_count +- zookeeper_avg_latency +- zookeeper_max_latency +- zookeeper_min_latency +- zookeeper_packets_received +- zookeeper_packets_sent +- zookeeper_outstanding_requests +- zookeeper_znode_count +- zookeeper_watch_count +- zookeeper_ephemerals_count +- zookeeper_approximate_data_size +- zookeeper_followers #only exposed by the Leader +- zookeeper_synced_followers #only exposed by the Leader +- zookeeper_pending_syncs #only exposed by the Leader +- zookeeper_open_file_descriptor_count +- zookeeper_max_file_descriptor_count #### Zookeeper string measurements: @@ -58,5 +58,5 @@ Meta: - tags: `server= port=` Measurement names: -- zk_version -- server_state \ No newline at end of file +- zookeeper_version +- zookeeper_server_state \ No newline at end of file From 188806bef2976025951d84287122b5f027a16a64 Mon Sep 17 00:00:00 2001 From: Jonathan Cross Date: Wed, 14 Oct 2015 11:54:02 -0400 Subject: [PATCH 4/4] remove zookeeper declaration since spotify/kafka docker image already exposes zookeeper --- scripts/docker-compose.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/scripts/docker-compose.yml b/scripts/docker-compose.yml index 3e72bb814..a04d9ff91 100644 --- a/scripts/docker-compose.yml +++ b/scripts/docker-compose.yml @@ -42,8 +42,3 @@ redis: image: redis ports: - "6379:6379" - -zookeeper: - image: spotify/kafka - ports: - - "2181:2181"