From b065573e23f761870831d06790ff233a4626fe56 Mon Sep 17 00:00:00 2001 From: Cameron Sparr Date: Wed, 18 May 2016 19:31:36 +0100 Subject: [PATCH] influxdb input: Add shard counter closes #1221 --- plugins/inputs/influxdb/README.md | 124 +++++++++++------------ plugins/inputs/influxdb/influxdb.go | 16 +++ plugins/inputs/influxdb/influxdb_test.go | 14 ++- 3 files changed, 90 insertions(+), 64 deletions(-) diff --git a/plugins/inputs/influxdb/README.md b/plugins/inputs/influxdb/README.md index 8e10cf2c6..433191495 100644 --- a/plugins/inputs/influxdb/README.md +++ b/plugins/inputs/influxdb/README.md @@ -22,16 +22,78 @@ InfluxDB-formatted endpoints. See below for more information. ### Measurements & Fields +- influxdb + - n_shards - influxdb_database - influxdb_httpd - influxdb_measurement - influxdb_memstats + - heap_inuse + - heap_released + - mspan_inuse + - total_alloc + - sys + - mallocs + - frees + - heap_idle + - pause_total_ns + - lookups + - heap_sys + - mcache_sys + - next_gc + - gcc_pu_fraction + - other_sys + - alloc + - stack_inuse + - stack_sys + - buck_hash_sys + - gc_sys + - num_gc + - heap_alloc + - heap_objects + - mspan_sys + - mcache_inuse + - last_gc - influxdb_shard - influxdb_subscriber - influxdb_tsm1_cache - influxdb_tsm1_wal - influxdb_write +### Example Output: + +``` +telegraf -config ~/ws/telegraf.conf -input-filter influxdb -test +* Plugin: influxdb, Collection 1 +> influxdb_database,database=_internal,host=tyrion,url=http://localhost:8086/debug/vars numMeasurements=10,numSeries=29 1463590500247354636 +> influxdb_httpd,bind=:8086,host=tyrion,url=http://localhost:8086/debug/vars req=7,reqActive=1,reqDurationNs=14227734 1463590500247354636 +> influxdb_measurement,database=_internal,host=tyrion,measurement=database,url=http://localhost:8086/debug/vars numSeries=1 1463590500247354636 +> influxdb_measurement,database=_internal,host=tyrion,measurement=httpd,url=http://localhost:8086/debug/vars numSeries=1 1463590500247354636 +> influxdb_measurement,database=_internal,host=tyrion,measurement=measurement,url=http://localhost:8086/debug/vars numSeries=10 1463590500247354636 +> influxdb_measurement,database=_internal,host=tyrion,measurement=runtime,url=http://localhost:8086/debug/vars numSeries=1 1463590500247354636 +> influxdb_measurement,database=_internal,host=tyrion,measurement=shard,url=http://localhost:8086/debug/vars numSeries=4 1463590500247354636 +> influxdb_measurement,database=_internal,host=tyrion,measurement=subscriber,url=http://localhost:8086/debug/vars numSeries=1 1463590500247354636 +> influxdb_measurement,database=_internal,host=tyrion,measurement=tsm1_cache,url=http://localhost:8086/debug/vars numSeries=4 1463590500247354636 +> influxdb_measurement,database=_internal,host=tyrion,measurement=tsm1_filestore,url=http://localhost:8086/debug/vars numSeries=2 1463590500247354636 +> influxdb_measurement,database=_internal,host=tyrion,measurement=tsm1_wal,url=http://localhost:8086/debug/vars numSeries=4 1463590500247354636 +> influxdb_measurement,database=_internal,host=tyrion,measurement=write,url=http://localhost:8086/debug/vars numSeries=1 1463590500247354636 +> influxdb_memstats,host=tyrion,url=http://localhost:8086/debug/vars alloc=7642384i,buck_hash_sys=1463471i,frees=1169558i,gc_sys=653312i,gcc_pu_fraction=0.00003825652361068311,heap_alloc=7642384i,heap_idle=9912320i,heap_inuse=9125888i,heap_objects=48276i,heap_released=0i,heap_sys=19038208i,last_gc=1463590480877651621i,lookups=90i,mallocs=1217834i,mcache_inuse=4800i,mcache_sys=16384i,mspan_inuse=70920i,mspan_sys=81920i,next_gc=11679787i,num_gc=141i,other_sys=1244233i,pause_total_ns=24034027i,stack_inuse=884736i,stack_sys=884736i,sys=23382264i,total_alloc=679012200i 1463590500277918755 +> influxdb_shard,database=_internal,engine=tsm1,host=tyrion,id=4,path=/Users/sparrc/.influxdb/data/_internal/monitor/4,retentionPolicy=monitor,url=http://localhost:8086/debug/vars fieldsCreate=65,seriesCreate=26,writePointsOk=7274,writeReq=280 1463590500247354636 +> influxdb_subscriber,host=tyrion,url=http://localhost:8086/debug/vars pointsWritten=7274 1463590500247354636 +> influxdb_tsm1_cache,database=_internal,host=tyrion,path=/Users/sparrc/.influxdb/data/_internal/monitor/1,retentionPolicy=monitor,url=http://localhost:8086/debug/vars WALCompactionTimeMs=0,cacheAgeMs=2809192,cachedBytes=0,diskBytes=0,memBytes=0,snapshotCount=0 1463590500247354636 +> influxdb_tsm1_cache,database=_internal,host=tyrion,path=/Users/sparrc/.influxdb/data/_internal/monitor/2,retentionPolicy=monitor,url=http://localhost:8086/debug/vars WALCompactionTimeMs=0,cacheAgeMs=2809184,cachedBytes=0,diskBytes=0,memBytes=0,snapshotCount=0 1463590500247354636 +> influxdb_tsm1_cache,database=_internal,host=tyrion,path=/Users/sparrc/.influxdb/data/_internal/monitor/3,retentionPolicy=monitor,url=http://localhost:8086/debug/vars WALCompactionTimeMs=0,cacheAgeMs=2809180,cachedBytes=0,diskBytes=0,memBytes=42368,snapshotCount=0 1463590500247354636 +> influxdb_tsm1_cache,database=_internal,host=tyrion,path=/Users/sparrc/.influxdb/data/_internal/monitor/4,retentionPolicy=monitor,url=http://localhost:8086/debug/vars WALCompactionTimeMs=0,cacheAgeMs=2799155,cachedBytes=0,diskBytes=0,memBytes=331216,snapshotCount=0 1463590500247354636 +> influxdb_tsm1_filestore,database=_internal,host=tyrion,path=/Users/sparrc/.influxdb/data/_internal/monitor/1,retentionPolicy=monitor,url=http://localhost:8086/debug/vars diskBytes=37892 1463590500247354636 +> influxdb_tsm1_filestore,database=_internal,host=tyrion,path=/Users/sparrc/.influxdb/data/_internal/monitor/2,retentionPolicy=monitor,url=http://localhost:8086/debug/vars diskBytes=52907 1463590500247354636 +> influxdb_tsm1_wal,database=_internal,host=tyrion,path=/Users/sparrc/.influxdb/wal/_internal/monitor/1,retentionPolicy=monitor,url=http://localhost:8086/debug/vars currentSegmentDiskBytes=0,oldSegmentsDiskBytes=0 1463590500247354636 +> influxdb_tsm1_wal,database=_internal,host=tyrion,path=/Users/sparrc/.influxdb/wal/_internal/monitor/2,retentionPolicy=monitor,url=http://localhost:8086/debug/vars currentSegmentDiskBytes=0,oldSegmentsDiskBytes=0 1463590500247354636 +> influxdb_tsm1_wal,database=_internal,host=tyrion,path=/Users/sparrc/.influxdb/wal/_internal/monitor/3,retentionPolicy=monitor,url=http://localhost:8086/debug/vars currentSegmentDiskBytes=0,oldSegmentsDiskBytes=65651 1463590500247354636 +> influxdb_tsm1_wal,database=_internal,host=tyrion,path=/Users/sparrc/.influxdb/wal/_internal/monitor/4,retentionPolicy=monitor,url=http://localhost:8086/debug/vars currentSegmentDiskBytes=495687,oldSegmentsDiskBytes=0 1463590500247354636 +> influxdb_write,host=tyrion,url=http://localhost:8086/debug/vars pointReq=7274,pointReqLocal=7274,req=280,subWriteOk=280,writeOk=280 1463590500247354636 +> influxdb_shard,host=tyrion n_shards=4i 1463590500247354636 +``` + ### InfluxDB-formatted endpoints The influxdb plugin can collect InfluxDB-formatted data from JSON endpoints. @@ -46,65 +108,3 @@ With a configuration of: "http://192.168.2.1:8086/debug/vars" ] ``` - -And if 127.0.0.1 responds with this JSON: - -```json -{ - "k1": { - "name": "fruit", - "tags": { - "kind": "apple" - }, - "values": { - "inventory": 371, - "sold": 112 - } - }, - "k2": { - "name": "fruit", - "tags": { - "kind": "banana" - }, - "values": { - "inventory": 1000, - "sold": 403 - } - } -} -``` - -And if 192.168.2.1 responds like so: - -```json -{ - "k3": { - "name": "transactions", - "tags": {}, - "values": { - "total": 100, - "balance": 184.75 - } - } -} -``` - -Then the collected metrics will be: - -``` -influxdb_fruit,url='http://127.0.0.1:8086/debug/vars',kind='apple' inventory=371.0,sold=112.0 -influxdb_fruit,url='http://127.0.0.1:8086/debug/vars',kind='banana' inventory=1000.0,sold=403.0 - -influxdb_transactions,url='http://192.168.2.1:8086/debug/vars' total=100.0,balance=184.75 -``` - -There are two important details to note about the collected metrics: - -1. Even though the values in JSON are being displayed as integers, -the metrics are reported as floats. -JSON encoders usually don't print the fractional part for round floats. -Because you cannot change the type of an existing field in InfluxDB, -we assume all numbers are floats. - -2. The top-level keys' names (in the example above, `"k1"`, `"k2"`, and `"k3"`) -are not considered when recording the metrics. diff --git a/plugins/inputs/influxdb/influxdb.go b/plugins/inputs/influxdb/influxdb.go index e5756835f..a0baaa266 100644 --- a/plugins/inputs/influxdb/influxdb.go +++ b/plugins/inputs/influxdb/influxdb.go @@ -120,6 +120,9 @@ func (i *InfluxDB) gatherURL( acc telegraf.Accumulator, url string, ) error { + shardCounter := 0 + now := time.Now() + resp, err := client.Get(url) if err != nil { return err @@ -201,6 +204,10 @@ func (i *InfluxDB) gatherURL( continue } + if p.Name == "shard" { + shardCounter++ + } + // If the object was a point, but was not fully initialized, // ignore it and move on. if p.Name == "" || p.Tags == nil || p.Values == nil || len(p.Values) == 0 { @@ -214,9 +221,18 @@ func (i *InfluxDB) gatherURL( "influxdb_"+p.Name, p.Values, p.Tags, + now, ) } + acc.AddFields("influxdb", + map[string]interface{}{ + "n_shards": shardCounter, + }, + nil, + now, + ) + return nil } diff --git a/plugins/inputs/influxdb/influxdb_test.go b/plugins/inputs/influxdb/influxdb_test.go index 63056d0ed..9125b606a 100644 --- a/plugins/inputs/influxdb/influxdb_test.go +++ b/plugins/inputs/influxdb/influxdb_test.go @@ -27,7 +27,7 @@ func TestBasic(t *testing.T) { var acc testutil.Accumulator require.NoError(t, plugin.Gather(&acc)) - require.Len(t, acc.Metrics, 2) + require.Len(t, acc.Metrics, 3) fields := map[string]interface{}{ // JSON will truncate floats to integer representations. // Since there's no distinction in JSON, we can't assume it's an int. @@ -50,6 +50,11 @@ func TestBasic(t *testing.T) { "url": fakeServer.URL + "/endpoint", } acc.AssertContainsTaggedFields(t, "influxdb_bar", fields, tags) + + acc.AssertContainsTaggedFields(t, "influxdb", + map[string]interface{}{ + "n_shards": 0, + }, map[string]string{}) } func TestInfluxDB(t *testing.T) { @@ -69,7 +74,7 @@ func TestInfluxDB(t *testing.T) { var acc testutil.Accumulator require.NoError(t, plugin.Gather(&acc)) - require.Len(t, acc.Metrics, 33) + require.Len(t, acc.Metrics, 34) fields := map[string]interface{}{ "heap_inuse": int64(18046976), @@ -104,6 +109,11 @@ func TestInfluxDB(t *testing.T) { "url": fakeInfluxServer.URL + "/endpoint", } acc.AssertContainsTaggedFields(t, "influxdb_memstats", fields, tags) + + acc.AssertContainsTaggedFields(t, "influxdb", + map[string]interface{}{ + "n_shards": 2, + }, map[string]string{}) } func TestErrorHandling(t *testing.T) {