2015-12-08 06:47:43 +00:00
|
|
|
package influxdbjson_test
|
Add expvar plugin
This was primarily intended to consume InfluxDB-style expvars,
particularly InfluxDB's `/debug/vars` endpoint.
That endpoint follows a structure like
```json
{
"httpd::8086": {
"name": "httpd",
"tags": {
"bind": ":8086"
},
"values": {
"pointsWrittenOK": 33756,
"queryReq": 19,
"queryRespBytes": 26973,
"req": 428,
"writeReq": 205,
"writeReqBytes": 3939161
}
}
}
```
There are an arbitrary number of top-level keys in the JSON response at
the configured URLs, and this plugin will iterate through all of their
values looking for objects with keys "name", "tags", and "values"
indicating a metric to be consumed by telegraf.
Running this on current master of InfluxDB, I am able to record nearly
the same information that is normally stored in the `_internal`
database; the only measurement missing from `_internal` is `runtime`,
which is present in expvars under the "memstats" key but does not follow
the expvar format and so is not consumed in this plugin.
```
$ influx -database=telegraf -execute 'SHOW FIELD KEYS FROM /expvar/'
name: expvar_influxdb_engine
----------------------------
fieldKey
blksWrite
blksWriteBytes
blksWriteBytesC
pointsWrite
pointsWriteDedupe
name: expvar_influxdb_httpd
---------------------------
fieldKey
pingReq
pointsWrittenOK
queryReq
queryRespBytes
req
writeReq
writeReqBytes
name: expvar_influxdb_shard
---------------------------
fieldKey
fieldsCreate
seriesCreate
writePointsOk
writeReq
name: expvar_influxdb_subscriber
--------------------------------
fieldKey
pointsWritten
name: expvar_influxdb_wal
-------------------------
fieldKey
autoFlush
flushDuration
idleFlush
memSize
metaFlush
pointsFlush
pointsWrite
pointsWriteReq
seriesFlush
name: expvar_influxdb_write
---------------------------
fieldKey
pointReq
pointReqLocal
req
subWriteOk
writeOk
```
2015-12-05 22:15:58 +00:00
|
|
|
|
|
|
|
import (
|
|
|
|
"net/http"
|
|
|
|
"net/http/httptest"
|
|
|
|
"testing"
|
|
|
|
|
2015-12-08 06:47:43 +00:00
|
|
|
"github.com/influxdb/telegraf/plugins/influxdbjson"
|
Add expvar plugin
This was primarily intended to consume InfluxDB-style expvars,
particularly InfluxDB's `/debug/vars` endpoint.
That endpoint follows a structure like
```json
{
"httpd::8086": {
"name": "httpd",
"tags": {
"bind": ":8086"
},
"values": {
"pointsWrittenOK": 33756,
"queryReq": 19,
"queryRespBytes": 26973,
"req": 428,
"writeReq": 205,
"writeReqBytes": 3939161
}
}
}
```
There are an arbitrary number of top-level keys in the JSON response at
the configured URLs, and this plugin will iterate through all of their
values looking for objects with keys "name", "tags", and "values"
indicating a metric to be consumed by telegraf.
Running this on current master of InfluxDB, I am able to record nearly
the same information that is normally stored in the `_internal`
database; the only measurement missing from `_internal` is `runtime`,
which is present in expvars under the "memstats" key but does not follow
the expvar format and so is not consumed in this plugin.
```
$ influx -database=telegraf -execute 'SHOW FIELD KEYS FROM /expvar/'
name: expvar_influxdb_engine
----------------------------
fieldKey
blksWrite
blksWriteBytes
blksWriteBytesC
pointsWrite
pointsWriteDedupe
name: expvar_influxdb_httpd
---------------------------
fieldKey
pingReq
pointsWrittenOK
queryReq
queryRespBytes
req
writeReq
writeReqBytes
name: expvar_influxdb_shard
---------------------------
fieldKey
fieldsCreate
seriesCreate
writePointsOk
writeReq
name: expvar_influxdb_subscriber
--------------------------------
fieldKey
pointsWritten
name: expvar_influxdb_wal
-------------------------
fieldKey
autoFlush
flushDuration
idleFlush
memSize
metaFlush
pointsFlush
pointsWrite
pointsWriteReq
seriesFlush
name: expvar_influxdb_write
---------------------------
fieldKey
pointReq
pointReqLocal
req
subWriteOk
writeOk
```
2015-12-05 22:15:58 +00:00
|
|
|
"github.com/influxdb/telegraf/testutil"
|
|
|
|
"github.com/stretchr/testify/require"
|
|
|
|
)
|
|
|
|
|
|
|
|
func TestBasic(t *testing.T) {
|
|
|
|
js := `
|
|
|
|
{
|
|
|
|
"_1": {
|
|
|
|
"name": "foo",
|
|
|
|
"tags": {
|
|
|
|
"id": "ex1"
|
|
|
|
},
|
|
|
|
"values": {
|
|
|
|
"i": -1,
|
|
|
|
"f": 0.5,
|
|
|
|
"b": true,
|
|
|
|
"s": "string"
|
|
|
|
}
|
|
|
|
},
|
|
|
|
"ignored": {
|
|
|
|
"willBeRecorded": false
|
|
|
|
},
|
|
|
|
"ignoredAndNested": {
|
|
|
|
"hash": {
|
|
|
|
"is": "nested"
|
|
|
|
}
|
|
|
|
},
|
|
|
|
"array": [
|
|
|
|
"makes parsing more difficult than necessary"
|
|
|
|
],
|
|
|
|
"string": "makes parsing more difficult than necessary",
|
|
|
|
"_2": {
|
|
|
|
"name": "bar",
|
|
|
|
"tags": {
|
|
|
|
"id": "ex2"
|
|
|
|
},
|
|
|
|
"values": {
|
|
|
|
"x": "x"
|
|
|
|
}
|
|
|
|
},
|
|
|
|
"pointWithoutFields_willNotBeIncluded": {
|
|
|
|
"name": "asdf",
|
|
|
|
"tags": {
|
|
|
|
"id": "ex3"
|
|
|
|
},
|
|
|
|
"values": {}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
`
|
|
|
|
fakeServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
2015-12-08 06:47:43 +00:00
|
|
|
if r.URL.Path == "/endpoint" {
|
Add expvar plugin
This was primarily intended to consume InfluxDB-style expvars,
particularly InfluxDB's `/debug/vars` endpoint.
That endpoint follows a structure like
```json
{
"httpd::8086": {
"name": "httpd",
"tags": {
"bind": ":8086"
},
"values": {
"pointsWrittenOK": 33756,
"queryReq": 19,
"queryRespBytes": 26973,
"req": 428,
"writeReq": 205,
"writeReqBytes": 3939161
}
}
}
```
There are an arbitrary number of top-level keys in the JSON response at
the configured URLs, and this plugin will iterate through all of their
values looking for objects with keys "name", "tags", and "values"
indicating a metric to be consumed by telegraf.
Running this on current master of InfluxDB, I am able to record nearly
the same information that is normally stored in the `_internal`
database; the only measurement missing from `_internal` is `runtime`,
which is present in expvars under the "memstats" key but does not follow
the expvar format and so is not consumed in this plugin.
```
$ influx -database=telegraf -execute 'SHOW FIELD KEYS FROM /expvar/'
name: expvar_influxdb_engine
----------------------------
fieldKey
blksWrite
blksWriteBytes
blksWriteBytesC
pointsWrite
pointsWriteDedupe
name: expvar_influxdb_httpd
---------------------------
fieldKey
pingReq
pointsWrittenOK
queryReq
queryRespBytes
req
writeReq
writeReqBytes
name: expvar_influxdb_shard
---------------------------
fieldKey
fieldsCreate
seriesCreate
writePointsOk
writeReq
name: expvar_influxdb_subscriber
--------------------------------
fieldKey
pointsWritten
name: expvar_influxdb_wal
-------------------------
fieldKey
autoFlush
flushDuration
idleFlush
memSize
metaFlush
pointsFlush
pointsWrite
pointsWriteReq
seriesFlush
name: expvar_influxdb_write
---------------------------
fieldKey
pointReq
pointReqLocal
req
subWriteOk
writeOk
```
2015-12-05 22:15:58 +00:00
|
|
|
_, _ = w.Write([]byte(js))
|
|
|
|
} else {
|
|
|
|
w.WriteHeader(http.StatusNotFound)
|
|
|
|
}
|
|
|
|
}))
|
|
|
|
defer fakeServer.Close()
|
|
|
|
|
2015-12-08 06:47:43 +00:00
|
|
|
plugin := &influxdbjson.InfluxDBJSON{
|
|
|
|
Name: "test",
|
|
|
|
URLs: []string{fakeServer.URL + "/endpoint"},
|
Add expvar plugin
This was primarily intended to consume InfluxDB-style expvars,
particularly InfluxDB's `/debug/vars` endpoint.
That endpoint follows a structure like
```json
{
"httpd::8086": {
"name": "httpd",
"tags": {
"bind": ":8086"
},
"values": {
"pointsWrittenOK": 33756,
"queryReq": 19,
"queryRespBytes": 26973,
"req": 428,
"writeReq": 205,
"writeReqBytes": 3939161
}
}
}
```
There are an arbitrary number of top-level keys in the JSON response at
the configured URLs, and this plugin will iterate through all of their
values looking for objects with keys "name", "tags", and "values"
indicating a metric to be consumed by telegraf.
Running this on current master of InfluxDB, I am able to record nearly
the same information that is normally stored in the `_internal`
database; the only measurement missing from `_internal` is `runtime`,
which is present in expvars under the "memstats" key but does not follow
the expvar format and so is not consumed in this plugin.
```
$ influx -database=telegraf -execute 'SHOW FIELD KEYS FROM /expvar/'
name: expvar_influxdb_engine
----------------------------
fieldKey
blksWrite
blksWriteBytes
blksWriteBytesC
pointsWrite
pointsWriteDedupe
name: expvar_influxdb_httpd
---------------------------
fieldKey
pingReq
pointsWrittenOK
queryReq
queryRespBytes
req
writeReq
writeReqBytes
name: expvar_influxdb_shard
---------------------------
fieldKey
fieldsCreate
seriesCreate
writePointsOk
writeReq
name: expvar_influxdb_subscriber
--------------------------------
fieldKey
pointsWritten
name: expvar_influxdb_wal
-------------------------
fieldKey
autoFlush
flushDuration
idleFlush
memSize
metaFlush
pointsFlush
pointsWrite
pointsWriteReq
seriesFlush
name: expvar_influxdb_write
---------------------------
fieldKey
pointReq
pointReqLocal
req
subWriteOk
writeOk
```
2015-12-05 22:15:58 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
var acc testutil.Accumulator
|
2015-12-08 06:47:43 +00:00
|
|
|
require.NoError(t, plugin.Gather(&acc))
|
Add expvar plugin
This was primarily intended to consume InfluxDB-style expvars,
particularly InfluxDB's `/debug/vars` endpoint.
That endpoint follows a structure like
```json
{
"httpd::8086": {
"name": "httpd",
"tags": {
"bind": ":8086"
},
"values": {
"pointsWrittenOK": 33756,
"queryReq": 19,
"queryRespBytes": 26973,
"req": 428,
"writeReq": 205,
"writeReqBytes": 3939161
}
}
}
```
There are an arbitrary number of top-level keys in the JSON response at
the configured URLs, and this plugin will iterate through all of their
values looking for objects with keys "name", "tags", and "values"
indicating a metric to be consumed by telegraf.
Running this on current master of InfluxDB, I am able to record nearly
the same information that is normally stored in the `_internal`
database; the only measurement missing from `_internal` is `runtime`,
which is present in expvars under the "memstats" key but does not follow
the expvar format and so is not consumed in this plugin.
```
$ influx -database=telegraf -execute 'SHOW FIELD KEYS FROM /expvar/'
name: expvar_influxdb_engine
----------------------------
fieldKey
blksWrite
blksWriteBytes
blksWriteBytesC
pointsWrite
pointsWriteDedupe
name: expvar_influxdb_httpd
---------------------------
fieldKey
pingReq
pointsWrittenOK
queryReq
queryRespBytes
req
writeReq
writeReqBytes
name: expvar_influxdb_shard
---------------------------
fieldKey
fieldsCreate
seriesCreate
writePointsOk
writeReq
name: expvar_influxdb_subscriber
--------------------------------
fieldKey
pointsWritten
name: expvar_influxdb_wal
-------------------------
fieldKey
autoFlush
flushDuration
idleFlush
memSize
metaFlush
pointsFlush
pointsWrite
pointsWriteReq
seriesFlush
name: expvar_influxdb_write
---------------------------
fieldKey
pointReq
pointReqLocal
req
subWriteOk
writeOk
```
2015-12-05 22:15:58 +00:00
|
|
|
|
|
|
|
require.Len(t, acc.Points, 2)
|
|
|
|
require.NoError(t, acc.ValidateTaggedFieldsValue(
|
|
|
|
"test_foo",
|
|
|
|
map[string]interface{}{
|
|
|
|
// JSON will truncate floats to integer representations.
|
|
|
|
// Since there's no distinction in JSON, we can't assume it's an int.
|
|
|
|
"i": -1.0,
|
|
|
|
"f": 0.5,
|
|
|
|
"b": true,
|
|
|
|
"s": "string",
|
|
|
|
},
|
|
|
|
map[string]string{
|
2015-12-08 06:47:43 +00:00
|
|
|
"id": "ex1",
|
|
|
|
"influxdbjson_url": fakeServer.URL + "/endpoint",
|
Add expvar plugin
This was primarily intended to consume InfluxDB-style expvars,
particularly InfluxDB's `/debug/vars` endpoint.
That endpoint follows a structure like
```json
{
"httpd::8086": {
"name": "httpd",
"tags": {
"bind": ":8086"
},
"values": {
"pointsWrittenOK": 33756,
"queryReq": 19,
"queryRespBytes": 26973,
"req": 428,
"writeReq": 205,
"writeReqBytes": 3939161
}
}
}
```
There are an arbitrary number of top-level keys in the JSON response at
the configured URLs, and this plugin will iterate through all of their
values looking for objects with keys "name", "tags", and "values"
indicating a metric to be consumed by telegraf.
Running this on current master of InfluxDB, I am able to record nearly
the same information that is normally stored in the `_internal`
database; the only measurement missing from `_internal` is `runtime`,
which is present in expvars under the "memstats" key but does not follow
the expvar format and so is not consumed in this plugin.
```
$ influx -database=telegraf -execute 'SHOW FIELD KEYS FROM /expvar/'
name: expvar_influxdb_engine
----------------------------
fieldKey
blksWrite
blksWriteBytes
blksWriteBytesC
pointsWrite
pointsWriteDedupe
name: expvar_influxdb_httpd
---------------------------
fieldKey
pingReq
pointsWrittenOK
queryReq
queryRespBytes
req
writeReq
writeReqBytes
name: expvar_influxdb_shard
---------------------------
fieldKey
fieldsCreate
seriesCreate
writePointsOk
writeReq
name: expvar_influxdb_subscriber
--------------------------------
fieldKey
pointsWritten
name: expvar_influxdb_wal
-------------------------
fieldKey
autoFlush
flushDuration
idleFlush
memSize
metaFlush
pointsFlush
pointsWrite
pointsWriteReq
seriesFlush
name: expvar_influxdb_write
---------------------------
fieldKey
pointReq
pointReqLocal
req
subWriteOk
writeOk
```
2015-12-05 22:15:58 +00:00
|
|
|
},
|
|
|
|
))
|
|
|
|
require.NoError(t, acc.ValidateTaggedFieldsValue(
|
|
|
|
"test_bar",
|
|
|
|
map[string]interface{}{
|
|
|
|
"x": "x",
|
|
|
|
},
|
|
|
|
map[string]string{
|
2015-12-08 06:47:43 +00:00
|
|
|
"id": "ex2",
|
|
|
|
"influxdbjson_url": fakeServer.URL + "/endpoint",
|
Add expvar plugin
This was primarily intended to consume InfluxDB-style expvars,
particularly InfluxDB's `/debug/vars` endpoint.
That endpoint follows a structure like
```json
{
"httpd::8086": {
"name": "httpd",
"tags": {
"bind": ":8086"
},
"values": {
"pointsWrittenOK": 33756,
"queryReq": 19,
"queryRespBytes": 26973,
"req": 428,
"writeReq": 205,
"writeReqBytes": 3939161
}
}
}
```
There are an arbitrary number of top-level keys in the JSON response at
the configured URLs, and this plugin will iterate through all of their
values looking for objects with keys "name", "tags", and "values"
indicating a metric to be consumed by telegraf.
Running this on current master of InfluxDB, I am able to record nearly
the same information that is normally stored in the `_internal`
database; the only measurement missing from `_internal` is `runtime`,
which is present in expvars under the "memstats" key but does not follow
the expvar format and so is not consumed in this plugin.
```
$ influx -database=telegraf -execute 'SHOW FIELD KEYS FROM /expvar/'
name: expvar_influxdb_engine
----------------------------
fieldKey
blksWrite
blksWriteBytes
blksWriteBytesC
pointsWrite
pointsWriteDedupe
name: expvar_influxdb_httpd
---------------------------
fieldKey
pingReq
pointsWrittenOK
queryReq
queryRespBytes
req
writeReq
writeReqBytes
name: expvar_influxdb_shard
---------------------------
fieldKey
fieldsCreate
seriesCreate
writePointsOk
writeReq
name: expvar_influxdb_subscriber
--------------------------------
fieldKey
pointsWritten
name: expvar_influxdb_wal
-------------------------
fieldKey
autoFlush
flushDuration
idleFlush
memSize
metaFlush
pointsFlush
pointsWrite
pointsWriteReq
seriesFlush
name: expvar_influxdb_write
---------------------------
fieldKey
pointReq
pointReqLocal
req
subWriteOk
writeOk
```
2015-12-05 22:15:58 +00:00
|
|
|
},
|
|
|
|
))
|
|
|
|
}
|