From 0379c7309cecc71a22401decd419518d6851d321 Mon Sep 17 00:00:00 2001 From: Cameron Sparr Date: Fri, 17 Feb 2017 19:36:44 +0000 Subject: [PATCH] switch out deprecated docker client library closes #2071 --- CHANGELOG.md | 1 + Godeps | 5 +- plugins/inputs/docker/README.md | 16 ++- plugins/inputs/docker/docker.go | 63 ++++++++--- plugins/inputs/docker/docker_test.go | 150 ++------------------------- plugins/inputs/docker/fake_client.go | 143 +++++++++++++++++++++++++ 6 files changed, 210 insertions(+), 168 deletions(-) create mode 100644 plugins/inputs/docker/fake_client.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 8febc3e50..509d6f2f1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -48,6 +48,7 @@ be deprecated eventually. - [#2201](https://github.com/influxdata/telegraf/pull/2201): Add lock option to the IPtables input plugin. - [#2244](https://github.com/influxdata/telegraf/pull/2244): Support ipmi_sensor plugin querying local ipmi sensors. - [#2339](https://github.com/influxdata/telegraf/pull/2339): Increment gather_errors for all errors emitted by inputs. +- [#2071](https://github.com/influxdata/telegraf/issues/2071): Use official docker SDK. ### Bugfixes diff --git a/Godeps b/Godeps index fb8e23860..de326cb19 100644 --- a/Godeps +++ b/Godeps @@ -9,10 +9,7 @@ github.com/couchbase/go-couchbase bfe555a140d53dc1adf390f1a1d4b0fd4ceadb28 github.com/couchbase/gomemcached 4a25d2f4e1dea9ea7dd76dfd943407abf9b07d29 github.com/couchbase/goutils 5823a0cbaaa9008406021dc5daf80125ea30bba6 github.com/davecgh/go-spew 346938d642f2ec3594ed81d874461961cd0faa76 -github.com/docker/distribution fb0bebc4b64e3881cc52a2478d749845ed76d2a8 -github.com/docker/engine-api 4290f40c056686fcaa5c9caf02eac1dde9315adf -github.com/docker/go-connections 9670439d95da2651d9dfc7acc5d2ed92d3f25ee6 -github.com/docker/go-units 0dadbb0345b35ec7ef35e228dabb8de89a65bf52 +github.com/docker/docker b89aff1afa1f61993ab2ba18fd62d9375a195f5d github.com/eapache/go-resiliency b86b1ec0dd4209a588dc1285cdd471e73525c0b3 github.com/eapache/go-xerial-snappy bb955e01b9346ac19dc29eb16586c90ded99a98c github.com/eapache/queue 44cc805cf13205b55f69e14bcb69867d1ae92f98 diff --git a/plugins/inputs/docker/README.md b/plugins/inputs/docker/README.md index 5e8910677..94965213f 100644 --- a/plugins/inputs/docker/README.md +++ b/plugins/inputs/docker/README.md @@ -16,12 +16,20 @@ for the stat structure can be found ``` # Read metrics about docker containers [[inputs.docker]] - # Docker Endpoint - # To use TCP, set endpoint = "tcp://[ip]:[port]" - # To use environment variables (ie, docker-machine), set endpoint = "ENV" + ## Docker Endpoint + ## To use TCP, set endpoint = "tcp://[ip]:[port]" + ## To use environment variables (ie, docker-machine), set endpoint = "ENV" endpoint = "unix:///var/run/docker.sock" - # Only collect metrics for these containers, collect all if empty + ## Only collect metrics for these containers, collect all if empty container_names = [] + ## Timeout for docker list, info, and stats commands + timeout = "5s" + + ## Whether to report for each container per-device blkio (8:0, 8:1...) and + ## network (eth0, eth1, ...) stats or not + perdevice = true + ## Whether to report for each container total blkio and network stats or not + total = false ``` ### Measurements & Fields: diff --git a/plugins/inputs/docker/docker.go b/plugins/inputs/docker/docker.go index 82a3791b6..ec192efd5 100644 --- a/plugins/inputs/docker/docker.go +++ b/plugins/inputs/docker/docker.go @@ -1,6 +1,7 @@ -package system +package docker import ( + "context" "encoding/json" "fmt" "io" @@ -11,10 +12,9 @@ import ( "sync" "time" - "golang.org/x/net/context" + "github.com/docker/docker/api/types" + "github.com/docker/docker/client" - "github.com/docker/engine-api/client" - "github.com/docker/engine-api/types" "github.com/influxdata/telegraf" "github.com/influxdata/telegraf/internal" "github.com/influxdata/telegraf/plugins/inputs" @@ -28,15 +28,46 @@ type Docker struct { PerDevice bool `toml:"perdevice"` Total bool `toml:"total"` - client DockerClient + client *client.Client engine_host string + + testing bool } -// DockerClient interface, useful for testing -type DockerClient interface { - Info(ctx context.Context) (types.Info, error) - ContainerList(ctx context.Context, options types.ContainerListOptions) ([]types.Container, error) - ContainerStats(ctx context.Context, containerID string, stream bool) (io.ReadCloser, error) +// infoWrapper wraps client.Client.List for testing. +func infoWrapper(c *client.Client, ctx context.Context) (types.Info, error) { + if c != nil { + return c.Info(ctx) + } + fc := FakeDockerClient{} + return fc.Info(ctx) +} + +// listWrapper wraps client.Client.ContainerList for testing. +func listWrapper( + c *client.Client, + ctx context.Context, + options types.ContainerListOptions, +) ([]types.Container, error) { + if c != nil { + return c.ContainerList(ctx, options) + } + fc := FakeDockerClient{} + return fc.ContainerList(ctx, options) +} + +// statsWrapper wraps client.Client.ContainerStats for testing. +func statsWrapper( + c *client.Client, + ctx context.Context, + containerID string, + stream bool, +) (types.ContainerStats, error) { + if c != nil { + return c.ContainerStats(ctx, containerID, stream) + } + fc := FakeDockerClient{} + return fc.ContainerStats(ctx, containerID, stream) } // KB, MB, GB, TB, PB...human friendly @@ -80,7 +111,7 @@ func (d *Docker) SampleConfig() string { return sampleConfig } // Gather starts stats collection func (d *Docker) Gather(acc telegraf.Accumulator) error { - if d.client == nil { + if d.client == nil && !d.testing { var c *client.Client var err error defaultHeaders := map[string]string{"User-Agent": "engine-api-cli-1.0"} @@ -113,7 +144,7 @@ func (d *Docker) Gather(acc telegraf.Accumulator) error { opts := types.ContainerListOptions{} ctx, cancel := context.WithTimeout(context.Background(), d.Timeout.Duration) defer cancel() - containers, err := d.client.ContainerList(ctx, opts) + containers, err := listWrapper(d.client, ctx, opts) if err != nil { return err } @@ -144,7 +175,7 @@ func (d *Docker) gatherInfo(acc telegraf.Accumulator) error { // Get info from docker daemon ctx, cancel := context.WithTimeout(context.Background(), d.Timeout.Duration) defer cancel() - info, err := d.client.Info(ctx) + info, err := infoWrapper(d.client, ctx) if err != nil { return err } @@ -247,12 +278,12 @@ func (d *Docker) gatherContainer( ctx, cancel := context.WithTimeout(context.Background(), d.Timeout.Duration) defer cancel() - r, err := d.client.ContainerStats(ctx, container.ID, false) + r, err := statsWrapper(d.client, ctx, container.ID, false) if err != nil { return fmt.Errorf("Error getting docker stats: %s", err.Error()) } - defer r.Close() - dec := json.NewDecoder(r) + defer r.Body.Close() + dec := json.NewDecoder(r.Body) if err = dec.Decode(&v); err != nil { if err == io.EOF { return nil diff --git a/plugins/inputs/docker/docker_test.go b/plugins/inputs/docker/docker_test.go index cc0ada3c4..f0add03ea 100644 --- a/plugins/inputs/docker/docker_test.go +++ b/plugins/inputs/docker/docker_test.go @@ -1,18 +1,12 @@ -package system +package docker import ( - "io" - "io/ioutil" - "strings" "testing" "time" - "golang.org/x/net/context" - - "github.com/docker/engine-api/types" - "github.com/docker/engine-api/types/registry" "github.com/influxdata/telegraf/testutil" + "github.com/docker/docker/api/types" "github.com/stretchr/testify/require" ) @@ -250,146 +244,14 @@ func testStats() *types.StatsJSON { return stats } -type FakeDockerClient struct { -} - -func (d FakeDockerClient) Info(ctx context.Context) (types.Info, error) { - env := types.Info{ - Containers: 108, - ContainersRunning: 98, - ContainersStopped: 6, - ContainersPaused: 3, - OomKillDisable: false, - SystemTime: "2016-02-24T00:55:09.15073105-05:00", - NEventsListener: 0, - ID: "5WQQ:TFWR:FDNG:OKQ3:37Y4:FJWG:QIKK:623T:R3ME:QTKB:A7F7:OLHD", - Debug: false, - LoggingDriver: "json-file", - KernelVersion: "4.3.0-1-amd64", - IndexServerAddress: "https://index.docker.io/v1/", - MemTotal: 3840757760, - Images: 199, - CPUCfsQuota: true, - Name: "absol", - SwapLimit: false, - IPv4Forwarding: true, - ExperimentalBuild: false, - CPUCfsPeriod: true, - RegistryConfig: ®istry.ServiceConfig{ - IndexConfigs: map[string]*registry.IndexInfo{ - "docker.io": { - Name: "docker.io", - Mirrors: []string{}, - Official: true, - Secure: true, - }, - }, InsecureRegistryCIDRs: []*registry.NetIPNet{{IP: []byte{127, 0, 0, 0}, Mask: []byte{255, 0, 0, 0}}}, Mirrors: []string{}}, - OperatingSystem: "Linux Mint LMDE (containerized)", - BridgeNfIptables: true, - HTTPSProxy: "", - Labels: []string{}, - MemoryLimit: false, - DriverStatus: [][2]string{{"Pool Name", "docker-8:1-1182287-pool"}, {"Pool Blocksize", "65.54 kB"}, {"Backing Filesystem", "extfs"}, {"Data file", "/dev/loop0"}, {"Metadata file", "/dev/loop1"}, {"Data Space Used", "17.3 GB"}, {"Data Space Total", "107.4 GB"}, {"Data Space Available", "36.53 GB"}, {"Metadata Space Used", "20.97 MB"}, {"Metadata Space Total", "2.147 GB"}, {"Metadata Space Available", "2.127 GB"}, {"Udev Sync Supported", "true"}, {"Deferred Removal Enabled", "false"}, {"Data loop file", "/var/lib/docker/devicemapper/devicemapper/data"}, {"Metadata loop file", "/var/lib/docker/devicemapper/devicemapper/metadata"}, {"Library Version", "1.02.115 (2016-01-25)"}}, - NFd: 19, - HTTPProxy: "", - Driver: "devicemapper", - NGoroutines: 39, - NCPU: 4, - DockerRootDir: "/var/lib/docker", - NoProxy: "", - BridgeNfIP6tables: true, - } - return env, nil -} - -func (d FakeDockerClient) ContainerList(octx context.Context, options types.ContainerListOptions) ([]types.Container, error) { - container1 := types.Container{ - ID: "e2173b9478a6ae55e237d4d74f8bbb753f0817192b5081334dc78476296b7dfb", - Names: []string{"/etcd"}, - Image: "quay.io/coreos/etcd:v2.2.2", - Command: "/etcd -name etcd0 -advertise-client-urls http://localhost:2379 -listen-client-urls http://0.0.0.0:2379", - Created: 1455941930, - Status: "Up 4 hours", - Ports: []types.Port{ - types.Port{ - PrivatePort: 7001, - PublicPort: 0, - Type: "tcp", - }, - types.Port{ - PrivatePort: 4001, - PublicPort: 0, - Type: "tcp", - }, - types.Port{ - PrivatePort: 2380, - PublicPort: 0, - Type: "tcp", - }, - types.Port{ - PrivatePort: 2379, - PublicPort: 2379, - Type: "tcp", - IP: "0.0.0.0", - }, - }, - SizeRw: 0, - SizeRootFs: 0, - } - container2 := types.Container{ - ID: "b7dfbb9478a6ae55e237d4d74f8bbb753f0817192b5081334dc78476296e2173", - Names: []string{"/etcd2"}, - Image: "quay.io:4443/coreos/etcd:v2.2.2", - Command: "/etcd -name etcd2 -advertise-client-urls http://localhost:2379 -listen-client-urls http://0.0.0.0:2379", - Created: 1455941933, - Status: "Up 4 hours", - Ports: []types.Port{ - types.Port{ - PrivatePort: 7002, - PublicPort: 0, - Type: "tcp", - }, - types.Port{ - PrivatePort: 4002, - PublicPort: 0, - Type: "tcp", - }, - types.Port{ - PrivatePort: 2381, - PublicPort: 0, - Type: "tcp", - }, - types.Port{ - PrivatePort: 2382, - PublicPort: 2382, - Type: "tcp", - IP: "0.0.0.0", - }, - }, - SizeRw: 0, - SizeRootFs: 0, - } - - containers := []types.Container{container1, container2} - return containers, nil - - //#{e6a96c84ca91a5258b7cb752579fb68826b68b49ff957487695cd4d13c343b44 titilambert/snmpsim /bin/sh -c 'snmpsimd --agent-udpv4-endpoint=0.0.0.0:31161 --process-user=root --process-group=user' 1455724831 Up 4 hours [{31161 31161 udp 0.0.0.0}] 0 0 [/snmp] map[]}]2016/02/24 01:05:01 Gathered metrics, (3s interval), from 1 inputs in 1.233836656s -} - -func (d FakeDockerClient) ContainerStats(ctx context.Context, containerID string, stream bool) (io.ReadCloser, error) { - var stat io.ReadCloser - jsonStat := `{"read":"2016-02-24T11:42:27.472459608-05:00","memory_stats":{"stats":{},"limit":18935443456},"blkio_stats":{"io_service_bytes_recursive":[{"major":252,"minor":1,"op":"Read","value":753664},{"major":252,"minor":1,"op":"Write"},{"major":252,"minor":1,"op":"Sync"},{"major":252,"minor":1,"op":"Async","value":753664},{"major":252,"minor":1,"op":"Total","value":753664}],"io_serviced_recursive":[{"major":252,"minor":1,"op":"Read","value":26},{"major":252,"minor":1,"op":"Write"},{"major":252,"minor":1,"op":"Sync"},{"major":252,"minor":1,"op":"Async","value":26},{"major":252,"minor":1,"op":"Total","value":26}]},"cpu_stats":{"cpu_usage":{"percpu_usage":[17871,4959158,1646137,1231652,11829401,244656,369972,0],"usage_in_usermode":10000000,"total_usage":20298847},"system_cpu_usage":24052607520000000,"throttling_data":{}},"precpu_stats":{"cpu_usage":{"percpu_usage":[17871,4959158,1646137,1231652,11829401,244656,369972,0],"usage_in_usermode":10000000,"total_usage":20298847},"system_cpu_usage":24052599550000000,"throttling_data":{}}}` - stat = ioutil.NopCloser(strings.NewReader(jsonStat)) - return stat, nil -} - func TestDockerGatherInfo(t *testing.T) { var acc testutil.Accumulator - client := FakeDockerClient{} - d := Docker{client: client} + d := Docker{ + client: nil, + testing: true, + } err := d.Gather(&acc) - require.NoError(t, err) acc.AssertContainsTaggedFields(t, diff --git a/plugins/inputs/docker/fake_client.go b/plugins/inputs/docker/fake_client.go new file mode 100644 index 000000000..03da23198 --- /dev/null +++ b/plugins/inputs/docker/fake_client.go @@ -0,0 +1,143 @@ +package docker + +import ( + "context" + "io/ioutil" + "strings" + + "github.com/docker/docker/api/types" + "github.com/docker/docker/api/types/registry" +) + +type FakeDockerClient struct { +} + +func (d FakeDockerClient) Info(ctx context.Context) (types.Info, error) { + env := types.Info{ + Containers: 108, + ContainersRunning: 98, + ContainersStopped: 6, + ContainersPaused: 3, + OomKillDisable: false, + SystemTime: "2016-02-24T00:55:09.15073105-05:00", + NEventsListener: 0, + ID: "5WQQ:TFWR:FDNG:OKQ3:37Y4:FJWG:QIKK:623T:R3ME:QTKB:A7F7:OLHD", + Debug: false, + LoggingDriver: "json-file", + KernelVersion: "4.3.0-1-amd64", + IndexServerAddress: "https://index.docker.io/v1/", + MemTotal: 3840757760, + Images: 199, + CPUCfsQuota: true, + Name: "absol", + SwapLimit: false, + IPv4Forwarding: true, + ExperimentalBuild: false, + CPUCfsPeriod: true, + RegistryConfig: ®istry.ServiceConfig{ + IndexConfigs: map[string]*registry.IndexInfo{ + "docker.io": { + Name: "docker.io", + Mirrors: []string{}, + Official: true, + Secure: true, + }, + }, InsecureRegistryCIDRs: []*registry.NetIPNet{{IP: []byte{127, 0, 0, 0}, Mask: []byte{255, 0, 0, 0}}}, Mirrors: []string{}}, + OperatingSystem: "Linux Mint LMDE (containerized)", + BridgeNfIptables: true, + HTTPSProxy: "", + Labels: []string{}, + MemoryLimit: false, + DriverStatus: [][2]string{{"Pool Name", "docker-8:1-1182287-pool"}, {"Pool Blocksize", "65.54 kB"}, {"Backing Filesystem", "extfs"}, {"Data file", "/dev/loop0"}, {"Metadata file", "/dev/loop1"}, {"Data Space Used", "17.3 GB"}, {"Data Space Total", "107.4 GB"}, {"Data Space Available", "36.53 GB"}, {"Metadata Space Used", "20.97 MB"}, {"Metadata Space Total", "2.147 GB"}, {"Metadata Space Available", "2.127 GB"}, {"Udev Sync Supported", "true"}, {"Deferred Removal Enabled", "false"}, {"Data loop file", "/var/lib/docker/devicemapper/devicemapper/data"}, {"Metadata loop file", "/var/lib/docker/devicemapper/devicemapper/metadata"}, {"Library Version", "1.02.115 (2016-01-25)"}}, + NFd: 19, + HTTPProxy: "", + Driver: "devicemapper", + NGoroutines: 39, + NCPU: 4, + DockerRootDir: "/var/lib/docker", + NoProxy: "", + BridgeNfIP6tables: true, + } + return env, nil +} + +func (d FakeDockerClient) ContainerList(octx context.Context, options types.ContainerListOptions) ([]types.Container, error) { + container1 := types.Container{ + ID: "e2173b9478a6ae55e237d4d74f8bbb753f0817192b5081334dc78476296b7dfb", + Names: []string{"/etcd"}, + Image: "quay.io/coreos/etcd:v2.2.2", + Command: "/etcd -name etcd0 -advertise-client-urls http://localhost:2379 -listen-client-urls http://0.0.0.0:2379", + Created: 1455941930, + Status: "Up 4 hours", + Ports: []types.Port{ + types.Port{ + PrivatePort: 7001, + PublicPort: 0, + Type: "tcp", + }, + types.Port{ + PrivatePort: 4001, + PublicPort: 0, + Type: "tcp", + }, + types.Port{ + PrivatePort: 2380, + PublicPort: 0, + Type: "tcp", + }, + types.Port{ + PrivatePort: 2379, + PublicPort: 2379, + Type: "tcp", + IP: "0.0.0.0", + }, + }, + SizeRw: 0, + SizeRootFs: 0, + } + container2 := types.Container{ + ID: "b7dfbb9478a6ae55e237d4d74f8bbb753f0817192b5081334dc78476296e2173", + Names: []string{"/etcd2"}, + Image: "quay.io:4443/coreos/etcd:v2.2.2", + Command: "/etcd -name etcd2 -advertise-client-urls http://localhost:2379 -listen-client-urls http://0.0.0.0:2379", + Created: 1455941933, + Status: "Up 4 hours", + Ports: []types.Port{ + types.Port{ + PrivatePort: 7002, + PublicPort: 0, + Type: "tcp", + }, + types.Port{ + PrivatePort: 4002, + PublicPort: 0, + Type: "tcp", + }, + types.Port{ + PrivatePort: 2381, + PublicPort: 0, + Type: "tcp", + }, + types.Port{ + PrivatePort: 2382, + PublicPort: 2382, + Type: "tcp", + IP: "0.0.0.0", + }, + }, + SizeRw: 0, + SizeRootFs: 0, + } + + containers := []types.Container{container1, container2} + return containers, nil + + //#{e6a96c84ca91a5258b7cb752579fb68826b68b49ff957487695cd4d13c343b44 titilambert/snmpsim /bin/sh -c 'snmpsimd --agent-udpv4-endpoint=0.0.0.0:31161 --process-user=root --process-group=user' 1455724831 Up 4 hours [{31161 31161 udp 0.0.0.0}] 0 0 [/snmp] map[]}]2016/02/24 01:05:01 Gathered metrics, (3s interval), from 1 inputs in 1.233836656s +} + +func (d FakeDockerClient) ContainerStats(ctx context.Context, containerID string, stream bool) (types.ContainerStats, error) { + var stat types.ContainerStats + jsonStat := `{"read":"2016-02-24T11:42:27.472459608-05:00","memory_stats":{"stats":{},"limit":18935443456},"blkio_stats":{"io_service_bytes_recursive":[{"major":252,"minor":1,"op":"Read","value":753664},{"major":252,"minor":1,"op":"Write"},{"major":252,"minor":1,"op":"Sync"},{"major":252,"minor":1,"op":"Async","value":753664},{"major":252,"minor":1,"op":"Total","value":753664}],"io_serviced_recursive":[{"major":252,"minor":1,"op":"Read","value":26},{"major":252,"minor":1,"op":"Write"},{"major":252,"minor":1,"op":"Sync"},{"major":252,"minor":1,"op":"Async","value":26},{"major":252,"minor":1,"op":"Total","value":26}]},"cpu_stats":{"cpu_usage":{"percpu_usage":[17871,4959158,1646137,1231652,11829401,244656,369972,0],"usage_in_usermode":10000000,"total_usage":20298847},"system_cpu_usage":24052607520000000,"throttling_data":{}},"precpu_stats":{"cpu_usage":{"percpu_usage":[17871,4959158,1646137,1231652,11829401,244656,369972,0],"usage_in_usermode":10000000,"total_usage":20298847},"system_cpu_usage":24052599550000000,"throttling_data":{}}}` + stat.Body = ioutil.NopCloser(strings.NewReader(jsonStat)) + return stat, nil +}