Add container id as optional source tag to docker and docker_log input (#6473)

This commit is contained in:
Randy Coburn 2019-10-08 02:27:32 +02:00 committed by Daniel Nelson
parent e7cf8319b0
commit a5294fde32
6 changed files with 104 additions and 5 deletions

View File

@ -26,6 +26,9 @@ to gather stats from the [Engine API](https://docs.docker.com/engine/api/v1.24/)
## Deprecated (1.4.0), use container_name_include ## Deprecated (1.4.0), use container_name_include
container_names = [] container_names = []
## Set the source tag for the metrics to the container ID hostname, eg first 12 chars
source_tag = false
## Containers to include and exclude. Collect all if empty. Globs accepted. ## Containers to include and exclude. Collect all if empty. Globs accepted.
container_name_include = [] container_name_include = []
container_name_exclude = [] container_name_exclude = []
@ -93,6 +96,17 @@ volumes:
- /var/run/docker.sock:/var/run/docker.sock - /var/run/docker.sock:/var/run/docker.sock
``` ```
#### source tag
Selecting the containers measurements can be tricky if you have many containers with the same name.
To alleviate this issue you can set the below value to `true`
```toml
source_tag = true
```
This will cause all measurements to have the `source` tag be set to the first 12 characters of the container id. The first 12 characters is the common hostname for containers that have no explicit hostname set, as defined by docker.
#### Kubernetes Labels #### Kubernetes Labels
Kubernetes may add many labels to your containers, if they are not needed you Kubernetes may add many labels to your containers, if they are not needed you

View File

@ -44,6 +44,8 @@ type Docker struct {
ContainerStateInclude []string `toml:"container_state_include"` ContainerStateInclude []string `toml:"container_state_include"`
ContainerStateExclude []string `toml:"container_state_exclude"` ContainerStateExclude []string `toml:"container_state_exclude"`
IncludeSourceTag bool `toml:"source_tag"`
Log telegraf.Logger Log telegraf.Logger
tlsint.ClientConfig tlsint.ClientConfig
@ -90,6 +92,9 @@ var sampleConfig = `
## Only collect metrics for these containers, collect all if empty ## Only collect metrics for these containers, collect all if empty
container_names = [] container_names = []
## Set the source tag for the metrics to the container ID hostname, eg first 12 chars
source_tag = false
## Containers to include and exclude. Globs accepted. ## Containers to include and exclude. Globs accepted.
## Note that an empty array for both will include all containers ## Note that an empty array for both will include all containers
container_name_include = [] container_name_include = []
@ -412,6 +417,13 @@ func (d *Docker) gatherInfo(acc telegraf.Accumulator) error {
return nil return nil
} }
func hostnameFromID(id string) string {
if len(id) > 12 {
return id[0:12]
}
return id
}
func (d *Docker) gatherContainer( func (d *Docker) gatherContainer(
container types.Container, container types.Container,
acc telegraf.Accumulator, acc telegraf.Accumulator,
@ -443,6 +455,10 @@ func (d *Docker) gatherContainer(
"container_version": imageVersion, "container_version": imageVersion,
} }
if d.IncludeSourceTag {
tags["source"] = hostnameFromID(container.ID)
}
ctx, cancel := context.WithTimeout(context.Background(), d.Timeout.Duration) ctx, cancel := context.WithTimeout(context.Background(), d.Timeout.Duration)
defer cancel() defer cancel()

View File

@ -629,8 +629,9 @@ func TestContainerStatus(t *testing.T) {
return &client, nil return &client, nil
} }
d = Docker{ d = Docker{
Log: testutil.Logger{}, Log: testutil.Logger{},
newClient: newClientFunc, newClient: newClientFunc,
IncludeSourceTag: true,
} }
) )
@ -673,6 +674,7 @@ func TestContainerStatus(t *testing.T) {
"label2": "test_value_2", "label2": "test_value_2",
"server_version": "17.09.0-ce", "server_version": "17.09.0-ce",
"container_status": tt.expect.Status, "container_status": tt.expect.Status,
"source": "e2173b9478a6",
}) })
}) })
} }
@ -1017,3 +1019,37 @@ func TestContainerName(t *testing.T) {
}) })
} }
} }
func TestHostnameFromID(t *testing.T) {
tests := []struct {
name string
id string
expect string
}{
{
name: "Real ID",
id: "565e3a55f5843cfdd4aa5659a1a75e4e78d47f73c3c483f782fe4a26fc8caa07",
expect: "565e3a55f584",
},
{
name: "Short ID",
id: "shortid123",
expect: "shortid123",
},
{
name: "No ID",
id: "",
expect: "shortid123",
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
output := hostnameFromID(test.id)
if test.expect != output {
t.Logf("Container ID for hostname is wrong. Want: %s, Got: %s", output, test.expect)
}
})
}
}

View File

@ -43,6 +43,9 @@ The docker plugin uses the [Official Docker Client][] to gather logs from the
# docker_label_include = [] # docker_label_include = []
# docker_label_exclude = [] # docker_label_exclude = []
## Set the source tag for the metrics to the container ID hostname, eg first 12 chars
source_tag = false
## Optional TLS Config ## Optional TLS Config
# tls_ca = "/etc/telegraf/ca.pem" # tls_ca = "/etc/telegraf/ca.pem"
# tls_cert = "/etc/telegraf/cert.pem" # tls_cert = "/etc/telegraf/cert.pem"
@ -58,6 +61,17 @@ When using the `"ENV"` endpoint, the connection is configured using the
[env]: https://godoc.org/github.com/moby/moby/client#NewEnvClient [env]: https://godoc.org/github.com/moby/moby/client#NewEnvClient
### source tag
Selecting the containers can be tricky if you have many containers with the same name.
To alleviate this issue you can set the below value to `true`
```toml
source_tag = true
```
This will cause all data points to have the `source` tag be set to the first 12 characters of the container id. The first 12 characters is the common hostname for containers that have no explicit hostname set, as defined by docker.
### Metrics ### Metrics
- docker_log - docker_log
@ -66,6 +80,7 @@ When using the `"ENV"` endpoint, the connection is configured using the
- container_version - container_version
- container_name - container_name
- stream (stdout, stderr, or tty) - stream (stdout, stderr, or tty)
- source
- fields: - fields:
- container_id - container_id
- message - message

View File

@ -49,6 +49,9 @@ var sampleConfig = `
# docker_label_include = [] # docker_label_include = []
# docker_label_exclude = [] # docker_label_exclude = []
## Set the source tag for the metrics to the container ID hostname, eg first 12 chars
source_tag = false
## Optional TLS Config ## Optional TLS Config
# tls_ca = "/etc/telegraf/ca.pem" # tls_ca = "/etc/telegraf/ca.pem"
# tls_cert = "/etc/telegraf/cert.pem" # tls_cert = "/etc/telegraf/cert.pem"
@ -82,6 +85,7 @@ type DockerLogs struct {
ContainerExclude []string `toml:"container_name_exclude"` ContainerExclude []string `toml:"container_name_exclude"`
ContainerStateInclude []string `toml:"container_state_include"` ContainerStateInclude []string `toml:"container_state_include"`
ContainerStateExclude []string `toml:"container_state_exclude"` ContainerStateExclude []string `toml:"container_state_exclude"`
IncludeSourceTag bool `toml:"source_tag"`
tlsint.ClientConfig tlsint.ClientConfig
@ -258,6 +262,10 @@ func (d *DockerLogs) tailContainerLogs(
"container_version": imageVersion, "container_version": imageVersion,
} }
if d.IncludeSourceTag {
tags["source"] = hostnameFromID(container.ID)
}
// Add matching container labels as tags // Add matching container labels as tags
for k, label := range container.Labels { for k, label := range container.Labels {
if d.labelFilter.Match(k) { if d.labelFilter.Match(k) {
@ -435,3 +443,10 @@ func init() {
} }
}) })
} }
func hostnameFromID(id string) string {
if len(id) > 12 {
return id[0:12]
}
return id
}

View File

@ -98,6 +98,7 @@ func Test(t *testing.T) {
"container_image": "influxdata/telegraf", "container_image": "influxdata/telegraf",
"container_version": "1.11.0", "container_version": "1.11.0",
"stream": "tty", "stream": "tty",
"source": "deadbeef",
}, },
map[string]interface{}{ map[string]interface{}{
"container_id": "deadbeef", "container_id": "deadbeef",
@ -141,6 +142,7 @@ func Test(t *testing.T) {
"container_image": "influxdata/telegraf", "container_image": "influxdata/telegraf",
"container_version": "1.11.0", "container_version": "1.11.0",
"stream": "stdout", "stream": "stdout",
"source": "deadbeef",
}, },
map[string]interface{}{ map[string]interface{}{
"container_id": "deadbeef", "container_id": "deadbeef",
@ -155,9 +157,10 @@ func Test(t *testing.T) {
t.Run(tt.name, func(t *testing.T) { t.Run(tt.name, func(t *testing.T) {
var acc testutil.Accumulator var acc testutil.Accumulator
plugin := &DockerLogs{ plugin := &DockerLogs{
Timeout: internal.Duration{Duration: time.Second * 5}, Timeout: internal.Duration{Duration: time.Second * 5},
newClient: func(string, *tls.Config) (Client, error) { return tt.client, nil }, newClient: func(string, *tls.Config) (Client, error) { return tt.client, nil },
containerList: make(map[string]context.CancelFunc), containerList: make(map[string]context.CancelFunc),
IncludeSourceTag: true,
} }
err := plugin.Init() err := plugin.Init()