302 lines
9.4 KiB
Go
302 lines
9.4 KiB
Go
package docker
|
|
|
|
import (
|
|
"fmt"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
"github.com/influxdata/telegraf"
|
|
|
|
api "k8s.io/kubernetes/pkg/api"
|
|
kube "k8s.io/kubernetes/pkg/client/unversioned"
|
|
|
|
godocker "github.com/fsouza/go-dockerclient"
|
|
)
|
|
|
|
// CreateClient sets the client value in the Docker struct
|
|
func CreateClient(endpoint string) (*godocker.Client, error) {
|
|
var c *godocker.Client
|
|
var err error
|
|
if endpoint == "ENV" {
|
|
c, err = godocker.NewClientFromEnv()
|
|
} else if endpoint == "" {
|
|
c, err = godocker.NewClient("unix:///var/run/docker.sock")
|
|
} else {
|
|
c, err = godocker.NewClient(endpoint)
|
|
}
|
|
return c, err
|
|
}
|
|
|
|
func GatherContainerMetrics(client *godocker.Client, kubeClient *kube.Client, containerNames []string, acc telegraf.Accumulator) error {
|
|
opts := godocker.ListContainersOptions{}
|
|
containers, err := client.ListContainers(opts)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
var wg sync.WaitGroup
|
|
wg.Add(len(containers))
|
|
for _, container := range containers {
|
|
go func(c godocker.APIContainers) {
|
|
defer wg.Done()
|
|
err := gatherContainer(client, kubeClient, c, containerNames, acc)
|
|
if err != nil {
|
|
fmt.Println(err.Error())
|
|
}
|
|
}(container)
|
|
}
|
|
wg.Wait()
|
|
return nil
|
|
}
|
|
|
|
func gatherContainer(
|
|
client *godocker.Client,
|
|
kubeClient *kube.Client,
|
|
container godocker.APIContainers,
|
|
containerNames []string,
|
|
acc telegraf.Accumulator,
|
|
) error {
|
|
// Parse container name
|
|
cname := "unknown"
|
|
if len(container.Names) > 0 {
|
|
// Not sure what to do with other names, just take the first.
|
|
cname = strings.TrimPrefix(container.Names[0], "/")
|
|
}
|
|
|
|
tags := map[string]string{
|
|
"container_id": container.ID,
|
|
"container_name": cname,
|
|
"container_image": container.Image,
|
|
}
|
|
|
|
if kubeClient != nil {
|
|
tags = gatherKubernetesLabels(kubeClient, container.ID, tags)
|
|
}
|
|
|
|
if len(containerNames) > 0 {
|
|
if !sliceContains(cname, containerNames) {
|
|
return nil
|
|
}
|
|
}
|
|
|
|
statChan := make(chan *godocker.Stats)
|
|
done := make(chan bool)
|
|
statOpts := godocker.StatsOptions{
|
|
Stream: false,
|
|
ID: container.ID,
|
|
Stats: statChan,
|
|
Done: done,
|
|
Timeout: time.Duration(time.Second * 5),
|
|
}
|
|
|
|
go func() {
|
|
client.Stats(statOpts)
|
|
}()
|
|
|
|
stat := <-statChan
|
|
close(done)
|
|
|
|
// Add labels to tags
|
|
for k, v := range container.Labels {
|
|
tags[k] = v
|
|
}
|
|
|
|
gatherContainerStats(stat, acc, tags)
|
|
|
|
return nil
|
|
}
|
|
|
|
func gatherContainerStats(
|
|
stat *godocker.Stats,
|
|
acc telegraf.Accumulator,
|
|
tags map[string]string,
|
|
) {
|
|
now := stat.Read
|
|
|
|
memfields := map[string]interface{}{
|
|
"max_usage": stat.MemoryStats.MaxUsage,
|
|
"usage": stat.MemoryStats.Usage,
|
|
"fail_count": stat.MemoryStats.Failcnt,
|
|
"limit": stat.MemoryStats.Limit,
|
|
"total_pgmafault": stat.MemoryStats.Stats.TotalPgmafault,
|
|
"cache": stat.MemoryStats.Stats.Cache,
|
|
"mapped_file": stat.MemoryStats.Stats.MappedFile,
|
|
"total_inactive_file": stat.MemoryStats.Stats.TotalInactiveFile,
|
|
"pgpgout": stat.MemoryStats.Stats.Pgpgout,
|
|
"rss": stat.MemoryStats.Stats.Rss,
|
|
"total_mapped_file": stat.MemoryStats.Stats.TotalMappedFile,
|
|
"writeback": stat.MemoryStats.Stats.Writeback,
|
|
"unevictable": stat.MemoryStats.Stats.Unevictable,
|
|
"pgpgin": stat.MemoryStats.Stats.Pgpgin,
|
|
"total_unevictable": stat.MemoryStats.Stats.TotalUnevictable,
|
|
"pgmajfault": stat.MemoryStats.Stats.Pgmajfault,
|
|
"total_rss": stat.MemoryStats.Stats.TotalRss,
|
|
"total_rss_huge": stat.MemoryStats.Stats.TotalRssHuge,
|
|
"total_writeback": stat.MemoryStats.Stats.TotalWriteback,
|
|
"total_inactive_anon": stat.MemoryStats.Stats.TotalInactiveAnon,
|
|
"rss_huge": stat.MemoryStats.Stats.RssHuge,
|
|
"hierarchical_memory_limit": stat.MemoryStats.Stats.HierarchicalMemoryLimit,
|
|
"total_pgfault": stat.MemoryStats.Stats.TotalPgfault,
|
|
"total_active_file": stat.MemoryStats.Stats.TotalActiveFile,
|
|
"active_anon": stat.MemoryStats.Stats.ActiveAnon,
|
|
"total_active_anon": stat.MemoryStats.Stats.TotalActiveAnon,
|
|
"total_pgpgout": stat.MemoryStats.Stats.TotalPgpgout,
|
|
"total_cache": stat.MemoryStats.Stats.TotalCache,
|
|
"inactive_anon": stat.MemoryStats.Stats.InactiveAnon,
|
|
"active_file": stat.MemoryStats.Stats.ActiveFile,
|
|
"pgfault": stat.MemoryStats.Stats.Pgfault,
|
|
"inactive_file": stat.MemoryStats.Stats.InactiveFile,
|
|
"total_pgpgin": stat.MemoryStats.Stats.TotalPgpgin,
|
|
}
|
|
acc.AddFields("docker_mem", memfields, tags, now)
|
|
|
|
cpufields := map[string]interface{}{
|
|
"usage_total": stat.CPUStats.CPUUsage.TotalUsage,
|
|
"usage_in_usermode": stat.CPUStats.CPUUsage.UsageInUsermode,
|
|
"usage_in_kernelmode": stat.CPUStats.CPUUsage.UsageInKernelmode,
|
|
"usage_system": stat.CPUStats.SystemCPUUsage,
|
|
"throttling_periods": stat.CPUStats.ThrottlingData.Periods,
|
|
"throttling_throttled_periods": stat.CPUStats.ThrottlingData.ThrottledPeriods,
|
|
"throttling_throttled_time": stat.CPUStats.ThrottlingData.ThrottledTime,
|
|
}
|
|
cputags := copyTags(tags)
|
|
cputags["cpu"] = "cpu-total"
|
|
acc.AddFields("docker_cpu", cpufields, cputags, now)
|
|
|
|
for i, percpu := range stat.CPUStats.CPUUsage.PercpuUsage {
|
|
percputags := copyTags(tags)
|
|
percputags["cpu"] = fmt.Sprintf("cpu%d", i)
|
|
acc.AddFields("docker_cpu", map[string]interface{}{"usage_total": percpu}, percputags, now)
|
|
}
|
|
|
|
for network, netstats := range stat.Networks {
|
|
netfields := map[string]interface{}{
|
|
"rx_dropped": netstats.RxDropped,
|
|
"rx_bytes": netstats.RxBytes,
|
|
"rx_errors": netstats.RxErrors,
|
|
"tx_packets": netstats.TxPackets,
|
|
"tx_dropped": netstats.TxDropped,
|
|
"rx_packets": netstats.RxPackets,
|
|
"tx_errors": netstats.TxErrors,
|
|
"tx_bytes": netstats.TxBytes,
|
|
}
|
|
// Create a new network tag dictionary for the "network" tag
|
|
nettags := copyTags(tags)
|
|
nettags["network"] = network
|
|
acc.AddFields("docker_net", netfields, nettags, now)
|
|
}
|
|
|
|
gatherBlockIOMetrics(stat, acc, tags, now)
|
|
}
|
|
|
|
func gatherBlockIOMetrics(
|
|
stat *godocker.Stats,
|
|
acc telegraf.Accumulator,
|
|
tags map[string]string,
|
|
now time.Time,
|
|
) {
|
|
blkioStats := stat.BlkioStats
|
|
// Make a map of devices to their block io stats
|
|
deviceStatMap := make(map[string]map[string]interface{})
|
|
|
|
for _, metric := range blkioStats.IOServiceBytesRecursive {
|
|
device := fmt.Sprintf("%d:%d", metric.Major, metric.Minor)
|
|
_, ok := deviceStatMap[device]
|
|
if !ok {
|
|
deviceStatMap[device] = make(map[string]interface{})
|
|
}
|
|
|
|
field := fmt.Sprintf("io_service_bytes_recursive_%s", strings.ToLower(metric.Op))
|
|
deviceStatMap[device][field] = metric.Value
|
|
}
|
|
|
|
for _, metric := range blkioStats.IOServicedRecursive {
|
|
device := fmt.Sprintf("%d:%d", metric.Major, metric.Minor)
|
|
_, ok := deviceStatMap[device]
|
|
if !ok {
|
|
deviceStatMap[device] = make(map[string]interface{})
|
|
}
|
|
|
|
field := fmt.Sprintf("io_serviced_recursive_%s", strings.ToLower(metric.Op))
|
|
deviceStatMap[device][field] = metric.Value
|
|
}
|
|
|
|
for _, metric := range blkioStats.IOQueueRecursive {
|
|
device := fmt.Sprintf("%d:%d", metric.Major, metric.Minor)
|
|
field := fmt.Sprintf("io_queue_recursive_%s", strings.ToLower(metric.Op))
|
|
deviceStatMap[device][field] = metric.Value
|
|
}
|
|
|
|
for _, metric := range blkioStats.IOServiceTimeRecursive {
|
|
device := fmt.Sprintf("%d:%d", metric.Major, metric.Minor)
|
|
field := fmt.Sprintf("io_service_time_recursive_%s", strings.ToLower(metric.Op))
|
|
deviceStatMap[device][field] = metric.Value
|
|
}
|
|
|
|
for _, metric := range blkioStats.IOWaitTimeRecursive {
|
|
device := fmt.Sprintf("%d:%d", metric.Major, metric.Minor)
|
|
field := fmt.Sprintf("io_wait_time_%s", strings.ToLower(metric.Op))
|
|
deviceStatMap[device][field] = metric.Value
|
|
}
|
|
|
|
for _, metric := range blkioStats.IOMergedRecursive {
|
|
device := fmt.Sprintf("%d:%d", metric.Major, metric.Minor)
|
|
field := fmt.Sprintf("io_merged_recursive_%s", strings.ToLower(metric.Op))
|
|
deviceStatMap[device][field] = metric.Value
|
|
}
|
|
|
|
for _, metric := range blkioStats.IOTimeRecursive {
|
|
device := fmt.Sprintf("%d:%d", metric.Major, metric.Minor)
|
|
field := fmt.Sprintf("io_time_recursive_%s", strings.ToLower(metric.Op))
|
|
deviceStatMap[device][field] = metric.Value
|
|
}
|
|
|
|
for _, metric := range blkioStats.SectorsRecursive {
|
|
device := fmt.Sprintf("%d:%d", metric.Major, metric.Minor)
|
|
field := fmt.Sprintf("sectors_recursive_%s", strings.ToLower(metric.Op))
|
|
deviceStatMap[device][field] = metric.Value
|
|
}
|
|
|
|
for device, fields := range deviceStatMap {
|
|
iotags := copyTags(tags)
|
|
iotags["device"] = device
|
|
acc.AddFields("docker_blkio", fields, iotags, now)
|
|
}
|
|
}
|
|
|
|
func copyTags(in map[string]string) map[string]string {
|
|
out := make(map[string]string)
|
|
for k, v := range in {
|
|
out[k] = v
|
|
}
|
|
return out
|
|
}
|
|
|
|
func gatherKubernetesLabels(kubeClient *kube.Client, containerID string, tags map[string]string) map[string]string {
|
|
podClient := kubeClient.Pods(api.NamespaceAll)
|
|
pods, _ := podClient.List(api.ListOptions{})
|
|
|
|
for _, pod := range pods.Items {
|
|
for _, containerStatus := range pod.Status.ContainerStatuses {
|
|
podContainerID := strings.TrimPrefix(containerStatus.ContainerID, "docker://")
|
|
if podContainerID == containerID {
|
|
for k, v := range pod.ObjectMeta.Labels {
|
|
tags[k] = v
|
|
}
|
|
return tags
|
|
}
|
|
}
|
|
}
|
|
return tags
|
|
}
|
|
|
|
func sliceContains(in string, sl []string) bool {
|
|
for _, str := range sl {
|
|
if str == in {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|