From 9a637eda0549fa2de9efe571ee27f4c4394d92c8 Mon Sep 17 00:00:00 2001 From: Wojciech Kudla Date: Fri, 30 Nov 2018 22:42:55 +0000 Subject: [PATCH] Switch CPU from field to tag in interrupts input plugin (#4999) (#5024) --- plugins/inputs/interrupts/README.md | 27 +- plugins/inputs/interrupts/interrupts.go | 29 +- plugins/inputs/interrupts/interrupts_test.go | 314 +++++++++---------- 3 files changed, 185 insertions(+), 185 deletions(-) diff --git a/plugins/inputs/interrupts/README.md b/plugins/inputs/interrupts/README.md index eb1e3979d..188070f74 100644 --- a/plugins/inputs/interrupts/README.md +++ b/plugins/inputs/interrupts/README.md @@ -5,6 +5,9 @@ The interrupts plugin gathers metrics about IRQs from `/proc/interrupts` and `/p ### Configuration ``` [[inputs.interrupts]] + # To report cpus as tags instead of fields use cpu_as_tags + # cpu_as_tags = false + # ## To filter which IRQs to collect, make use of tagpass / tagdrop, i.e. # [inputs.interrupts.tagdrop] # irq = [ "NET_RX", "TASKLET" ] @@ -16,20 +19,32 @@ There are two measurements reported by this plugin. - `soft_interrupts` gathers metrics from the `/proc/softirqs` file ### Fields -- CPUx: the amount of interrupts for the IRQ handled by that CPU -- total: total amount of interrupts for all CPUs +For cpu_as_tags=false (default): +- CPUx: the amount of interrupts for the IRQ handled by the CPU +- Total: sum of interrupts for the IRS for all CPUs +For cpu_as_tags=true (): +- Count: the amount of interrupts for the IRQ handled by CPU described in CPU tag ### Tags - irq: the IRQ - type: the type of interrupt - device: the name of the device that is located at that IRQ +- cpu: the CPU (when cpus_as_tags=true) ### Example Output ``` ./telegraf --config ~/interrupts_config.conf --test +For cpus_as_tags=false (default): * Plugin: inputs.interrupts, Collection 1 -> interrupts,irq=0,type=IO-APIC,device=2-edge\ timer,host=hostname CPU0=23i,total=23i 1489346531000000000 -> interrupts,irq=1,host=hostname,type=IO-APIC,device=1-edge\ i8042 CPU0=9i,total=9i 1489346531000000000 -> interrupts,irq=30,type=PCI-MSI,device=65537-edge\ virtio1-input.0,host=hostname CPU0=1i,total=1i 1489346531000000000 -> soft_interrupts,irq=NET_RX,host=hostname CPU0=280879i,total=280879i 1489346531000000000 +> interrupts,irq=0,type=IO-APIC,device=2-edge\ timer,host=hostname,cpu=cpu0 count=23i 1489346531000000000 +> interrupts,irq=1,host=hostname,type=IO-APIC,device=1-edge\ i8042,cpu=cpu0 count=9i 1489346531000000000 +> interrupts,irq=30,type=PCI-MSI,device=65537-edge\ virtio1-input.0,host=hostname,cpu=cpu1 count=1i 1489346531000000000 +> soft_interrupts,irq=NET_RX,host=hostname,cpu=cpu0 count=280879i 1489346531000000000 + +For cpus_as_tags=true: +> interrupts,cpu=cpu6,host=hostname,irq=PIW,type=Posted-interrupt\ wakeup\ event count=0i 1543539773000000000 +> interrupts,cpu=cpu7,host=hostname,irq=PIW,type=Posted-interrupt\ wakeup\ event count=0i 1543539773000000000 +> soft_interrupts,cpu=cpu0,host=hostname,irq=HI count=246441i 1543539773000000000 +> soft_interrupts,cpu=cpu1,host=hostname,irq=HI count=159154i 1543539773000000000 + ``` diff --git a/plugins/inputs/interrupts/interrupts.go b/plugins/inputs/interrupts/interrupts.go index 9e8c8ea24..142dc34ec 100644 --- a/plugins/inputs/interrupts/interrupts.go +++ b/plugins/inputs/interrupts/interrupts.go @@ -12,7 +12,9 @@ import ( "github.com/influxdata/telegraf/plugins/inputs" ) -type Interrupts struct{} +type Interrupts struct { + CpuAsTags bool +} type IRQ struct { ID string @@ -27,6 +29,9 @@ func NewIRQ(id string) *IRQ { } const sampleConfig = ` + ## To report cpus as tags instead of fields use cpu_as_tags + # cpu_as_tags = false + # ## To filter which IRQs to collect, make use of tagpass / tagdrop, i.e. # [inputs.interrupts.tagdrop] # irq = [ "NET_RX", "TASKLET" ] @@ -92,7 +97,7 @@ func gatherTagsFields(irq IRQ) (map[string]string, map[string]interface{}) { tags := map[string]string{"irq": irq.ID, "type": irq.Type, "device": irq.Device} fields := map[string]interface{}{"total": irq.Total} for i := 0; i < len(irq.Cpus); i++ { - cpu := fmt.Sprintf("CPU%d", i) + cpu := fmt.Sprintf("cpu%d", i) fields[cpu] = irq.Cpus[i] } return tags, fields @@ -111,12 +116,26 @@ func (s *Interrupts) Gather(acc telegraf.Accumulator) error { acc.AddError(fmt.Errorf("Parsing %s: %s", file, err)) continue } - for _, irq := range irqs { - tags, fields := gatherTagsFields(irq) + reportMetrics(measurement, irqs, acc, s.CpuAsTags) + } + return nil +} + +func reportMetrics(measurement string, irqs []IRQ, acc telegraf.Accumulator, cpusAsTags bool) { + for _, irq := range irqs { + tags, fields := gatherTagsFields(irq) + if cpusAsTags { + for cpu, count := range irq.Cpus { + cpuTags := map[string]string{"cpu": fmt.Sprintf("cpu%d", cpu)} + for k, v := range tags { + cpuTags[k] = v + } + acc.AddFields(measurement, map[string]interface{}{"count": count}, cpuTags) + } + } else { acc.AddFields(measurement, fields, tags) } } - return nil } func init() { diff --git a/plugins/inputs/interrupts/interrupts_test.go b/plugins/inputs/interrupts/interrupts_test.go index 3990461b1..2579d926d 100644 --- a/plugins/inputs/interrupts/interrupts_test.go +++ b/plugins/inputs/interrupts/interrupts_test.go @@ -2,189 +2,155 @@ package interrupts import ( "bytes" + "fmt" "testing" - "github.com/stretchr/testify/assert" + "github.com/influxdata/telegraf/testutil" "github.com/stretchr/testify/require" ) -func TestParseInterrupts(t *testing.T) { - interruptStr := ` CPU0 CPU1 - 0: 134 0 IO-APIC-edge timer - 1: 7 3 IO-APIC-edge i8042 -NMI: 0 0 Non-maskable interrupts -LOC: 2338608687 2334309625 Local timer interrupts -MIS: 0 -NET_RX: 867028 225 -TASKLET: 205 0` - f := bytes.NewBufferString(interruptStr) - parsed := []IRQ{ - { - ID: "0", Type: "IO-APIC-edge", Device: "timer", - Cpus: []int64{int64(134), int64(0)}, Total: int64(134), - }, - { - ID: "1", Type: "IO-APIC-edge", Device: "i8042", - Cpus: []int64{int64(7), int64(3)}, Total: int64(10), - }, - { - ID: "NMI", Type: "Non-maskable interrupts", - Cpus: []int64{int64(0), int64(0)}, Total: int64(0), - }, - { - ID: "LOC", Type: "Local timer interrupts", - Cpus: []int64{int64(2338608687), int64(2334309625)}, - Total: int64(4672918312), - }, - { - ID: "MIS", Cpus: []int64{int64(0)}, Total: int64(0), - }, - { - ID: "NET_RX", Cpus: []int64{int64(867028), int64(225)}, - Total: int64(867253), - }, - { - ID: "TASKLET", Cpus: []int64{int64(205), int64(0)}, - Total: int64(205), - }, - } - got, err := parseInterrupts(f) - require.Equal(t, nil, err) - require.NotEqual(t, 0, len(got)) - require.Equal(t, len(got), len(parsed)) - for i := 0; i < len(parsed); i++ { - assert.Equal(t, parsed[i], got[i]) - for k := 0; k < len(parsed[i].Cpus); k++ { - assert.Equal(t, parsed[i].Cpus[k], got[i].Cpus[k]) - } +// ===================================================================================== +// Setup and helper functions +// ===================================================================================== + +func expectCpuAsTags(m *testutil.Accumulator, t *testing.T, measurement string, irq IRQ) { + for idx, value := range irq.Cpus { + m.AssertContainsTaggedFields(t, measurement, map[string]interface{}{"count": value}, map[string]string{"irq": irq.ID, "type": irq.Type, "device": irq.Device, "cpu": fmt.Sprintf("cpu%d", idx)}) } } -// Tests #4470 -func TestParseInterruptsBad(t *testing.T) { - interruptStr := ` CPU0 CPU1 CPU2 CPU3 - 16: 0 0 0 0 bcm2836-timer 0 Edge arch_timer - 17: 127224250 118424219 127224437 117885416 bcm2836-timer 1 Edge arch_timer - 21: 0 0 0 0 bcm2836-pmu 9 Edge arm-pmu - 23: 1549514 0 0 0 ARMCTRL-level 1 Edge 3f00b880.mailbox - 24: 2 0 0 0 ARMCTRL-level 2 Edge VCHIQ doorbell - 46: 0 0 0 0 ARMCTRL-level 48 Edge bcm2708_fb dma - 48: 0 0 0 0 ARMCTRL-level 50 Edge DMA IRQ - 50: 0 0 0 0 ARMCTRL-level 52 Edge DMA IRQ - 51: 208 0 0 0 ARMCTRL-level 53 Edge DMA IRQ - 54: 883002 0 0 0 ARMCTRL-level 56 Edge DMA IRQ - 59: 0 0 0 0 ARMCTRL-level 61 Edge bcm2835-auxirq - 62: 521451447 0 0 0 ARMCTRL-level 64 Edge dwc_otg, dwc_otg_pcd, dwc_otg_hcd:usb1 - 86: 857597 0 0 0 ARMCTRL-level 88 Edge mmc0 - 87: 4938 0 0 0 ARMCTRL-level 89 Edge uart-pl011 - 92: 5669 0 0 0 ARMCTRL-level 94 Edge mmc1 - FIQ: usb_fiq - IPI0: 0 0 0 0 CPU wakeup interrupts - IPI1: 0 0 0 0 Timer broadcast interrupts - IPI2: 23564958 23464876 23531165 23040826 Rescheduling interrupts - IPI3: 148438 639704 644266 588150 Function call interrupts - IPI4: 0 0 0 0 CPU stop interrupts - IPI5: 4348149 1843985 3819457 1822877 IRQ work interrupts - IPI6: 0 0 0 0 completion interrupts` - f := bytes.NewBufferString(interruptStr) - parsed := []IRQ{ - { - ID: "16", Type: "bcm2836-timer", Device: "0 Edge arch_timer", - Cpus: []int64{0, 0, 0, 0}, - }, - { - ID: "17", Type: "bcm2836-timer", Device: "1 Edge arch_timer", - Cpus: []int64{127224250, 118424219, 127224437, 117885416}, Total: 490758322, - }, - { - ID: "21", Type: "bcm2836-pmu", Device: "9 Edge arm-pmu", - Cpus: []int64{0, 0, 0, 0}, - }, - { - ID: "23", Type: "ARMCTRL-level", Device: "1 Edge 3f00b880.mailbox", - Cpus: []int64{1549514, 0, 0, 0}, Total: 1549514, - }, - { - ID: "24", Type: "ARMCTRL-level", Device: "2 Edge VCHIQ doorbell", - Cpus: []int64{2, 0, 0, 0}, Total: 2, - }, - { - ID: "46", Type: "ARMCTRL-level", Device: "48 Edge bcm2708_fb dma", - Cpus: []int64{0, 0, 0, 0}, - }, - { - ID: "48", Type: "ARMCTRL-level", Device: "50 Edge DMA IRQ", - Cpus: []int64{0, 0, 0, 0}, - }, - { - ID: "50", Type: "ARMCTRL-level", Device: "52 Edge DMA IRQ", - Cpus: []int64{0, 0, 0, 0}, - }, - { - ID: "51", Type: "ARMCTRL-level", Device: "53 Edge DMA IRQ", - Cpus: []int64{208, 0, 0, 0}, Total: 208, - }, - { - ID: "54", Type: "ARMCTRL-level", Device: "56 Edge DMA IRQ", - Cpus: []int64{883002, 0, 0, 0}, Total: 883002, - }, - { - ID: "59", Type: "ARMCTRL-level", Device: "61 Edge bcm2835-auxirq", - Cpus: []int64{0, 0, 0, 0}, - }, - { - ID: "62", Type: "ARMCTRL-level", Device: "64 Edge dwc_otg, dwc_otg_pcd, dwc_otg_hcd:usb1", - Cpus: []int64{521451447, 0, 0, 0}, Total: 521451447, - }, - { - ID: "86", Type: "ARMCTRL-level", Device: "88 Edge mmc0", - Cpus: []int64{857597, 0, 0, 0}, Total: 857597, - }, - { - ID: "87", Type: "ARMCTRL-level", Device: "89 Edge uart-pl011", - Cpus: []int64{4938, 0, 0, 0}, Total: 4938, - }, - { - ID: "92", Type: "ARMCTRL-level", Device: "94 Edge mmc1", - Cpus: []int64{5669, 0, 0, 0}, Total: 5669, - }, - { - ID: "IPI0", Type: "CPU wakeup interrupts", - Cpus: []int64{0, 0, 0, 0}, - }, - { - ID: "IPI1", Type: "Timer broadcast interrupts", - Cpus: []int64{0, 0, 0, 0}, - }, - { - ID: "IPI2", Type: "Rescheduling interrupts", - Cpus: []int64{23564958, 23464876, 23531165, 23040826}, Total: 93601825, - }, - { - ID: "IPI3", Type: "Function call interrupts", - Cpus: []int64{148438, 639704, 644266, 588150}, Total: 2020558, - }, - { - ID: "IPI4", Type: "CPU stop interrupts", - Cpus: []int64{0, 0, 0, 0}, - }, - { - ID: "IPI5", Type: "IRQ work interrupts", - Cpus: []int64{4348149, 1843985, 3819457, 1822877}, Total: 11834468, - }, - { - ID: "IPI6", Type: "completion interrupts", - Cpus: []int64{0, 0, 0, 0}, - }, +func expectCpuAsFields(m *testutil.Accumulator, t *testing.T, measurement string, irq IRQ) { + fields := map[string]interface{}{} + total := int64(0) + for idx, count := range irq.Cpus { + fields[fmt.Sprintf("cpu%d", idx)] = count + total += count } - got, err := parseInterrupts(f) + fields["total"] = total + + m.AssertContainsTaggedFields(t, measurement, fields, map[string]string{"irq": irq.ID, "type": irq.Type, "device": irq.Device}) +} + +func setup(t *testing.T, irqString string, cpuAsTags bool) (*testutil.Accumulator, []IRQ) { + f := bytes.NewBufferString(irqString) + irqs, err := parseInterrupts(f) require.Equal(t, nil, err) - require.NotEqual(t, 0, len(got)) - require.Equal(t, len(got), len(parsed)) - for i := 0; i < len(parsed); i++ { - assert.Equal(t, parsed[i], got[i]) - for k := 0; k < len(parsed[i].Cpus); k++ { - assert.Equal(t, parsed[i].Cpus[k], got[i].Cpus[k]) - } + require.NotEqual(t, 0, len(irqs)) + + acc := new(testutil.Accumulator) + reportMetrics("soft_interrupts", irqs, acc, cpuAsTags) + + return acc, irqs +} + +// ===================================================================================== +// Soft interrupts +// ===================================================================================== + +const softIrqsString = ` CPU0 CPU1 + 0: 134 0 IO-APIC-edge timer + 1: 7 3 IO-APIC-edge i8042 + NMI: 0 0 Non-maskable interrupts + LOC: 2338608687 2334309625 Local timer interrupts + MIS: 0 + NET_RX: 867028 225 + TASKLET: 205 0` + +var softIrqsExpectedArgs = []IRQ{ + {ID: "0", Type: "IO-APIC-edge", Device: "timer", Cpus: []int64{134, 0}}, + {ID: "1", Type: "IO-APIC-edge", Device: "i8042", Cpus: []int64{7, 3}}, + {ID: "NMI", Type: "Non-maskable interrupts", Cpus: []int64{0, 0}}, + {ID: "MIS", Cpus: []int64{0}}, + {ID: "NET_RX", Cpus: []int64{867028, 225}}, + {ID: "TASKLET", Cpus: []int64{205, 0}}, +} + +func TestCpuAsTagsSoftIrqs(t *testing.T) { + acc, irqs := setup(t, softIrqsString, true) + reportMetrics("soft_interrupts", irqs, acc, true) + + for _, irq := range softIrqsExpectedArgs { + expectCpuAsTags(acc, t, "soft_interrupts", irq) + } +} + +func TestCpuAsFieldsSoftIrqs(t *testing.T) { + acc, irqs := setup(t, softIrqsString, false) + reportMetrics("soft_interrupts", irqs, acc, false) + + for _, irq := range softIrqsExpectedArgs { + expectCpuAsFields(acc, t, "soft_interrupts", irq) + } +} + +// ===================================================================================== +// HW interrupts, tests #4470 +// ===================================================================================== + +const hwIrqsString = ` CPU0 CPU1 CPU2 CPU3 + 16: 0 0 0 0 bcm2836-timer 0 Edge arch_timer + 17: 127224250 118424219 127224437 117885416 bcm2836-timer 1 Edge arch_timer + 21: 0 0 0 0 bcm2836-pmu 9 Edge arm-pmu + 23: 1549514 0 0 0 ARMCTRL-level 1 Edge 3f00b880.mailbox + 24: 2 0 0 0 ARMCTRL-level 2 Edge VCHIQ doorbell + 46: 0 0 0 0 ARMCTRL-level 48 Edge bcm2708_fb dma + 48: 0 0 0 0 ARMCTRL-level 50 Edge DMA IRQ + 50: 0 0 0 0 ARMCTRL-level 52 Edge DMA IRQ + 51: 208 0 0 0 ARMCTRL-level 53 Edge DMA IRQ + 54: 883002 0 0 0 ARMCTRL-level 56 Edge DMA IRQ + 59: 0 0 0 0 ARMCTRL-level 61 Edge bcm2835-auxirq + 62: 521451447 0 0 0 ARMCTRL-level 64 Edge dwc_otg, dwc_otg_pcd, dwc_otg_hcd:usb1 + 86: 857597 0 0 0 ARMCTRL-level 88 Edge mmc0 + 87: 4938 0 0 0 ARMCTRL-level 89 Edge uart-pl011 + 92: 5669 0 0 0 ARMCTRL-level 94 Edge mmc1 + FIQ: usb_fiq + IPI0: 0 0 0 0 CPU wakeup interrupts + IPI1: 0 0 0 0 Timer broadcast interrupts + IPI2: 23564958 23464876 23531165 23040826 Rescheduling interrupts + IPI3: 148438 639704 644266 588150 Function call interrupts + IPI4: 0 0 0 0 CPU stop interrupts + IPI5: 4348149 1843985 3819457 1822877 IRQ work interrupts + IPI6: 0 0 0 0 completion interrupts` + +var hwIrqsExpectedArgs = []IRQ{ + {ID: "16", Type: "bcm2836-timer", Device: "0 Edge arch_timer", Cpus: []int64{0, 0, 0, 0}}, + {ID: "17", Type: "bcm2836-timer", Device: "1 Edge arch_timer", Cpus: []int64{127224250, 118424219, 127224437, 117885416}}, + {ID: "21", Type: "bcm2836-pmu", Device: "9 Edge arm-pmu", Cpus: []int64{0, 0, 0, 0}}, + {ID: "23", Type: "ARMCTRL-level", Device: "1 Edge 3f00b880.mailbox", Cpus: []int64{1549514, 0, 0, 0}}, + {ID: "24", Type: "ARMCTRL-level", Device: "2 Edge VCHIQ doorbell", Cpus: []int64{2, 0, 0, 0}}, + {ID: "46", Type: "ARMCTRL-level", Device: "48 Edge bcm2708_fb dma", Cpus: []int64{0, 0, 0, 0}}, + {ID: "48", Type: "ARMCTRL-level", Device: "50 Edge DMA IRQ", Cpus: []int64{0, 0, 0, 0}}, + {ID: "50", Type: "ARMCTRL-level", Device: "52 Edge DMA IRQ", Cpus: []int64{0, 0, 0, 0}}, + {ID: "51", Type: "ARMCTRL-level", Device: "53 Edge DMA IRQ", Cpus: []int64{208, 0, 0, 0}}, + {ID: "54", Type: "ARMCTRL-level", Device: "56 Edge DMA IRQ", Cpus: []int64{883002, 0, 0, 0}}, + {ID: "59", Type: "ARMCTRL-level", Device: "61 Edge bcm2835-auxirq", Cpus: []int64{0, 0, 0, 0}}, + {ID: "62", Type: "ARMCTRL-level", Device: "64 Edge dwc_otg, dwc_otg_pcd, dwc_otg_hcd:usb1", Cpus: []int64{521451447, 0, 0, 0}}, + {ID: "86", Type: "ARMCTRL-level", Device: "88 Edge mmc0", Cpus: []int64{857597, 0, 0, 0}}, + {ID: "87", Type: "ARMCTRL-level", Device: "89 Edge uart-pl011", Cpus: []int64{4938, 0, 0, 0}}, + {ID: "92", Type: "ARMCTRL-level", Device: "94 Edge mmc1", Cpus: []int64{5669, 0, 0, 0}}, + {ID: "IPI0", Type: "CPU wakeup interrupts", Cpus: []int64{0, 0, 0, 0}}, + {ID: "IPI1", Type: "Timer broadcast interrupts", Cpus: []int64{0, 0, 0, 0}}, + {ID: "IPI2", Type: "Rescheduling interrupts", Cpus: []int64{23564958, 23464876, 23531165, 23040826}}, + {ID: "IPI3", Type: "Function call interrupts", Cpus: []int64{148438, 639704, 644266, 588150}}, + {ID: "IPI4", Type: "CPU stop interrupts", Cpus: []int64{0, 0, 0, 0}}, + {ID: "IPI5", Type: "IRQ work interrupts", Cpus: []int64{4348149, 1843985, 3819457, 1822877}}, + {ID: "IPI6", Type: "completion interrupts", Cpus: []int64{0, 0, 0, 0}}, +} + +func TestCpuAsTagsHwIrqs(t *testing.T) { + acc, irqs := setup(t, hwIrqsString, true) + reportMetrics("interrupts", irqs, acc, true) + + for _, irq := range hwIrqsExpectedArgs { + expectCpuAsTags(acc, t, "interrupts", irq) + } +} + +func TestCpuAsFieldsHwIrqs(t *testing.T) { + acc, irqs := setup(t, hwIrqsString, false) + reportMetrics("interrupts", irqs, acc, false) + + for _, irq := range hwIrqsExpectedArgs { + expectCpuAsFields(acc, t, "interrupts", irq) } }