Add Suricata input plugin (#3145)

plugins/inputs/all/all.go

@@ -140,6 +140,7 @@ import (
 	_ "github.com/influxdata/telegraf/plugins/inputs/sqlserver"
 	_ "github.com/influxdata/telegraf/plugins/inputs/stackdriver"
 	_ "github.com/influxdata/telegraf/plugins/inputs/statsd"
+	_ "github.com/influxdata/telegraf/plugins/inputs/suricata"
 	_ "github.com/influxdata/telegraf/plugins/inputs/swap"
 	_ "github.com/influxdata/telegraf/plugins/inputs/syslog"
 	_ "github.com/influxdata/telegraf/plugins/inputs/sysstat"

plugins/inputs/suricata/README.md

@@ -0,0 +1,129 @@
+# Suricata plugin for Telegraf
+
+This plugin reports internal performance counters of the Suricata IDS/IPS
+engine, such as captured traffic volume, memory usage, uptime, flow counters,
+and much more. It provides a socket for the Suricata log output to write JSON
+stats output to, and processes the incoming data to fit Telegraf's format.
+
+### Configuration:
+
+```toml
+[[input.suricata]]
+  ## Data sink for Suricata stats log.
+  # This is expected to be a filename of a
+  # unix socket to be created for listening.
+  source = "/var/run/suricata-stats.sock"
+
+  # Delimiter for flattening field keys, e.g. subitem "alert" of "detect"
+  # becomes "detect_alert" when delimiter is "_".
+  delimiter = "_"
+```
+
+### Measurements & Fields:
+
+Fields in the 'suricata' measurement follow the JSON format used by Suricata's
+stats output.
+See http://suricata.readthedocs.io/en/latest/performance/statistics.html for
+more information.
+
+All fields are numeric.
+- suricata
+    - app_layer_flow_dcerpc_udp
+    - app_layer_flow_dns_tcp
+    - app_layer_flow_dns_udp
+    - app_layer_flow_enip_udp
+    - app_layer_flow_failed_tcp
+    - app_layer_flow_failed_udp
+    - app_layer_flow_http
+    - app_layer_flow_ssh
+    - app_layer_flow_tls
+    - app_layer_tx_dns_tcp
+    - app_layer_tx_dns_udp
+    - app_layer_tx_enip_udp
+    - app_layer_tx_http
+    - app_layer_tx_smtp
+    - capture_kernel_drops
+    - capture_kernel_packets
+    - decoder_avg_pkt_size
+    - decoder_bytes
+    - decoder_ethernet
+    - decoder_gre
+    - decoder_icmpv4
+    - decoder_icmpv4_ipv4_unknown_ver
+    - decoder_icmpv6
+    - decoder_invalid
+    - decoder_ipv4
+    - decoder_ipv6
+    - decoder_max_pkt_size
+    - decoder_pkts
+    - decoder_tcp
+    - decoder_tcp_hlen_too_small
+    - decoder_tcp_invalid_optlen
+    - decoder_teredo
+    - decoder_udp
+    - decoder_vlan
+    - detect_alert
+    - dns_memcap_global
+    - dns_memuse
+    - flow_memuse
+    - flow_mgr_closed_pruned
+    - flow_mgr_est_pruned
+    - flow_mgr_flows_checked
+    - flow_mgr_flows_notimeout
+    - flow_mgr_flows_removed
+    - flow_mgr_flows_timeout
+    - flow_mgr_flows_timeout_inuse
+    - flow_mgr_new_pruned
+    - flow_mgr_rows_checked
+    - flow_mgr_rows_empty
+    - flow_mgr_rows_maxlen
+    - flow_mgr_rows_skipped
+    - flow_spare
+    - flow_tcp_reuse
+    - http_memuse
+    - tcp_memuse
+    - tcp_pseudo
+    - tcp_reassembly_gap
+    - tcp_reassembly_memuse
+    - tcp_rst
+    - tcp_sessions
+    - tcp_syn
+    - tcp_synack
+    - ...
+
+### Tags:
+
+The `suricata` measurement has the following tags:
+
+- thread: `Global` for global statistics (if enabled), thread IDs (e.g. `W#03-enp0s31f6`) for thread-specific statistics
+
+## Suricata configuration
+
+Suricata needs to deliver the 'stats' event type to a given unix socket for
+this plugin to pick up. This can be done, for example, by creating an additional
+output in the Suricata configuration file:
+
+```yaml
+- eve-log:
+    enabled: yes
+    filetype: unix_stream
+    filename: /tmp/suricata-stats.sock
+    types:
+      - stats:
+         threads: yes
+```
+
+## Example Output:
+
+```text
+suricata,host=myhost,thread=FM#01 flow_mgr_rows_empty=0,flow_mgr_rows_checked=65536,flow_mgr_closed_pruned=0,flow_emerg_mode_over=0,flow_mgr_flows_timeout_inuse=0,flow_mgr_rows_skipped=65535,flow_mgr_bypassed_pruned=0,flow_mgr_flows_removed=0,flow_mgr_est_pruned=0,flow_mgr_flows_notimeout=1,flow_mgr_flows_checked=1,flow_mgr_rows_busy=0,flow_spare=10000,flow_mgr_rows_maxlen=1,flow_mgr_new_pruned=0,flow_emerg_mode_entered=0,flow_tcp_reuse=0,flow_mgr_flows_timeout=0 1568368562545197545
+suricata,host=myhost,thread=W#04-wlp4s0 decoder_ltnull_pkt_too_small=0,decoder_ipraw_invalid_ip_version=0,defrag_ipv4_reassembled=0,tcp_no_flow=0,app_layer_flow_tls=1,decoder_udp=25,defrag_ipv6_fragments=0,defrag_ipv4_fragments=0,decoder_tcp=59,decoder_vlan=0,decoder_pkts=84,decoder_vlan_qinq=0,decoder_avg_pkt_size=574,flow_memcap=0,defrag_max_frag_hits=0,tcp_ssn_memcap_drop=0,capture_kernel_packets=84,app_layer_flow_dcerpc_udp=0,app_layer_tx_dns_tcp=0,tcp_rst=0,decoder_icmpv4=0,app_layer_tx_tls=0,decoder_ipv4=84,decoder_erspan=0,decoder_ltnull_unsupported_type=0,decoder_invalid=0,app_layer_flow_ssh=0,capture_kernel_drops=0,app_layer_flow_ftp=0,app_layer_tx_http=0,tcp_pseudo_failed=0,defrag_ipv6_reassembled=0,defrag_ipv6_timeouts=0,tcp_pseudo=0,tcp_sessions=1,decoder_ethernet=84,decoder_raw=0,decoder_sctp=0,app_layer_flow_dns_udp=1,decoder_gre=0,app_layer_flow_http=0,app_layer_flow_imap=0,tcp_segment_memcap_drop=0,detect_alert=0,app_layer_flow_failed_tcp=0,decoder_teredo=0,decoder_mpls=0,decoder_ppp=0,decoder_max_pkt_size=1422,decoder_ipv6=0,tcp_reassembly_gap=0,app_layer_flow_dcerpc_tcp=0,decoder_ipv4_in_ipv6=0,tcp_stream_depth_reached=0,app_layer_flow_dns_tcp=0,app_layer_flow_smtp=0,tcp_syn=1,decoder_sll=0,tcp_invalid_checksum=0,app_layer_tx_dns_udp=1,decoder_bytes=48258,defrag_ipv4_timeouts=0,app_layer_flow_msn=0,decoder_pppoe=0,decoder_null=0,app_layer_flow_failed_udp=3,app_layer_tx_smtp=0,decoder_icmpv6=0,decoder_ipv6_in_ipv6=0,tcp_synack=1,app_layer_flow_smb=0,decoder_dce_pkt_too_small=0 1568368562545174807
+suricata,host=myhost,thread=W#01-wlp4s0 tcp_synack=0,app_layer_flow_imap=0,decoder_ipv4_in_ipv6=0,decoder_max_pkt_size=684,decoder_gre=0,defrag_ipv4_timeouts=0,tcp_invalid_checksum=0,decoder_ipv4=53,flow_memcap=0,app_layer_tx_http=0,app_layer_tx_smtp=0,decoder_null=0,tcp_no_flow=0,app_layer_tx_tls=0,app_layer_flow_ssh=0,app_layer_flow_smtp=0,decoder_pppoe=0,decoder_teredo=0,decoder_ipraw_invalid_ip_version=0,decoder_ltnull_pkt_too_small=0,tcp_rst=0,decoder_ppp=0,decoder_ipv6=29,app_layer_flow_dns_udp=3,decoder_vlan=0,app_layer_flow_dcerpc_tcp=0,tcp_syn=0,defrag_ipv4_fragments=0,defrag_ipv6_timeouts=0,decoder_raw=0,defrag_ipv6_reassembled=0,tcp_reassembly_gap=0,tcp_sessions=0,decoder_udp=44,tcp_segment_memcap_drop=0,app_layer_tx_dns_udp=3,app_layer_flow_tls=0,decoder_tcp=37,defrag_ipv4_reassembled=0,app_layer_flow_failed_udp=6,app_layer_flow_ftp=0,decoder_icmpv6=1,tcp_stream_depth_reached=0,capture_kernel_drops=0,decoder_sll=0,decoder_bytes=15883,decoder_ethernet=91,tcp_pseudo=0,app_layer_flow_http=0,decoder_sctp=0,decoder_pkts=91,decoder_avg_pkt_size=174,decoder_erspan=0,app_layer_flow_msn=0,app_layer_flow_smb=0,capture_kernel_packets=91,decoder_icmpv4=0,decoder_ipv6_in_ipv6=0,tcp_ssn_memcap_drop=0,decoder_vlan_qinq=0,decoder_ltnull_unsupported_type=0,decoder_invalid=0,defrag_max_frag_hits=0,tcp_pseudo_failed=0,detect_alert=0,app_layer_tx_dns_tcp=0,app_layer_flow_failed_tcp=0,app_layer_flow_dcerpc_udp=0,app_layer_flow_dns_tcp=0,defrag_ipv6_fragments=0,decoder_mpls=0,decoder_dce_pkt_too_small=0 1568368562545148438
+suricata,host=myhost flow_memuse=7094464,tcp_memuse=3276800,tcp_reassembly_memuse=12332832,dns_memuse=0,dns_memcap_state=0,dns_memcap_global=0,http_memuse=0,http_memcap=0 1568368562545144569
+suricata,host=myhost,thread=W#07-wlp4s0 app_layer_tx_http=0,app_layer_tx_dns_tcp=0,decoder_vlan=0,decoder_pppoe=0,decoder_sll=0,decoder_tcp=0,flow_memcap=0,app_layer_flow_msn=0,tcp_no_flow=0,tcp_rst=0,tcp_segment_memcap_drop=0,tcp_sessions=0,detect_alert=0,defrag_ipv6_reassembled=0,decoder_ipraw_invalid_ip_version=0,decoder_erspan=0,decoder_icmpv4=0,app_layer_tx_dns_udp=2,decoder_ltnull_pkt_too_small=0,decoder_bytes=1998,decoder_ipv6=1,defrag_ipv4_fragments=0,defrag_ipv6_fragments=0,app_layer_tx_smtp=0,decoder_ltnull_unsupported_type=0,decoder_max_pkt_size=342,app_layer_flow_ftp=0,decoder_ipv6_in_ipv6=0,defrag_ipv4_reassembled=0,defrag_ipv6_timeouts=0,app_layer_flow_dns_tcp=0,decoder_avg_pkt_size=181,defrag_ipv4_timeouts=0,tcp_stream_depth_reached=0,decoder_mpls=0,app_layer_flow_dns_udp=2,tcp_ssn_memcap_drop=0,app_layer_flow_dcerpc_tcp=0,app_layer_flow_failed_udp=2,app_layer_flow_smb=0,app_layer_flow_failed_tcp=0,decoder_invalid=0,decoder_null=0,decoder_gre=0,decoder_ethernet=11,app_layer_flow_ssh=0,defrag_max_frag_hits=0,capture_kernel_drops=0,tcp_pseudo_failed=0,app_layer_flow_smtp=0,decoder_udp=10,decoder_sctp=0,decoder_teredo=0,decoder_icmpv6=1,tcp_pseudo=0,tcp_synack=0,app_layer_tx_tls=0,app_layer_flow_imap=0,capture_kernel_packets=11,decoder_pkts=11,decoder_raw=0,decoder_ppp=0,tcp_syn=0,tcp_invalid_checksum=0,app_layer_flow_tls=0,decoder_ipv4_in_ipv6=0,app_layer_flow_http=0,decoder_dce_pkt_too_small=0,decoder_ipv4=10,decoder_vlan_qinq=0,tcp_reassembly_gap=0,app_layer_flow_dcerpc_udp=0 1568368562545110847
+suricata,host=myhost,thread=W#06-wlp4s0 app_layer_tx_smtp=0,decoder_ipv6_in_ipv6=0,decoder_dce_pkt_too_small=0,tcp_segment_memcap_drop=0,tcp_sessions=1,decoder_ppp=0,tcp_pseudo_failed=0,app_layer_tx_dns_tcp=0,decoder_invalid=0,defrag_ipv4_timeouts=0,app_layer_flow_smb=0,app_layer_flow_ssh=0,decoder_bytes=19407,decoder_null=0,app_layer_flow_tls=1,decoder_avg_pkt_size=473,decoder_pkts=41,decoder_pppoe=0,decoder_tcp=32,defrag_ipv4_reassembled=0,tcp_reassembly_gap=0,decoder_raw=0,flow_memcap=0,defrag_ipv6_timeouts=0,app_layer_flow_smtp=0,app_layer_tx_http=0,decoder_sll=0,decoder_udp=8,decoder_ltnull_pkt_too_small=0,decoder_ltnull_unsupported_type=0,decoder_ipv4_in_ipv6=0,decoder_vlan=0,decoder_max_pkt_size=1422,tcp_no_flow=0,app_layer_flow_failed_tcp=0,app_layer_flow_dns_tcp=0,app_layer_flow_ftp=0,decoder_icmpv4=0,defrag_max_frag_hits=0,tcp_rst=0,app_layer_flow_msn=0,app_layer_flow_failed_udp=2,app_layer_flow_dns_udp=0,app_layer_flow_dcerpc_udp=0,decoder_ipv4=39,decoder_ethernet=41,defrag_ipv6_reassembled=0,tcp_ssn_memcap_drop=0,app_layer_tx_tls=0,decoder_gre=0,decoder_vlan_qinq=0,tcp_pseudo=0,app_layer_flow_imap=0,app_layer_flow_dcerpc_tcp=0,defrag_ipv4_fragments=0,defrag_ipv6_fragments=0,tcp_synack=1,app_layer_flow_http=0,app_layer_tx_dns_udp=0,capture_kernel_packets=41,decoder_ipv6=2,tcp_invalid_checksum=0,tcp_stream_depth_reached=0,decoder_ipraw_invalid_ip_version=0,decoder_icmpv6=1,tcp_syn=1,detect_alert=0,capture_kernel_drops=0,decoder_teredo=0,decoder_erspan=0,decoder_sctp=0,decoder_mpls=0 1568368562545084670
+suricata,host=myhost,thread=W#02-wlp4s0 decoder_tcp=53,tcp_rst=3,tcp_reassembly_gap=0,defrag_ipv6_timeouts=0,tcp_ssn_memcap_drop=0,app_layer_flow_dcerpc_tcp=0,decoder_max_pkt_size=1422,decoder_ipv6_in_ipv6=0,tcp_no_flow=0,app_layer_flow_ftp=0,app_layer_flow_ssh=0,decoder_pkts=82,decoder_sctp=0,tcp_invalid_checksum=0,app_layer_flow_dns_tcp=0,decoder_ipraw_invalid_ip_version=0,decoder_bytes=26441,decoder_erspan=0,tcp_pseudo_failed=0,tcp_syn=1,app_layer_tx_http=0,app_layer_tx_smtp=0,decoder_teredo=0,decoder_ipv4=80,defrag_ipv4_fragments=0,tcp_stream_depth_reached=0,app_layer_flow_smb=0,capture_kernel_packets=82,decoder_null=0,decoder_ltnull_pkt_too_small=0,decoder_ppp=0,decoder_icmpv6=1,app_layer_flow_dns_udp=2,app_layer_flow_http=0,app_layer_tx_dns_udp=3,decoder_mpls=0,decoder_sll=0,defrag_ipv4_reassembled=0,tcp_segment_memcap_drop=0,app_layer_flow_imap=0,decoder_ltnull_unsupported_type=0,decoder_icmpv4=0,decoder_raw=0,defrag_ipv4_timeouts=0,app_layer_flow_failed_udp=8,decoder_gre=0,capture_kernel_drops=0,defrag_ipv6_reassembled=0,tcp_pseudo=0,app_layer_flow_tls=1,decoder_avg_pkt_size=322,decoder_dce_pkt_too_small=0,decoder_ethernet=82,defrag_ipv6_fragments=0,tcp_sessions=1,tcp_synack=1,app_layer_tx_dns_tcp=0,decoder_vlan=0,flow_memcap=0,decoder_vlan_qinq=0,decoder_udp=28,decoder_invalid=0,detect_alert=0,app_layer_flow_failed_tcp=0,app_layer_tx_tls=0,decoder_pppoe=0,decoder_ipv6=2,decoder_ipv4_in_ipv6=0,defrag_max_frag_hits=0,app_layer_flow_dcerpc_udp=0,app_layer_flow_smtp=0,app_layer_flow_msn=0 1568368562545061864
+suricata,host=myhost,thread=W#08-wlp4s0 decoder_dce_pkt_too_small=0,app_layer_tx_dns_tcp=0,decoder_pkts=58,decoder_ppp=0,decoder_raw=0,decoder_ipv4_in_ipv6=0,decoder_max_pkt_size=1392,tcp_invalid_checksum=0,tcp_syn=0,decoder_ipv4=51,decoder_ipv6_in_ipv6=0,decoder_tcp=0,decoder_ltnull_pkt_too_small=0,flow_memcap=0,decoder_udp=58,tcp_ssn_memcap_drop=0,tcp_pseudo=0,app_layer_flow_dcerpc_udp=0,app_layer_flow_dns_udp=5,app_layer_tx_http=0,capture_kernel_drops=0,decoder_vlan=0,tcp_segment_memcap_drop=0,app_layer_flow_ftp=0,app_layer_flow_imap=0,app_layer_flow_http=0,app_layer_flow_tls=0,decoder_icmpv4=0,decoder_sctp=0,defrag_ipv4_timeouts=0,tcp_reassembly_gap=0,detect_alert=0,decoder_ethernet=58,tcp_pseudo_failed=0,decoder_teredo=0,defrag_ipv4_reassembled=0,tcp_sessions=0,app_layer_flow_msn=0,decoder_ipraw_invalid_ip_version=0,tcp_no_flow=0,app_layer_flow_dns_tcp=0,decoder_null=0,defrag_ipv4_fragments=0,app_layer_flow_dcerpc_tcp=0,app_layer_flow_failed_udp=8,app_layer_tx_tls=0,decoder_bytes=15800,decoder_ipv6=7,tcp_stream_depth_reached=0,decoder_invalid=0,decoder_ltnull_unsupported_type=0,app_layer_tx_dns_udp=6,decoder_pppoe=0,decoder_avg_pkt_size=272,decoder_erspan=0,defrag_ipv6_timeouts=0,app_layer_flow_failed_tcp=0,decoder_gre=0,decoder_sll=0,defrag_max_frag_hits=0,app_layer_flow_ssh=0,capture_kernel_packets=58,decoder_mpls=0,decoder_vlan_qinq=0,tcp_rst=0,app_layer_flow_smb=0,app_layer_tx_smtp=0,decoder_icmpv6=0,defrag_ipv6_fragments=0,defrag_ipv6_reassembled=0,tcp_synack=0,app_layer_flow_smtp=0 1568368562545035575
+suricata,host=myhost,thread=W#05-wlp4s0 tcp_reassembly_gap=0,capture_kernel_drops=0,decoder_ltnull_unsupported_type=0,tcp_sessions=0,tcp_stream_depth_reached=0,tcp_pseudo_failed=0,app_layer_flow_failed_tcp=0,app_layer_tx_dns_tcp=0,decoder_null=0,decoder_dce_pkt_too_small=0,decoder_udp=7,tcp_rst=3,app_layer_flow_dns_tcp=0,decoder_invalid=0,defrag_ipv4_reassembled=0,tcp_synack=0,app_layer_flow_ftp=0,decoder_bytes=3117,decoder_pppoe=0,app_layer_flow_dcerpc_tcp=0,app_layer_flow_smb=0,decoder_ipv6_in_ipv6=0,decoder_ipraw_invalid_ip_version=0,app_layer_flow_imap=0,app_layer_tx_dns_udp=2,decoder_ppp=0,decoder_ipv4=21,decoder_tcp=14,flow_memcap=0,tcp_syn=0,tcp_invalid_checksum=0,decoder_teredo=0,decoder_ltnull_pkt_too_small=0,defrag_max_frag_hits=0,app_layer_tx_tls=0,decoder_pkts=24,decoder_sll=0,defrag_ipv6_fragments=0,app_layer_flow_dcerpc_udp=0,app_layer_flow_smtp=0,decoder_icmpv6=3,defrag_ipv6_timeouts=0,decoder_ipv6=3,decoder_raw=0,defrag_ipv6_reassembled=0,tcp_no_flow=0,detect_alert=0,app_layer_flow_tls=0,decoder_ethernet=24,decoder_vlan=0,decoder_icmpv4=0,decoder_ipv4_in_ipv6=0,app_layer_flow_failed_udp=1,decoder_mpls=0,decoder_max_pkt_size=653,decoder_sctp=0,defrag_ipv4_timeouts=0,tcp_ssn_memcap_drop=0,app_layer_flow_dns_udp=1,app_layer_tx_smtp=0,capture_kernel_packets=24,decoder_vlan_qinq=0,decoder_gre=0,app_layer_flow_ssh=0,app_layer_flow_msn=0,defrag_ipv4_fragments=0,app_layer_flow_http=0,tcp_segment_memcap_drop=0,tcp_pseudo=0,app_layer_tx_http=0,decoder_erspan=0,decoder_avg_pkt_size=129 1568368562545009684
+suricata,host=myhost,thread=W#03-wlp4s0 app_layer_flow_failed_tcp=0,decoder_teredo=0,decoder_ipv6_in_ipv6=0,tcp_pseudo_failed=0,tcp_stream_depth_reached=0,tcp_syn=0,decoder_gre=0,tcp_segment_memcap_drop=0,tcp_ssn_memcap_drop=0,app_layer_tx_smtp=0,decoder_raw=0,decoder_ltnull_pkt_too_small=0,tcp_sessions=0,tcp_reassembly_gap=0,app_layer_flow_ssh=0,app_layer_flow_imap=0,decoder_ipv4=463,decoder_ethernet=463,capture_kernel_packets=463,decoder_pppoe=0,defrag_ipv4_reassembled=0,app_layer_flow_tls=0,app_layer_flow_dcerpc_udp=0,app_layer_flow_dns_udp=0,decoder_vlan=0,decoder_ipraw_invalid_ip_version=0,decoder_mpls=0,tcp_no_flow=0,decoder_avg_pkt_size=445,decoder_udp=432,flow_memcap=0,app_layer_tx_dns_udp=0,app_layer_flow_msn=0,app_layer_flow_http=0,app_layer_flow_dcerpc_tcp=0,decoder_ipv6=0,decoder_ipv4_in_ipv6=0,defrag_ipv4_timeouts=0,defrag_ipv4_fragments=0,defrag_ipv6_timeouts=0,decoder_sctp=0,defrag_ipv6_fragments=0,app_layer_flow_dns_tcp=0,app_layer_tx_tls=0,defrag_max_frag_hits=0,decoder_bytes=206345,decoder_vlan_qinq=0,decoder_invalid=0,decoder_ppp=0,tcp_rst=0,detect_alert=0,capture_kernel_drops=0,app_layer_flow_failed_udp=4,decoder_null=0,decoder_icmpv4=0,decoder_icmpv6=0,decoder_ltnull_unsupported_type=0,defrag_ipv6_reassembled=0,tcp_invalid_checksum=0,tcp_synack=0,decoder_tcp=31,tcp_pseudo=0,app_layer_flow_smb=0,app_layer_flow_smtp=0,decoder_max_pkt_size=1463,decoder_dce_pkt_too_small=0,app_layer_tx_http=0,decoder_pkts=463,decoder_sll=0,app_layer_flow_ftp=0,app_layer_tx_dns_tcp=0,decoder_erspan=0 1568368562544966078
+```

plugins/inputs/suricata/suricata.go

@@ -0,0 +1,229 @@
+package suricata
+
+import (
+	"bufio"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net"
+	"strings"
+	"sync"
+
+	"github.com/influxdata/telegraf"
+	"github.com/influxdata/telegraf/plugins/inputs"
+)
+
+const (
+	// InBufSize is the input buffer size for JSON received via socket.
+	// Set to 10MB, as depending on the number of threads the output might be
+	// large.
+	InBufSize = 10 * 1024 * 1024
+)
+
+// Suricata is a Telegraf input plugin for Suricata runtime statistics.
+type Suricata struct {
+	Source    string `toml:"source"`
+	Delimiter string `toml:"delimiter"`
+
+	inputListener *net.UnixListener
+	cancel        context.CancelFunc
+
+	Log telegraf.Logger `toml:"-"`
+
+	wg sync.WaitGroup
+}
+
+// Description returns the plugin description.
+func (s *Suricata) Description() string {
+	return "Suricata stats plugin"
+}
+
+const sampleConfig = `
+  ## Data sink for Suricata stats log
+  # This is expected to be a filename of a
+  # unix socket to be created for listening.
+  source = "/var/run/suricata-stats.sock"
+
+  # Delimiter for flattening field keys, e.g. subitem "alert" of "detect"
+  # becomes "detect_alert" when delimiter is "_".
+  delimiter = "_"
+`
+
+// SampleConfig returns a sample TOML section to illustrate configuration
+// options.
+func (s *Suricata) SampleConfig() string {
+	return sampleConfig
+}
+
+// Start initiates background collection of JSON data from the socket
+// provided to Suricata.
+func (s *Suricata) Start(acc telegraf.Accumulator) error {
+	var err error
+	s.inputListener, err = net.ListenUnix("unix", &net.UnixAddr{
+		Name: s.Source,
+		Net:  "unix",
+	})
+	if err != nil {
+		return err
+	}
+	ctx, cancel := context.WithCancel(context.Background())
+	s.cancel = cancel
+	s.inputListener.SetUnlinkOnClose(true)
+	s.wg.Add(1)
+	go func() {
+		defer s.wg.Done()
+		go s.handleServerConnection(ctx, acc)
+	}()
+	return nil
+}
+
+// Stop causes the plugin to cease collecting JSON data from the socket provided
+// to Suricata.
+func (s *Suricata) Stop() {
+	s.inputListener.Close()
+	if s.cancel != nil {
+		s.cancel()
+	}
+	s.wg.Wait()
+}
+
+func (s *Suricata) readInput(ctx context.Context, acc telegraf.Accumulator, conn net.Conn) error {
+	reader := bufio.NewReaderSize(conn, InBufSize)
+	for {
+		select {
+		case <-ctx.Done():
+			return nil
+		default:
+			line, rerr := reader.ReadBytes('\n')
+			if rerr != nil {
+				return rerr
+			} else if len(line) > 0 {
+				s.parse(acc, line)
+			}
+		}
+	}
+}
+
+func (s *Suricata) handleServerConnection(ctx context.Context, acc telegraf.Accumulator) {
+	var err error
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		default:
+			var conn net.Conn
+			conn, err = s.inputListener.Accept()
+			if err != nil {
+				if !strings.HasSuffix(err.Error(), ": use of closed network connection") {
+					acc.AddError(err)
+				}
+				continue
+			}
+			err = s.readInput(ctx, acc, conn)
+			// we want to handle EOF as an opportunity to wait for a new
+			// connection -- this could, for example, happen when Suricata is
+			// restarted while Telegraf is running.
+			if err != io.EOF {
+				acc.AddError(err)
+				return
+			}
+		}
+	}
+}
+
+func flexFlatten(outmap map[string]interface{}, field string, v interface{}, delimiter string) error {
+	switch t := v.(type) {
+	case map[string]interface{}:
+		for k, v := range t {
+			var err error
+			if field == "" {
+				err = flexFlatten(outmap, k, v, delimiter)
+			} else {
+				err = flexFlatten(outmap, fmt.Sprintf("%s%s%s", field, delimiter, k), v, delimiter)
+			}
+			if err != nil {
+				return err
+			}
+		}
+	case float64:
+		outmap[field] = v.(float64)
+	default:
+		return fmt.Errorf("Unsupported type %T encountered", t)
+	}
+	return nil
+}
+
+func (s *Suricata) parse(acc telegraf.Accumulator, sjson []byte) {
+	// initial parsing
+	var result map[string]interface{}
+	err := json.Unmarshal([]byte(sjson), &result)
+	if err != nil {
+		acc.AddError(err)
+		return
+	}
+
+	// check for presence of relevant stats
+	if _, ok := result["stats"]; !ok {
+		s.Log.Debug("Input does not contain necessary 'stats' sub-object")
+		return
+	}
+
+	if _, ok := result["stats"].(map[string]interface{}); !ok {
+		s.Log.Debug("The 'stats' sub-object does not have required structure")
+		return
+	}
+
+	fields := make(map[string](map[string]interface{}))
+	totalmap := make(map[string]interface{})
+	for k, v := range result["stats"].(map[string]interface{}) {
+		if k == "threads" {
+			if v, ok := v.(map[string]interface{}); ok {
+				for k, t := range v {
+					outmap := make(map[string]interface{})
+					if threadStruct, ok := t.(map[string]interface{}); ok {
+						err = flexFlatten(outmap, "", threadStruct, s.Delimiter)
+						if err != nil {
+							s.Log.Debug(err)
+							// we skip this thread as something did not parse correctly
+							continue
+						}
+						fields[k] = outmap
+					}
+				}
+			} else {
+				s.Log.Debug("The 'threads' sub-object does not have required structure")
+			}
+		} else {
+			err = flexFlatten(totalmap, k, v, s.Delimiter)
+			if err != nil {
+				s.Log.Debug(err.Error())
+				// we skip this subitem as something did not parse correctly
+			}
+		}
+	}
+	fields["total"] = totalmap
+
+	for k := range fields {
+		if k == "Global" {
+			acc.AddFields("suricata", fields[k], nil)
+		} else {
+			acc.AddFields("suricata", fields[k], map[string]string{"thread": k})
+		}
+	}
+}
+
+// Gather measures and submits one full set of telemetry to Telegraf.
+// Not used here, submission is completely input-driven.
+func (s *Suricata) Gather(acc telegraf.Accumulator) error {
+	return nil
+}
+
+func init() {
+	inputs.Add("suricata", func() telegraf.Input {
+		return &Suricata{
+			Source:    "/var/run/suricata-stats.sock",
+			Delimiter: "_",
+		}
+	})
+}

plugins/inputs/suricata/suricata_test.go

@@ -0,0 +1,472 @@
+package suricata
+
+import (
+	"bytes"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"math/rand"
+	"net"
+	"os"
+	"path/filepath"
+	"regexp"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/influxdata/telegraf/plugins/inputs"
+	"github.com/influxdata/telegraf/testutil"
+	"github.com/stretchr/testify/assert"
+)
+
+var ex2 = `{"timestamp":"2017-03-06T07:43:39.000397+0000","event_type":"stats","stats":{"capture":{"kernel_packets":905344474,"kernel_drops":78355440,"kernel_packets_delta":2376742,"kernel_drops_delta":82049}}}`
+var ex3 = `{"timestamp":"2017-03-06T07:43:39.000397+0000","event_type":"stats","stats":{"threads": { "foo": { "capture":{"kernel_packets":905344474,"kernel_drops":78355440}}}}}`
+var ex4 = `{"timestamp":"2017-03-06T07:43:39.000397+0000","event_type":"stats","stats":{"threads": { "W1#en..bar1": { "capture":{"kernel_packets":905344474,"kernel_drops":78355440}}}}}`
+var brokenType1 = `{"timestamp":"2017-03-06T07:43:39.000397+0000","event_type":"stats","stats":{"threads": { "W1#en..bar1": { "capture":{"kernel_packets":905344474,"kernel_drops": true}}}}}`
+var brokenType2 = `{"timestamp":"2017-03-06T07:43:39.000397+0000","event_type":"stats","stats":{"threads": { "W1#en..bar1": { "capture":{"kernel_packets":905344474,"kernel_drops": ["foo"]}}}}}`
+var brokenType3 = `{"timestamp":"2017-03-06T07:43:39.000397+0000","event_type":"stats","stats":{"threads": { "W1#en..bar1": { "capture":{"kernel_packets":905344474,"kernel_drops":"none this time"}}}}}`
+var brokenType4 = `{"timestamp":"2017-03-06T07:43:39.000397+0000","event_type":"stats","stats":{"threads": { "W1#en..bar1": { "capture":{"kernel_packets":905344474,"kernel_drops":null}}}}}`
+var brokenType5 = `{"timestamp":"2017-03-06T07:43:39.000397+0000","event_type":"stats","stats":{"foo": null}}`
+var brokenStruct1 = `{"timestamp":"2017-03-06T07:43:39.000397+0000","event_type":"stats","stats":{"threads": ["foo"]}}`
+var brokenStruct2 = `{"timestamp":"2017-03-06T07:43:39.000397+0000","event_type":"stats"}`
+var brokenStruct3 = `{"timestamp":"2017-03-06T07:43:39.000397+0000","event_type":"stats","stats": "foobar"}`
+var brokenStruct4 = `{"timestamp":"2017-03-06T07:43:39.000397+0000","event_type":"stats","stats": null}`
+var singleDotRegexp = regexp.MustCompilePOSIX(`[^.]\.[^.]`)
+
+func TestSuricataLarge(t *testing.T) {
+	dir, err := ioutil.TempDir("", "test")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(dir)
+	tmpfn := filepath.Join(dir, fmt.Sprintf("t%d", rand.Int63()))
+
+	s := Suricata{
+		Source:    tmpfn,
+		Delimiter: ".",
+		Log: testutil.Logger{
+			Name: "inputs.suricata",
+		},
+	}
+	acc := testutil.Accumulator{}
+	acc.SetDebug(true)
+	assert.NoError(t, s.Start(&acc))
+
+	data, err := ioutil.ReadFile("testdata/test1.json")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	c, err := net.Dial("unix", tmpfn)
+	if err != nil {
+		t.Fatal(err)
+	}
+	c.Write([]byte(data))
+	c.Write([]byte("\n"))
+	c.Close()
+
+	acc.Wait(1)
+
+	s.Stop()
+}
+
+func TestSuricata(t *testing.T) {
+	dir, err := ioutil.TempDir("", "test")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(dir)
+	tmpfn := filepath.Join(dir, fmt.Sprintf("t%d", rand.Int63()))
+
+	s := Suricata{
+		Source:    tmpfn,
+		Delimiter: ".",
+		Log: testutil.Logger{
+			Name: "inputs.suricata",
+		},
+	}
+	acc := testutil.Accumulator{}
+	acc.SetDebug(true)
+	assert.NoError(t, s.Start(&acc))
+
+	c, err := net.Dial("unix", tmpfn)
+	if err != nil {
+		t.Fatalf("failed: %s", err.Error())
+	}
+	c.Write([]byte(ex2))
+	c.Write([]byte("\n"))
+	c.Close()
+
+	acc.Wait(1)
+
+	s.Stop()
+	s = Suricata{
+		Source:    tmpfn,
+		Delimiter: ".",
+		Log: testutil.Logger{
+			Name: "inputs.suricata",
+		},
+	}
+
+	acc.AssertContainsTaggedFields(t, "suricata",
+		map[string]interface{}{
+			"capture.kernel_packets":       float64(905344474),
+			"capture.kernel_drops":         float64(78355440),
+			"capture.kernel_packets_delta": float64(2376742),
+			"capture.kernel_drops_delta":   float64(82049),
+		},
+		map[string]string{"thread": "total"})
+
+	acc = testutil.Accumulator{}
+	acc.SetDebug(true)
+	assert.NoError(t, s.Start(&acc))
+
+	c, err = net.Dial("unix", tmpfn)
+	if err != nil {
+		log.Println(err)
+	}
+	c.Write([]byte(""))
+	c.Write([]byte("\n"))
+	c.Write([]byte("foobard}\n"))
+	c.Write([]byte(ex3))
+	c.Write([]byte("\n"))
+	c.Close()
+	acc.Wait(1)
+
+	s.Stop()
+
+	acc.AssertContainsTaggedFields(t, "suricata",
+		map[string]interface{}{
+			"capture.kernel_packets": float64(905344474),
+			"capture.kernel_drops":   float64(78355440),
+		},
+		map[string]string{"thread": "foo"})
+}
+
+func TestSuricataInvalid(t *testing.T) {
+	dir, err := ioutil.TempDir("", "test")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(dir)
+	tmpfn := filepath.Join(dir, fmt.Sprintf("t%d", rand.Int63()))
+
+	s := Suricata{
+		Source: tmpfn,
+		Log: testutil.Logger{
+			Name: "inputs.suricata",
+		},
+	}
+	acc := testutil.Accumulator{}
+	acc.SetDebug(true)
+
+	assert.NoError(t, s.Start(&acc))
+
+	c, err := net.Dial("unix", tmpfn)
+	if err != nil {
+		log.Println(err)
+	}
+	c.Write([]byte("sfjiowef"))
+	c.Write([]byte("\n"))
+	c.Close()
+
+	acc.WaitError(1)
+	s.Stop()
+}
+
+func splitAtSingleDot(in string) []string {
+	res := singleDotRegexp.FindAllStringIndex(in, -1)
+	if res == nil {
+		return []string{in}
+	}
+	ret := make([]string, 0)
+	startpos := 0
+	for _, v := range res {
+		ret = append(ret, in[startpos:v[0]+1])
+		startpos = v[1] - 1
+	}
+	return append(ret, in[startpos:])
+}
+
+func TestSuricataSplitDots(t *testing.T) {
+	dir, err := ioutil.TempDir("", "test")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(dir)
+	tmpfn := filepath.Join(dir, fmt.Sprintf("t%d", rand.Int63()))
+
+	out := splitAtSingleDot("foo")
+	if len(out) != 1 {
+		t.Fatalf("splitting 'foo' should yield one result")
+	}
+	if out[0] != "foo" {
+		t.Fatalf("splitting 'foo' should yield one result, 'foo'")
+	}
+
+	s := Suricata{
+		Source:    tmpfn,
+		Delimiter: ".",
+		Log: testutil.Logger{
+			Name: "inputs.suricata",
+		},
+	}
+	acc := testutil.Accumulator{}
+	acc.SetDebug(true)
+
+	assert.NoError(t, s.Start(&acc))
+
+	c, err := net.Dial("unix", tmpfn)
+	if err != nil {
+		log.Println(err)
+	}
+	c.Write([]byte(ex4))
+	c.Write([]byte("\n"))
+	c.Close()
+
+	acc.Wait(1)
+	acc.AssertContainsTaggedFields(t, "suricata",
+		map[string]interface{}{
+			"capture.kernel_packets": float64(905344474),
+			"capture.kernel_drops":   float64(78355440),
+		},
+		map[string]string{"thread": "W1#en..bar1"})
+
+	s.Stop()
+}
+
+func TestSuricataInvalidPath(t *testing.T) {
+	tmpfn := fmt.Sprintf("/t%d/X", rand.Int63())
+	s := Suricata{
+		Source: tmpfn,
+		Log: testutil.Logger{
+			Name: "inputs.suricata",
+		},
+	}
+
+	acc := testutil.Accumulator{}
+	acc.SetDebug(true)
+
+	assert.Error(t, s.Start(&acc))
+}
+
+func TestSuricataTooLongLine(t *testing.T) {
+	dir, err := ioutil.TempDir("", "test")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(dir)
+	tmpfn := filepath.Join(dir, fmt.Sprintf("t%d", rand.Int63()))
+
+	s := Suricata{
+		Source: tmpfn,
+		Log: testutil.Logger{
+			Name: "inputs.suricata",
+		},
+	}
+	acc := testutil.Accumulator{}
+	acc.SetDebug(true)
+
+	assert.NoError(t, s.Start(&acc))
+
+	c, err := net.Dial("unix", tmpfn)
+	if err != nil {
+		log.Println(err)
+	}
+	c.Write([]byte(strings.Repeat("X", 20000000)))
+	c.Write([]byte("\n"))
+	c.Close()
+
+	acc.WaitError(1)
+
+	s.Stop()
+}
+
+func TestSuricataEmptyJSON(t *testing.T) {
+	dir, err := ioutil.TempDir("", "test")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(dir)
+	tmpfn := filepath.Join(dir, fmt.Sprintf("t%d", rand.Int63()))
+
+	s := Suricata{
+		Source: tmpfn,
+		Log: testutil.Logger{
+			Name: "inputs.suricata",
+		},
+	}
+	acc := testutil.Accumulator{}
+	acc.SetDebug(true)
+
+	assert.NoError(t, s.Start(&acc))
+
+	c, err := net.Dial("unix", tmpfn)
+	if err != nil {
+		log.Println(err)
+	}
+	c.Write([]byte("\n"))
+	c.Close()
+
+	acc.WaitError(1)
+
+	s.Stop()
+}
+
+func TestSuricataInvalidInputs(t *testing.T) {
+	dir, err := ioutil.TempDir("", "test")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(dir)
+	defer func() {
+		log.SetOutput(os.Stderr)
+	}()
+	tmpfn := filepath.Join(dir, fmt.Sprintf("t%d", rand.Int63()))
+
+	for input, errmsg := range map[string]string{
+		brokenType1:   `Unsupported type bool encountered`,
+		brokenType2:   `Unsupported type []interface {} encountered`,
+		brokenType3:   `Unsupported type string encountered`,
+		brokenType4:   `Unsupported type <nil> encountered`,
+		brokenType5:   `Unsupported type <nil> encountered`,
+		brokenStruct1: `The 'threads' sub-object does not have required structure`,
+		brokenStruct2: `Input does not contain necessary 'stats' sub-object`,
+		brokenStruct3: `The 'stats' sub-object does not have required structure`,
+		brokenStruct4: `The 'stats' sub-object does not have required structure`,
+	} {
+		var logBuf buffer
+		logBuf.Reset()
+		log.SetOutput(&logBuf)
+
+		acc := testutil.Accumulator{}
+		acc.SetDebug(true)
+
+		s := Suricata{
+			Source:    tmpfn,
+			Delimiter: ".",
+			Log: testutil.Logger{
+				Name: "inputs.suricata",
+			},
+		}
+		assert.NoError(t, s.Start(&acc))
+
+		c, err := net.Dial("unix", tmpfn)
+		if err != nil {
+			t.Fatal(err)
+		}
+		c.Write([]byte(input))
+		c.Write([]byte("\n"))
+		c.Close()
+
+		for {
+			if bytes.Count(logBuf.Bytes(), []byte{'\n'}) > 0 {
+				break
+			}
+			time.Sleep(50 * time.Millisecond)
+		}
+
+		assert.Contains(t, logBuf.String(), errmsg)
+		s.Stop()
+	}
+}
+
+func TestSuricataDisconnectSocket(t *testing.T) {
+	dir, err := ioutil.TempDir("", "test")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(dir)
+	tmpfn := filepath.Join(dir, fmt.Sprintf("t%d", rand.Int63()))
+
+	s := Suricata{
+		Source: tmpfn,
+		Log: testutil.Logger{
+			Name: "inputs.suricata",
+		},
+	}
+	acc := testutil.Accumulator{}
+	acc.SetDebug(true)
+
+	assert.NoError(t, s.Start(&acc))
+
+	c, err := net.Dial("unix", tmpfn)
+	if err != nil {
+		log.Println(err)
+	}
+	c.Write([]byte(ex2))
+	c.Write([]byte("\n"))
+	c.Close()
+
+	c, err = net.Dial("unix", tmpfn)
+	if err != nil {
+		log.Println(err)
+	}
+	c.Write([]byte(ex3))
+	c.Write([]byte("\n"))
+	c.Close()
+
+	acc.Wait(2)
+
+	s.Stop()
+}
+
+func TestSuricataPluginDesc(t *testing.T) {
+	v, ok := inputs.Inputs["suricata"]
+	if !ok {
+		t.Fatal("suricata plugin not registered")
+	}
+	desc := v().Description()
+	if desc != "Suricata stats plugin" {
+		t.Fatal("invalid description ", desc)
+	}
+}
+
+func TestSuricataStartStop(t *testing.T) {
+	dir, err := ioutil.TempDir("", "test")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(dir)
+	tmpfn := filepath.Join(dir, fmt.Sprintf("t%d", rand.Int63()))
+
+	s := Suricata{
+		Source: tmpfn,
+		Log: testutil.Logger{
+			Name: "inputs.suricata",
+		},
+	}
+	acc := testutil.Accumulator{}
+	acc.SetDebug(true)
+	assert.NoError(t, s.Start(&acc))
+	s.Stop()
+}
+
+func TestSuricataGather(t *testing.T) {
+	dir, err := ioutil.TempDir("", "test")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(dir)
+	tmpfn := filepath.Join(dir, fmt.Sprintf("t%d", rand.Int63()))
+
+	s := Suricata{
+		Source: tmpfn,
+		Log: testutil.Logger{
+			Name: "inputs.suricata",
+		},
+	}
+	acc := testutil.Accumulator{}
+	acc.SetDebug(true)
+	assert.NoError(t, s.Gather(&acc))
+}
+
+func TestSuricataSampleConfig(t *testing.T) {
+	v, ok := inputs.Inputs["suricata"]
+	if !ok {
+		t.Fatal("suricata plugin not registered")
+	}
+	if v().SampleConfig() != sampleConfig {
+		t.Fatal("wrong sampleconfig")
+	}
+}

plugins/inputs/suricata/suricata_testutil.go

@@ -0,0 +1,38 @@
+package suricata
+
+import (
+	"bytes"
+	"sync"
+)
+
+// A thread-safe Buffer wrapper to enable concurrent access to log output.
+type buffer struct {
+	b bytes.Buffer
+	m sync.Mutex
+}
+
+func (b *buffer) Read(p []byte) (n int, err error) {
+	b.m.Lock()
+	defer b.m.Unlock()
+	return b.b.Read(p)
+}
+func (b *buffer) Write(p []byte) (n int, err error) {
+	b.m.Lock()
+	defer b.m.Unlock()
+	return b.b.Write(p)
+}
+func (b *buffer) String() string {
+	b.m.Lock()
+	defer b.m.Unlock()
+	return b.b.String()
+}
+func (b *buffer) Reset() {
+	b.m.Lock()
+	defer b.m.Unlock()
+	b.b.Reset()
+}
+func (b *buffer) Bytes() []byte {
+	b.m.Lock()
+	defer b.m.Unlock()
+	return b.b.Bytes()
+}