Ignore errors serializing single metrics (#5943)

This commit is contained in:
Daniel Nelson 2019-06-03 17:34:48 -07:00 committed by GitHub
parent 3c83a53d51
commit 4197426a73
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 42 additions and 28 deletions

View File

@ -301,7 +301,8 @@ func (q *AMQP) serialize(metrics []telegraf.Metric) ([]byte, error) {
for _, metric := range metrics { for _, metric := range metrics {
octets, err := q.serializer.Serialize(metric) octets, err := q.serializer.Serialize(metric)
if err != nil { if err != nil {
return nil, err log.Printf("D! [outputs.amqp] Could not serialize metric: %v", err)
continue
} }
_, err = buf.Write(octets) _, err = buf.Write(octets)
if err != nil { if err != nil {

View File

@ -2,11 +2,12 @@ package cloud_pubsub
import ( import (
"context" "context"
"encoding/base64"
"fmt" "fmt"
"log"
"sync" "sync"
"cloud.google.com/go/pubsub" "cloud.google.com/go/pubsub"
"encoding/base64"
"github.com/influxdata/telegraf" "github.com/influxdata/telegraf"
"github.com/influxdata/telegraf/internal" "github.com/influxdata/telegraf/internal"
"github.com/influxdata/telegraf/plugins/outputs" "github.com/influxdata/telegraf/plugins/outputs"
@ -229,7 +230,8 @@ func (ps *PubSub) toMessages(metrics []telegraf.Metric) ([]*pubsub.Message, erro
for i, m := range metrics { for i, m := range metrics {
b, err := ps.serializer.Serialize(m) b, err := ps.serializer.Serialize(m)
if err != nil { if err != nil {
return nil, err log.Printf("D! [outputs.cloud_pubsub] Could not serialize metric: %v", err)
continue
} }
if ps.Base64Data { if ps.Base64Data {

View File

@ -3,6 +3,7 @@ package file
import ( import (
"fmt" "fmt"
"io" "io"
"log"
"os" "os"
"time" "time"
@ -101,7 +102,7 @@ func (f *File) Write(metrics []telegraf.Metric) error {
for _, metric := range metrics { for _, metric := range metrics {
b, err := f.serializer.Serialize(metric) b, err := f.serializer.Serialize(metric)
if err != nil { if err != nil {
return fmt.Errorf("failed to serialize message: %s", err) log.Printf("D! [outputs.file] Could not serialize metric: %v", err)
} }
_, err = f.writer.Write(b) _, err = f.writer.Write(b)

View File

@ -110,7 +110,8 @@ func (i *Instrumental) Write(metrics []telegraf.Metric) error {
buf, err := s.Serialize(m) buf, err := s.Serialize(m)
if err != nil { if err != nil {
log.Printf("E! Error serializing a metric to Instrumental: %s", err) log.Printf("D! [outputs.instrumental] Could not serialize metric: %v", err)
continue
} }
switch metricType { switch metricType {

View File

@ -6,13 +6,12 @@ import (
"log" "log"
"strings" "strings"
"github.com/Shopify/sarama"
"github.com/influxdata/telegraf" "github.com/influxdata/telegraf"
tlsint "github.com/influxdata/telegraf/internal/tls" tlsint "github.com/influxdata/telegraf/internal/tls"
"github.com/influxdata/telegraf/plugins/outputs" "github.com/influxdata/telegraf/plugins/outputs"
"github.com/influxdata/telegraf/plugins/serializers" "github.com/influxdata/telegraf/plugins/serializers"
uuid "github.com/satori/go.uuid" uuid "github.com/satori/go.uuid"
"github.com/Shopify/sarama"
) )
var ValidTopicSuffixMethods = []string{ var ValidTopicSuffixMethods = []string{
@ -294,7 +293,8 @@ func (k *Kafka) Write(metrics []telegraf.Metric) error {
for _, metric := range metrics { for _, metric := range metrics {
buf, err := k.serializer.Serialize(metric) buf, err := k.serializer.Serialize(metric)
if err != nil { if err != nil {
return err log.Printf("D! [outputs.kafka] Could not serialize metric: %v", err)
continue
} }
m := &sarama.ProducerMessage{ m := &sarama.ProducerMessage{

View File

@ -6,12 +6,11 @@ import (
"github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/kinesis" "github.com/aws/aws-sdk-go/service/kinesis"
"github.com/satori/go.uuid"
"github.com/influxdata/telegraf" "github.com/influxdata/telegraf"
internalaws "github.com/influxdata/telegraf/internal/config/aws" internalaws "github.com/influxdata/telegraf/internal/config/aws"
"github.com/influxdata/telegraf/plugins/outputs" "github.com/influxdata/telegraf/plugins/outputs"
"github.com/influxdata/telegraf/plugins/serializers" "github.com/influxdata/telegraf/plugins/serializers"
"github.com/satori/go.uuid"
) )
type ( type (
@ -221,7 +220,8 @@ func (k *KinesisOutput) Write(metrics []telegraf.Metric) error {
values, err := k.serializer.Serialize(metric) values, err := k.serializer.Serialize(metric)
if err != nil { if err != nil {
return err log.Printf("D! [outputs.kinesis] Could not serialize metric: %v", err)
continue
} }
partitionKey := k.getPartitionKey(metric) partitionKey := k.getPartitionKey(metric)

View File

@ -2,6 +2,7 @@ package mqtt
import ( import (
"fmt" "fmt"
"log"
"strings" "strings"
"sync" "sync"
"time" "time"
@ -150,9 +151,9 @@ func (m *MQTT) Write(metrics []telegraf.Metric) error {
metricsmap[topic] = append(metricsmap[topic], metric) metricsmap[topic] = append(metricsmap[topic], metric)
} else { } else {
buf, err := m.serializer.Serialize(metric) buf, err := m.serializer.Serialize(metric)
if err != nil { if err != nil {
return err log.Printf("D! [outputs.mqtt] Could not serialize metric: %v", err)
continue
} }
err = m.publish(topic, buf) err = m.publish(topic, buf)

View File

@ -2,13 +2,13 @@ package nats
import ( import (
"fmt" "fmt"
"log"
nats_client "github.com/nats-io/go-nats"
"github.com/influxdata/telegraf" "github.com/influxdata/telegraf"
"github.com/influxdata/telegraf/internal/tls" "github.com/influxdata/telegraf/internal/tls"
"github.com/influxdata/telegraf/plugins/outputs" "github.com/influxdata/telegraf/plugins/outputs"
"github.com/influxdata/telegraf/plugins/serializers" "github.com/influxdata/telegraf/plugins/serializers"
nats_client "github.com/nats-io/go-nats"
) )
type NATS struct { type NATS struct {
@ -108,7 +108,8 @@ func (n *NATS) Write(metrics []telegraf.Metric) error {
for _, metric := range metrics { for _, metric := range metrics {
buf, err := n.serializer.Serialize(metric) buf, err := n.serializer.Serialize(metric)
if err != nil { if err != nil {
return err log.Printf("D! [outputs.nats] Could not serialize metric: %v", err)
continue
} }
err = n.conn.Publish(n.Subject, buf) err = n.conn.Publish(n.Subject, buf)

View File

@ -2,12 +2,12 @@ package nsq
import ( import (
"fmt" "fmt"
"log"
"github.com/nsqio/go-nsq"
"github.com/influxdata/telegraf" "github.com/influxdata/telegraf"
"github.com/influxdata/telegraf/plugins/outputs" "github.com/influxdata/telegraf/plugins/outputs"
"github.com/influxdata/telegraf/plugins/serializers" "github.com/influxdata/telegraf/plugins/serializers"
"github.com/nsqio/go-nsq"
) )
type NSQ struct { type NSQ struct {
@ -68,7 +68,8 @@ func (n *NSQ) Write(metrics []telegraf.Metric) error {
for _, metric := range metrics { for _, metric := range metrics {
buf, err := n.serializer.Serialize(metric) buf, err := n.serializer.Serialize(metric)
if err != nil { if err != nil {
return err log.Printf("D! [outputs.nsq] Could not serialize metric: %v", err)
continue
} }
err = n.producer.Publish(n.Topic, buf) err = n.producer.Publish(n.Topic, buf)

View File

@ -1,13 +1,12 @@
package socket_writer package socket_writer
import ( import (
"crypto/tls"
"fmt" "fmt"
"log" "log"
"net" "net"
"strings" "strings"
"crypto/tls"
"github.com/influxdata/telegraf" "github.com/influxdata/telegraf"
"github.com/influxdata/telegraf/internal" "github.com/influxdata/telegraf/internal"
tlsint "github.com/influxdata/telegraf/internal/tls" tlsint "github.com/influxdata/telegraf/internal/tls"
@ -128,8 +127,8 @@ func (sw *SocketWriter) Write(metrics []telegraf.Metric) error {
for _, m := range metrics { for _, m := range metrics {
bs, err := sw.Serialize(m) bs, err := sw.Serialize(m)
if err != nil { if err != nil {
//TODO log & keep going with remaining metrics log.Printf("D! [outputs.socket_writer] Could not serialize metric: %v", err)
return err continue
} }
if _, err := sw.Conn.Write(bs); err != nil { if _, err := sw.Conn.Write(bs); err != nil {
//TODO log & keep going with remaining strings //TODO log & keep going with remaining strings

View File

@ -113,6 +113,9 @@ func (s *Serializer) SerializeBatch(metrics []telegraf.Metric) ([]byte, error) {
for _, m := range metrics { for _, m := range metrics {
_, err := s.Write(&s.buf, m) _, err := s.Write(&s.buf, m)
if err != nil { if err != nil {
if _, ok := err.(*MetricError); ok {
continue
}
return nil, err return nil, err
} }
} }

View File

@ -53,13 +53,14 @@ func (r *reader) Read(p []byte) (int, error) {
r.offset += 1 r.offset += 1
if err != nil { if err != nil {
r.buf.Reset() r.buf.Reset()
if err != nil { if _, ok := err.(*MetricError); ok {
continue
}
// Since we are serializing multiple metrics, don't fail the // Since we are serializing multiple metrics, don't fail the
// the entire batch just because of one unserializable metric. // the entire batch just because of one unserializable metric.
log.Printf("E! [serializers.influx] could not serialize metric: %v; discarding metric", err) log.Printf("E! [serializers.influx] could not serialize metric: %v; discarding metric", err)
continue continue
} }
}
break break
} }

View File

@ -30,6 +30,9 @@ type Serializer interface {
// Serialize takes a single telegraf metric and turns it into a byte buffer. // Serialize takes a single telegraf metric and turns it into a byte buffer.
// separate metrics should be separated by a newline, and there should be // separate metrics should be separated by a newline, and there should be
// a newline at the end of the buffer. // a newline at the end of the buffer.
//
// New plugins should use SerializeBatch instead to allow for non-line
// delimited metrics.
Serialize(metric telegraf.Metric) ([]byte, error) Serialize(metric telegraf.Metric) ([]byte, error)
// SerializeBatch takes an array of telegraf metric and serializes it into // SerializeBatch takes an array of telegraf metric and serializes it into
@ -41,7 +44,7 @@ type Serializer interface {
// Config is a struct that covers the data types needed for all serializer types, // Config is a struct that covers the data types needed for all serializer types,
// and can be used to instantiate _any_ of the serializers. // and can be used to instantiate _any_ of the serializers.
type Config struct { type Config struct {
// Dataformat can be one of: influx, graphite, or json // Dataformat can be one of the serializer types listed in NewSerializer.
DataFormat string DataFormat string
// Support tags in graphite protocol // Support tags in graphite protocol