Add Base64-encoding/decoding for Google Cloud PubSub plugins (#5543)

plugins/inputs/cloud_pubsub/README.md

@@ -75,6 +75,11 @@ and creates metrics using one of the supported [input data formats][].
   ## 1. Note this setting does not limit the number of messages that can be
   ## processed concurrently (use "max_outstanding_messages" instead).
   # max_receiver_go_routines = 0
+  
+  ## Optional. If true, Telegraf will attempt to base64 decode the 
+  ## PubSub message data before parsing. Many GCP services that
+  ## output JSON to Google PubSub base64-encode the JSON payload.
+  # base64_data = false
 ```
 
 ### Multiple Subscriptions and Topics

plugins/inputs/cloud_pubsub/pubsub.go

@@ -6,6 +6,7 @@ import (
 	"sync"
 
 	"cloud.google.com/go/pubsub"
+	"encoding/base64"
 	"github.com/influxdata/telegraf"
 	"github.com/influxdata/telegraf/internal"
 	"github.com/influxdata/telegraf/plugins/inputs"
@@ -40,6 +41,8 @@ type PubSub struct {
 	MaxUndeliveredMessages   int `toml:"max_undelivered_messages"`
 	RetryReceiveDelaySeconds int `toml:"retry_delay_seconds"`
 
+	Base64Data bool `toml:"base64_data"`
+
 	sub     subscription
 	stubSub func() subscription
 
@@ -169,7 +172,18 @@ func (ps *PubSub) onMessage(ctx context.Context, msg message) error {
 		return fmt.Errorf("message longer than max_message_len (%d > %d)", len(msg.Data()), ps.MaxMessageLen)
 	}
 
-	metrics, err := ps.parser.Parse(msg.Data())
+	var data []byte
+	if ps.Base64Data {
+		strData, err := base64.StdEncoding.DecodeString(string(msg.Data()))
+		if err != nil {
+			return fmt.Errorf("unable to base64 decode message: %v", err)
+		}
+		data = []byte(strData)
+	} else {
+		data = msg.Data()
+	}
+
+	metrics, err := ps.parser.Parse(data)
 	if err != nil {
 		msg.Ack()
 		return err
@@ -345,4 +359,8 @@ const sampleConfig = `
   ## 1. Note this setting does not limit the number of messages that can be
   ## processed concurrently (use "max_outstanding_messages" instead).
   # max_receiver_go_routines = 0
+
+  ## Optional. If true, Telegraf will attempt to base64 decode the 
+  ## PubSub message data before parsing
+  # base64_data = false
 `

plugins/inputs/cloud_pubsub/pubsub_test.go

@@ -1,6 +1,7 @@
 package cloud_pubsub
 
 import (
+	"encoding/base64"
 	"errors"
 	"github.com/influxdata/telegraf/plugins/parsers"
 	"github.com/influxdata/telegraf/testutil"
@@ -55,6 +56,50 @@ func TestRunParse(t *testing.T) {
 	validateTestInfluxMetric(t, metric)
 }
 
+// Test ingesting InfluxDB-format PubSub message
+func TestRunBase64(t *testing.T) {
+	subId := "sub-run-base64"
+
+	testParser, _ := parsers.NewInfluxParser()
+
+	sub := &stubSub{
+		id:       subId,
+		messages: make(chan *testMsg, 100),
+	}
+	sub.receiver = testMessagesReceive(sub)
+
+	ps := &PubSub{
+		parser:                 testParser,
+		stubSub:                func() subscription { return sub },
+		Project:                "projectIDontMatterForTests",
+		Subscription:           subId,
+		MaxUndeliveredMessages: defaultMaxUndeliveredMessages,
+		Base64Data:             true,
+	}
+
+	acc := &testutil.Accumulator{}
+	if err := ps.Start(acc); err != nil {
+		t.Fatalf("test PubSub failed to start: %s", err)
+	}
+	defer ps.Stop()
+
+	if ps.sub == nil {
+		t.Fatal("expected plugin subscription to be non-nil")
+	}
+
+	testTracker := &testTracker{}
+	msg := &testMsg{
+		value:   base64.StdEncoding.EncodeToString([]byte(msgInflux)),
+		tracker: testTracker,
+	}
+	sub.messages <- msg
+
+	acc.Wait(1)
+	assert.Equal(t, acc.NFields(), 1)
+	metric := acc.Metrics[0]
+	validateTestInfluxMetric(t, metric)
+}
+
 func TestRunInvalidMessages(t *testing.T) {
 	subId := "sub-invalid-messages"