Add mutual TLS support to prometheus_client output plugin

Signed-off-by: Robert Sullivan <rsullivan@pivotal.io>

Gopkg.lock

@@ -821,6 +821,27 @@
   revision = "eee57a3ac4174c55924125bb15eeeda8cffb6e6f"
   version = "v1.0.7"
 
+[[projects]]
+  digest = "1:c8f0c8c28c9c1c51db72d0e7f04797cfe5d0d50528274099b6b2d6c314db7f97"
+  name = "github.com/onsi/gomega"
+  packages = [
+    ".",
+    "format",
+    "internal/assertion",
+    "internal/asyncassertion",
+    "internal/oraclematcher",
+    "internal/testingtsupport",
+    "matchers",
+    "matchers/support/goraph/bipartitegraph",
+    "matchers/support/goraph/edge",
+    "matchers/support/goraph/node",
+    "matchers/support/goraph/util",
+    "types",
+  ]
+  pruneopts = ""
+  revision = "65fb64232476ad9046e57c26cd0bff3d3a8dc6cd"
+  version = "v1.4.3"
+
 [[projects]]
   digest = "1:5d9b668b0b4581a978f07e7d2e3314af18eb27b3fb5d19b70185b7c575723d11"
   name = "github.com/opencontainers/go-digest"
@@ -1541,6 +1562,7 @@
     "github.com/go-sql-driver/mysql",
     "github.com/gobwas/glob",
     "github.com/golang/protobuf/proto",
+    "github.com/golang/protobuf/ptypes/duration",
     "github.com/golang/protobuf/ptypes/empty",
     "github.com/golang/protobuf/ptypes/timestamp",
     "github.com/google/go-cmp/cmp",
@@ -1567,6 +1589,7 @@
     "github.com/nats-io/gnatsd/server",
     "github.com/nats-io/go-nats",
     "github.com/nsqio/go-nsq",
+    "github.com/onsi/gomega",
     "github.com/openzipkin/zipkin-go-opentracing",
     "github.com/openzipkin/zipkin-go-opentracing/thrift/gen-go/zipkincore",
     "github.com/prometheus/client_golang/prometheus",
@@ -1612,8 +1635,10 @@
     "golang.org/x/sys/windows",
     "golang.org/x/sys/windows/svc",
     "golang.org/x/sys/windows/svc/mgr",
+    "google.golang.org/api/iterator",
     "google.golang.org/api/option",
     "google.golang.org/api/support/bundler",
+    "google.golang.org/genproto/googleapis/api/distribution",
     "google.golang.org/genproto/googleapis/api/metric",
     "google.golang.org/genproto/googleapis/api/monitoredres",
     "google.golang.org/genproto/googleapis/monitoring/v3",

plugins/outputs/prometheus_client/.gitignore

@@ -0,0 +1,2 @@
+vendor
+assets

plugins/outputs/prometheus_client/README.md

@@ -36,6 +36,12 @@ This plugin starts a [Prometheus](https://prometheus.io/) Client, it exposes all
   # tls_cert = "/etc/ssl/telegraf.crt"
   # tls_key = "/etc/ssl/telegraf.key"
   
+  ## If set, enable TLS client authentication with the given CA.
+  # tls_ca = "/etc/ssl/telegraf_ca.crt"
+
+  ## Boolean value indicating whether or not to skip SSL verification
+  # insecure_skip_verify = false
+
   ## Export metric collection time.
   # export_timestamp = false
 ```

plugins/outputs/prometheus_client/prometheus_client.go

@@ -3,7 +3,10 @@ package prometheus_client
 import (
 	"context"
 	"crypto/subtle"
+	cryptotls "crypto/tls"
+	"crypto/x509"
 	"fmt"
+	"io/ioutil"
 	"log"
 	"net"
 	"net/http"
@@ -16,6 +19,7 @@ import (
 
 	"github.com/influxdata/telegraf"
 	"github.com/influxdata/telegraf/internal"
+	"github.com/influxdata/telegraf/internal/tls"
 	"github.com/influxdata/telegraf/plugins/outputs"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
@@ -56,8 +60,6 @@ type MetricFamily struct {
 
 type PrometheusClient struct {
 	Listen             string
-	TLSCert            string            `toml:"tls_cert"`
-	TLSKey             string            `toml:"tls_key"`
 	BasicUsername      string            `toml:"basic_username"`
 	BasicPassword      string            `toml:"basic_password"`
 	IPRange            []string          `toml:"ip_range"`
@@ -67,6 +69,7 @@ type PrometheusClient struct {
 	StringAsLabel      bool              `toml:"string_as_label"`
 	ExportTimestamp    bool              `toml:"export_timestamp"`
 
+	tls.ClientConfig
 	server *http.Server
 
 	sync.Mutex
@@ -106,6 +109,12 @@ var sampleConfig = `
   # tls_cert = "/etc/ssl/telegraf.crt"
   # tls_key = "/etc/ssl/telegraf.key"
   
+  ## If set, enable TLS client authentication with the given CA.
+  # tls_ca = "/etc/ssl/telegraf_ca.crt"
+
+  ## Boolean value indicating whether or not to skip SSL verification
+  # insecure_skip_verify = false
+
   ## Export metric collection time.
   # export_timestamp = false
 `
@@ -184,9 +193,18 @@ func (p *PrometheusClient) Connect() error {
 	mux.Handle(p.Path, p.auth(promhttp.HandlerFor(
 		registry, promhttp.HandlerOpts{ErrorHandling: promhttp.ContinueOnError})))
 
+	if p.TLSCA != "" {
+		log.Printf("Starting Prometheus Output Plugin Server with Mutual TLS enabled.\n")
 		p.server = &http.Server{
 			Addr:      p.Listen,
 			Handler:   mux,
+			TLSConfig: p.buildMutualTLSConfig(),
+		}
+	} else {
+		p.server = &http.Server{
+			Addr:    p.Listen,
+			Handler: mux,
+		}
 	}
 
 	go func() {
@@ -205,6 +223,34 @@ func (p *PrometheusClient) Connect() error {
 	return nil
 }
 
+func (p *PrometheusClient) buildMutualTLSConfig() *cryptotls.Config {
+	certPool := x509.NewCertPool()
+	caCert, err := ioutil.ReadFile(p.TLSCA)
+	if err != nil {
+		log.Printf("failed to read client ca cert: %s", err.Error())
+		panic(err)
+	}
+	ok := certPool.AppendCertsFromPEM(caCert)
+	if !ok {
+		log.Printf("failed to append client certs: %s", err.Error())
+		panic(err)
+	}
+
+	clientAuth := cryptotls.RequireAndVerifyClientCert
+	if p.InsecureSkipVerify {
+		clientAuth = cryptotls.RequestClientCert
+	}
+
+	return &cryptotls.Config{
+		ClientAuth:               clientAuth,
+		ClientCAs:                certPool,
+		MinVersion:               cryptotls.VersionTLS12,
+		CipherSuites:             []uint16{cryptotls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, cryptotls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256},
+		PreferServerCipherSuites: true,
+		InsecureSkipVerify:       p.InsecureSkipVerify,
+	}
+}
+
 func (p *PrometheusClient) Close() error {
 	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
 	defer cancel()

plugins/outputs/prometheus_client/prometheus_client_tls_test.go

@@ -0,0 +1,158 @@
+package prometheus_client_test
+
+import (
+	"crypto/tls"
+	"crypto/x509"
+	"fmt"
+	"github.com/influxdata/telegraf/plugins/outputs/prometheus_client"
+	"github.com/influxdata/telegraf/testutil"
+	"github.com/influxdata/toml"
+	. "github.com/onsi/gomega"
+	"io/ioutil"
+	"net/http"
+	"os/exec"
+	"path/filepath"
+	"testing"
+)
+
+var ca, _ = filepath.Abs("assets/telegrafCA.crt")
+var cert, _ = filepath.Abs("assets/telegraf.crt")
+var key, _ = filepath.Abs("assets/telegraf.key")
+var configWithTLS = fmt.Sprintf(`
+ listen = "127.0.0.1:9090"
+ tls_ca = "%s"
+ tls_cert = "%s"
+ tls_key = "%s"
+`, ca, cert, key)
+
+var configWithoutTLS = `
+  listen = "127.0.0.1:9090"
+`
+
+type PrometheusClientTestContext struct {
+	Output      *prometheus_client.PrometheusClient
+	Accumulator *testutil.Accumulator
+	Client      *http.Client
+
+	*GomegaWithT
+}
+
+func init() {
+	path, _ := filepath.Abs("./scripts/generate_certs.sh")
+	_, err := exec.Command(path).CombinedOutput()
+	if err != nil {
+		panic(err)
+	}
+}
+
+func TestWorksWithoutTLS(t *testing.T) {
+	tc := buildTestContext(t, []byte(configWithoutTLS))
+	err := tc.Output.Connect()
+	defer tc.Output.Close()
+
+	if err != nil {
+		panic(err)
+	}
+
+	var response *http.Response
+	tc.Eventually(func() bool {
+		response, err = tc.Client.Get("http://localhost:9090/metrics")
+		return err == nil
+	}, "5s").Should(BeTrue())
+
+	if err != nil {
+		panic(err)
+	}
+
+	tc.Expect(response.StatusCode).To(Equal(http.StatusOK))
+}
+
+func TestWorksWithTLS(t *testing.T) {
+	tc := buildTestContext(t, []byte(configWithTLS))
+	err := tc.Output.Connect()
+	defer tc.Output.Close()
+
+	if err != nil {
+		panic(err)
+	}
+
+	var response *http.Response
+	tc.Eventually(func() bool {
+		response, err = tc.Client.Get("https://localhost:9090/metrics")
+		return err == nil
+	}, "5s").Should(BeTrue())
+
+	if err != nil {
+		panic(err)
+	}
+
+	tc.Expect(response.StatusCode).To(Equal(http.StatusOK))
+
+	response, err = tc.Client.Get("http://localhost:9090/metrics")
+
+	tc.Expect(err).To(HaveOccurred())
+
+	tr := &http.Transport{
+		TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
+	}
+
+	client := &http.Client{Transport: tr}
+	response, err = client.Get("https://localhost:9090/metrics")
+
+	tc.Expect(err).To(HaveOccurred())
+}
+
+func buildTestContext(t *testing.T, config []byte) *PrometheusClientTestContext {
+	output := prometheus_client.NewClient()
+	err := toml.Unmarshal(config, output)
+
+	if err != nil {
+		panic(err)
+	}
+
+	var (
+		httpClient *http.Client
+	)
+
+	if output.TLSCA != "" {
+		httpClient = buildClientWithTLS(output)
+	} else {
+		httpClient = buildClientWithoutTLS()
+	}
+
+	return &PrometheusClientTestContext{
+		Output:      output,
+		Accumulator: &testutil.Accumulator{},
+		Client:      httpClient,
+		GomegaWithT: NewGomegaWithT(t),
+	}
+}
+
+func buildClientWithoutTLS() *http.Client {
+	return &http.Client{}
+}
+
+func buildClientWithTLS(output *prometheus_client.PrometheusClient) *http.Client {
+	cert, err := tls.LoadX509KeyPair(output.TLSCert, output.TLSKey)
+	if err != nil {
+		panic(err)
+	}
+
+	caCert, err := ioutil.ReadFile(output.TLSCA)
+	if err != nil {
+		panic(err)
+	}
+	caCertPool := x509.NewCertPool()
+	caCertPool.AppendCertsFromPEM(caCert)
+
+	tlsConfig := &tls.Config{
+		Certificates: []tls.Certificate{cert},
+		RootCAs:      caCertPool,
+		MinVersion:   tls.VersionTLS12,
+		CipherSuites: []uint16{tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256},
+		ServerName:   "telegraf",
+	}
+	tlsConfig.BuildNameToCertificate()
+	transport := &http.Transport{TLSClientConfig: tlsConfig}
+	return &http.Client{Transport: transport}
+}

plugins/outputs/prometheus_client/scripts/generate_certs.sh

@@ -0,0 +1,17 @@
+#!/bin/bash -e
+
+scripts_dir=$(cd $(dirname $0) && pwd)
+
+mkdir -p ${scripts_dir}/../assets
+assets_dir=$(cd ${scripts_dir}/../assets && pwd)
+
+echo "Generating certs into ${assets_dir}"
+
+test ! `which certstrap` && go get -u -v github.com/square/certstrap
+
+rm -f ${assets_dir}/*
+
+# CA to distribute to loggregator certs
+certstrap --depot-path ${assets_dir} init --passphrase '' --common-name telegrafCA --expires "25 years"
+certstrap --depot-path ${assets_dir} request-cert --passphrase '' --common-name telegraf
+certstrap --depot-path ${assets_dir} sign telegraf --CA telegrafCA --expires "25 years"