Fix prometheus_client reload behavior

fixes #2282

plugins/outputs/prometheus_client/prometheus_client.go

@@ -1,6 +1,7 @@
 package prometheus_client
 
 import (
+	"context"
 	"fmt"
 	"log"
 	"net/http"
@@ -24,6 +25,7 @@ type MetricWithExpiration struct {
 type PrometheusClient struct {
 	Listen             string
 	ExpirationInterval internal.Duration `toml:"expiration_interval"`
+	server             *http.Server
 
 	metrics map[string]*MetricWithExpiration
 
@@ -41,30 +43,25 @@ var sampleConfig = `
 func (p *PrometheusClient) Start() error {
 	p.metrics = make(map[string]*MetricWithExpiration)
 	prometheus.Register(p)
-	defer func() {
-		if r := recover(); r != nil {
-			// recovering from panic here because there is no way to stop a
-			// running http go server except by a kill signal. Since the server
-			// does not stop on SIGHUP, Start() will panic when the process
-			// is reloaded.
-		}
-	}()
+
 	if p.Listen == "" {
 		p.Listen = "localhost:9126"
 	}
 
-	http.Handle("/metrics", prometheus.Handler())
-	server := &http.Server{
-		Addr: p.Listen,
+	mux := http.NewServeMux()
+	mux.Handle("/metrics", prometheus.Handler())
+
+	p.server = &http.Server{
+		Addr:    p.Listen,
+		Handler: mux,
 	}
 
-	go server.ListenAndServe()
+	go p.server.ListenAndServe()
 	return nil
 }
 
 func (p *PrometheusClient) Stop() {
-	// TODO: Use a listener for http.Server that counts active connections
-	//       that can be stopped and closed gracefully
+	// plugin gets cleaned up in Close() already.
 }
 
 func (p *PrometheusClient) Connect() error {
@@ -73,8 +70,9 @@ func (p *PrometheusClient) Connect() error {
 }
 
 func (p *PrometheusClient) Close() error {
-	// This service output does not need to close any of its connections
-	return nil
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	defer cancel()
+	return p.server.Shutdown(ctx)
 }
 
 func (p *PrometheusClient) SampleConfig() string {