Collect from newly discovered/launched pods (#5293)

plugins/inputs/prometheus/kubernetes.go

@@ -45,6 +45,7 @@ func (p *Prometheus) start(ctx context.Context) error {
 		if err != nil {
 			return fmt.Errorf("Failed to get current user - %v", err)
 		}
+
 		configLocation := filepath.Join(u.HomeDir, ".kube/config")
 		if p.KubeConfig != "" {
 			configLocation = p.KubeConfig
@@ -76,6 +77,10 @@ func (p *Prometheus) start(ctx context.Context) error {
 	return nil
 }
 
+// An edge case exists if a pod goes offline at the same time a new pod is created
+// (without the scrape annotations). K8s may re-assign the old pod ip to the non-scrape
+// pod, causing errors in the logs. This is only true if the pod going offline is not
+// directed to do so by K8s.
 func (p *Prometheus) watch(ctx context.Context, client *k8s.Client) error {
 	pod := &corev1.Pod{}
 	watcher, err := client.Watch(ctx, "", &corev1.Pod{})
@@ -96,18 +101,44 @@ func (p *Prometheus) watch(ctx context.Context, client *k8s.Client) error {
 				return err
 			}
 
+			// If the pod is not "ready", there will be no ip associated with it.
+			if pod.GetMetadata().GetAnnotations()["prometheus.io/scrape"] != "true" ||
+				!podReady(pod.Status.GetContainerStatuses()) {
+				continue
+			}
+
 			switch eventType {
 			case k8s.EventAdded:
 				registerPod(pod, p)
-			case k8s.EventDeleted:
-				unregisterPod(pod, p)
 			case k8s.EventModified:
+				// To avoid multiple actions for each event, unregister on the first event
+				// in the delete sequence, when the containers are still "ready".
+				if pod.Metadata.GetDeletionTimestamp() != nil {
+					unregisterPod(pod, p)
+				} else {
+					registerPod(pod, p)
+				}
 			}
 		}
 	}
 }
 
+func podReady(statuss []*corev1.ContainerStatus) bool {
+	if len(statuss) == 0 {
+		return false
+	}
+	for _, cs := range statuss {
+		if !cs.GetReady() {
+			return false
+		}
+	}
+	return true
+}
+
 func registerPod(pod *corev1.Pod, p *Prometheus) {
+	if p.kubernetesPods == nil {
+		p.kubernetesPods = map[string]URLAndAddress{}
+	}
 	targetURL := getScrapeURL(pod)
 	if targetURL == nil {
 		return
@@ -116,6 +147,9 @@ func registerPod(pod *corev1.Pod, p *Prometheus) {
 	log.Printf("D! [inputs.prometheus] will scrape metrics from %s", *targetURL)
 	// add annotation as metrics tags
 	tags := pod.GetMetadata().GetAnnotations()
+	if tags == nil {
+		tags = map[string]string{}
+	}
 	tags["pod_name"] = pod.GetMetadata().GetName()
 	tags["namespace"] = pod.GetMetadata().GetNamespace()
 	// add labels as metrics tags
@@ -129,20 +163,16 @@ func registerPod(pod *corev1.Pod, p *Prometheus) {
 	}
 	podURL := p.AddressToURL(URL, URL.Hostname())
 	p.lock.Lock()
-	p.kubernetesPods = append(p.kubernetesPods,
-		URLAndAddress{
-			URL:         podURL,
-			Address:     URL.Hostname(),
-			OriginalURL: URL,
-			Tags:        tags})
+	p.kubernetesPods[podURL.String()] = URLAndAddress{
+		URL:         podURL,
+		Address:     URL.Hostname(),
+		OriginalURL: URL,
+		Tags:        tags,
+	}
 	p.lock.Unlock()
 }
 
 func getScrapeURL(pod *corev1.Pod) *string {
-	scrape := pod.GetMetadata().GetAnnotations()["prometheus.io/scrape"]
-	if scrape != "true" {
-		return nil
-	}
 	ip := pod.Status.GetPodIP()
 	if ip == "" {
 		// return as if scrape was disabled, we will be notified again once the pod
@@ -181,18 +211,13 @@ func unregisterPod(pod *corev1.Pod, p *Prometheus) {
 		return
 	}
 
-	p.lock.Lock()
-	defer p.lock.Unlock()
 	log.Printf("D! [inputs.prometheus] registered a delete request for %s in namespace %s",
 		pod.GetMetadata().GetName(), pod.GetMetadata().GetNamespace())
-	var result []URLAndAddress
-	for _, v := range p.kubernetesPods {
-		if v.URL.String() != *url {
-			result = append(result, v)
-		} else {
-			log.Printf("D! [inputs.prometheus] will stop scraping for %s", *url)
-		}
 
+	p.lock.Lock()
+	defer p.lock.Unlock()
+	if _, ok := p.kubernetesPods[*url]; ok {
+		delete(p.kubernetesPods, *url)
+		log.Printf("D! [inputs.prometheus] will stop scraping for %s", *url)
 	}
-	p.kubernetesPods = result
 }