telegraf/producer/main.go

package main

import (
	"bufio"
	"log"
	"os"
	"time"

	"github.com/streadway/amqp"
)

func main() {

	// Open file with zabbix data
	t := time.Now()
	f, err := os.Open("zabbix.txt")
	defer f.Close()
	logErr(err, "error opening file")
	log.Printf("opened file in %v\n", time.Now().Sub(t))
	// Make a new scanner to feed points into Queue
	scan := bufio.NewScanner(f)

	t = time.Now()
	// Create producer to publish to Queue
	p := newProducer()
	defer p.conn.Close()
	defer p.ch.Close()
	log.Printf("established connection in %v\n", time.Now().Sub(t))
	cntr := 0
	cl := NewConcurrencyLimiter(1000)
	t = time.Now()
	for scan.Scan() {
		body := scan.Text()
		cl.Increment()
		go p.publish(body, cl)
		cntr++
		if cntr%10000 == 0 {
			log.Printf("sent %v lines in %v", cntr, time.Now().Sub(t))
		}
	}
	log.Printf("sent %v lines in %v", cntr, time.Now().Sub(t))
}

type producer struct {
	conn *amqp.Connection
	ch   *amqp.Channel
	q    amqp.Queue
}

func (p producer) publish(body string, cl *ConcurrencyLimiter) {
	err := p.ch.Publish("", p.q.Name, false, false, amqp.Publishing{
		DeliveryMode: amqp.Persistent,
		ContentType:  "text/plain",
		Body:         []byte(body),
	})
	if err != nil {
		logErr(err, "Failed to publish a message")
	}
	cl.Decrement()
}

func newProducer() producer {
	conn, err := amqp.Dial("amqp://guest:guest@localhost:5672/")
	logErr(err, "Failed to connect to RabbitMQ")

	ch, err := conn.Channel()
	logErr(err, "Failed to open a channel")

	q, err := ch.QueueDeclare(
		"task_queue", // name
		true,         // durable
		false,        // delete when unused
		false,        // exclusive
		false,        // no-wait
		nil,          // arguments
	)
	logErr(err, "Failed to declare a queue")
	return producer{
		conn: conn,
		ch:   ch,
		q:    q,
	}
}

func logErr(err error, msg string) {
	if err != nil {
		log.Printf("%s: %s\n", msg, err)
	}
}

// ConcurrencyLimiter is a go routine safe struct that can be used to
// ensure that no more than a specifid max number of goroutines are
// executing.
type ConcurrencyLimiter struct {
	inc   chan chan struct{}
	dec   chan struct{}
	max   int
	count int
}

// NewConcurrencyLimiter returns a configured limiter that will
// ensure that calls to Increment will block if the max is hit.
func NewConcurrencyLimiter(max int) *ConcurrencyLimiter {
	c := &ConcurrencyLimiter{
		inc: make(chan chan struct{}),
		dec: make(chan struct{}, max),
		max: max,
	}
	go c.handleLimits()
	return c
}

// Increment will increase the count of running goroutines by 1.
// if the number is currently at the max, the call to Increment
// will block until another goroutine decrements.
func (c *ConcurrencyLimiter) Increment() {
	r := make(chan struct{})
	c.inc <- r
	<-r
}

// Decrement will reduce the count of running goroutines by 1
func (c *ConcurrencyLimiter) Decrement() {
	c.dec <- struct{}{}
}

// handleLimits runs in a goroutine to manage the count of
// running goroutines.
func (c *ConcurrencyLimiter) handleLimits() {
	for {
		r := <-c.inc
		if c.count >= c.max {
			<-c.dec
			c.count--
		}
		c.count++
		r <- struct{}{}
	}
}