add rabbit_mq_parser plugin

producer/main.go

@@ -0,0 +1,139 @@
+package main
+
+import (
+	"bufio"
+	"log"
+	"os"
+	"time"
+
+	"github.com/streadway/amqp"
+)
+
+func main() {
+
+	// Open file with zabbix data
+	t := time.Now()
+	f, err := os.Open("zabbix.txt")
+	defer f.Close()
+	logErr(err, "error opening file")
+	log.Printf("opened file in %v\n", time.Now().Sub(t))
+	// Make a new scanner to feed points into Queue
+	scan := bufio.NewScanner(f)
+
+	t = time.Now()
+	// Create producer to publish to Queue
+	p := newProducer()
+	defer p.conn.Close()
+	defer p.ch.Close()
+	log.Printf("established connection in %v\n", time.Now().Sub(t))
+	cntr := 0
+	cl := NewConcurrencyLimiter(1000)
+	t = time.Now()
+	for scan.Scan() {
+		body := scan.Text()
+		cl.Increment()
+		go p.publish(body, cl)
+		cntr++
+		if cntr%10000 == 0 {
+			log.Printf("sent %v lines in %v", cntr, time.Now().Sub(t))
+		}
+	}
+	log.Printf("sent %v lines in %v", cntr, time.Now().Sub(t))
+}
+
+type producer struct {
+	conn *amqp.Connection
+	ch   *amqp.Channel
+	q    amqp.Queue
+}
+
+func (p producer) publish(body string, cl *ConcurrencyLimiter) {
+	err := p.ch.Publish("", p.q.Name, false, false, amqp.Publishing{
+		DeliveryMode: amqp.Persistent,
+		ContentType:  "text/plain",
+		Body:         []byte(body),
+	})
+	if err != nil {
+		logErr(err, "Failed to publish a message")
+	}
+	cl.Decrement()
+}
+
+func newProducer() producer {
+	conn, err := amqp.Dial("amqp://guest:guest@localhost:5672/")
+	logErr(err, "Failed to connect to RabbitMQ")
+
+	ch, err := conn.Channel()
+	logErr(err, "Failed to open a channel")
+
+	q, err := ch.QueueDeclare(
+		"task_queue", // name
+		true,         // durable
+		false,        // delete when unused
+		false,        // exclusive
+		false,        // no-wait
+		nil,          // arguments
+	)
+	logErr(err, "Failed to declare a queue")
+	return producer{
+		conn: conn,
+		ch:   ch,
+		q:    q,
+	}
+}
+
+func logErr(err error, msg string) {
+	if err != nil {
+		log.Printf("%s: %s\n", msg, err)
+	}
+}
+
+// ConcurrencyLimiter is a go routine safe struct that can be used to
+// ensure that no more than a specifid max number of goroutines are
+// executing.
+type ConcurrencyLimiter struct {
+	inc   chan chan struct{}
+	dec   chan struct{}
+	max   int
+	count int
+}
+
+// NewConcurrencyLimiter returns a configured limiter that will
+// ensure that calls to Increment will block if the max is hit.
+func NewConcurrencyLimiter(max int) *ConcurrencyLimiter {
+	c := &ConcurrencyLimiter{
+		inc: make(chan chan struct{}),
+		dec: make(chan struct{}, max),
+		max: max,
+	}
+	go c.handleLimits()
+	return c
+}
+
+// Increment will increase the count of running goroutines by 1.
+// if the number is currently at the max, the call to Increment
+// will block until another goroutine decrements.
+func (c *ConcurrencyLimiter) Increment() {
+	r := make(chan struct{})
+	c.inc <- r
+	<-r
+}
+
+// Decrement will reduce the count of running goroutines by 1
+func (c *ConcurrencyLimiter) Decrement() {
+	c.dec <- struct{}{}
+}
+
+// handleLimits runs in a goroutine to manage the count of
+// running goroutines.
+func (c *ConcurrencyLimiter) handleLimits() {
+	for {
+		r := <-c.inc
+		if c.count >= c.max {
+			<-c.dec
+			c.count--
+		}
+		c.count++
+		r <- struct{}{}
+	}
+}