140 lines
3.0 KiB
Go
140 lines
3.0 KiB
Go
package main
|
|
|
|
import (
|
|
"bufio"
|
|
"log"
|
|
"os"
|
|
"time"
|
|
|
|
"github.com/streadway/amqp"
|
|
)
|
|
|
|
func main() {
|
|
|
|
// Open file with zabbix data
|
|
t := time.Now()
|
|
f, err := os.Open("zabbix.txt")
|
|
defer f.Close()
|
|
logErr(err, "error opening file")
|
|
log.Printf("opened file in %v\n", time.Now().Sub(t))
|
|
// Make a new scanner to feed points into Queue
|
|
scan := bufio.NewScanner(f)
|
|
|
|
t = time.Now()
|
|
// Create producer to publish to Queue
|
|
p := newProducer()
|
|
defer p.conn.Close()
|
|
defer p.ch.Close()
|
|
log.Printf("established connection in %v\n", time.Now().Sub(t))
|
|
cntr := 0
|
|
cl := NewConcurrencyLimiter(1000)
|
|
t = time.Now()
|
|
for scan.Scan() {
|
|
body := scan.Text()
|
|
cl.Increment()
|
|
go p.publish(body, cl)
|
|
cntr++
|
|
if cntr%10000 == 0 {
|
|
log.Printf("sent %v lines in %v", cntr, time.Now().Sub(t))
|
|
}
|
|
}
|
|
log.Printf("sent %v lines in %v", cntr, time.Now().Sub(t))
|
|
}
|
|
|
|
type producer struct {
|
|
conn *amqp.Connection
|
|
ch *amqp.Channel
|
|
q amqp.Queue
|
|
}
|
|
|
|
func (p producer) publish(body string, cl *ConcurrencyLimiter) {
|
|
err := p.ch.Publish("", p.q.Name, false, false, amqp.Publishing{
|
|
DeliveryMode: amqp.Persistent,
|
|
ContentType: "text/plain",
|
|
Body: []byte(body),
|
|
})
|
|
if err != nil {
|
|
logErr(err, "Failed to publish a message")
|
|
}
|
|
cl.Decrement()
|
|
}
|
|
|
|
func newProducer() producer {
|
|
conn, err := amqp.Dial("amqp://guest:guest@localhost:5672/")
|
|
logErr(err, "Failed to connect to RabbitMQ")
|
|
|
|
ch, err := conn.Channel()
|
|
logErr(err, "Failed to open a channel")
|
|
|
|
q, err := ch.QueueDeclare(
|
|
"task_queue", // name
|
|
true, // durable
|
|
false, // delete when unused
|
|
false, // exclusive
|
|
false, // no-wait
|
|
nil, // arguments
|
|
)
|
|
logErr(err, "Failed to declare a queue")
|
|
return producer{
|
|
conn: conn,
|
|
ch: ch,
|
|
q: q,
|
|
}
|
|
}
|
|
|
|
func logErr(err error, msg string) {
|
|
if err != nil {
|
|
log.Printf("%s: %s\n", msg, err)
|
|
}
|
|
}
|
|
|
|
// ConcurrencyLimiter is a go routine safe struct that can be used to
|
|
// ensure that no more than a specifid max number of goroutines are
|
|
// executing.
|
|
type ConcurrencyLimiter struct {
|
|
inc chan chan struct{}
|
|
dec chan struct{}
|
|
max int
|
|
count int
|
|
}
|
|
|
|
// NewConcurrencyLimiter returns a configured limiter that will
|
|
// ensure that calls to Increment will block if the max is hit.
|
|
func NewConcurrencyLimiter(max int) *ConcurrencyLimiter {
|
|
c := &ConcurrencyLimiter{
|
|
inc: make(chan chan struct{}),
|
|
dec: make(chan struct{}, max),
|
|
max: max,
|
|
}
|
|
go c.handleLimits()
|
|
return c
|
|
}
|
|
|
|
// Increment will increase the count of running goroutines by 1.
|
|
// if the number is currently at the max, the call to Increment
|
|
// will block until another goroutine decrements.
|
|
func (c *ConcurrencyLimiter) Increment() {
|
|
r := make(chan struct{})
|
|
c.inc <- r
|
|
<-r
|
|
}
|
|
|
|
// Decrement will reduce the count of running goroutines by 1
|
|
func (c *ConcurrencyLimiter) Decrement() {
|
|
c.dec <- struct{}{}
|
|
}
|
|
|
|
// handleLimits runs in a goroutine to manage the count of
|
|
// running goroutines.
|
|
func (c *ConcurrencyLimiter) handleLimits() {
|
|
for {
|
|
r := <-c.inc
|
|
if c.count >= c.max {
|
|
<-c.dec
|
|
c.count--
|
|
}
|
|
c.count++
|
|
r <- struct{}{}
|
|
}
|
|
}
|