dom
/
telegraf
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

update godep to include paho.mqtt.

This commit is contained in:
Shirou WAKAYAMA 2015-10-05 11:07:41 +09:00
parent 7e96a9afda
commit 0cd7d6ae55
160 changed files with 44 additions and 48413 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

50
Godeps/Godeps.json generated
View File

@ -5,6 +5,11 @@
"./..."
],
"Deps": [
{
"ImportPath": "git.eclipse.org/gitroot/paho/org.eclipse.paho.mqtt.golang.git",
"Comment": "v0.9.1-14-g546c47a",
"Rev": "546c47a6d0e9492e77f6f37473d59c36a708e08b"
},
{
"ImportPath": "github.com/Shopify/sarama",
"Comment": "v1.4.3-45-g5b18996",
@ -96,10 +101,40 @@
"Rev": "d1e82c1ec3f15ee991f7cc7ffd5b67ff6f5bbaee"
},
{
"ImportPath": "github.com/influxdb/influxdb",
"ImportPath": "github.com/influxdb/influxdb/client",
"Comment": "v0.9.3",
"Rev": "5d42b212fca8facfe9db0c83822f09b88be643ec"
},
{
"ImportPath": "github.com/influxdb/influxdb/influxql",
"Comment": "v0.9.3",
"Rev": "5d42b212fca8facfe9db0c83822f09b88be643ec"
},
{
"ImportPath": "github.com/influxdb/influxdb/meta",
"Comment": "v0.9.3",
"Rev": "5d42b212fca8facfe9db0c83822f09b88be643ec"
},
{
"ImportPath": "github.com/influxdb/influxdb/snapshot",
"Comment": "v0.9.3",
"Rev": "5d42b212fca8facfe9db0c83822f09b88be643ec"
},
{
"ImportPath": "github.com/influxdb/influxdb/toml",
"Comment": "v0.9.3",
"Rev": "5d42b212fca8facfe9db0c83822f09b88be643ec"
},
{
"ImportPath": "github.com/influxdb/influxdb/tsdb",
"Comment": "v0.9.3",
"Rev": "5d42b212fca8facfe9db0c83822f09b88be643ec"
},
{
"ImportPath": "github.com/influxdb/telegraf",
"Comment": "v0.1.9-22-g64a3a71",
"Rev": "64a3a718e6cac559b8d2ab459ab1780fca9c9e22"
},
{
"ImportPath": "github.com/lib/pq",
"Comment": "go1.0-cutoff-59-gb269bd0",
@ -117,6 +152,10 @@
"ImportPath": "github.com/naoina/toml",
"Rev": "5811abcabb29d6af0fdf060f96d328962bd3cd5e"
},
{
"ImportPath": "github.com/pborman/uuid",
"Rev": "ca53cad383cad2479bbba7f7a1a05797ec1386e4"
},
{
"ImportPath": "github.com/prometheus/client_golang/extraction",
"Comment": "0.7.0-22-gbbd006b",
@ -215,6 +254,10 @@
"ImportPath": "golang.org/x/crypto/blowfish",
"Rev": "173ce04bfaf66c7bb0fa9d5c0bfd93e773909dbd"
},
{
"ImportPath": "golang.org/x/net/websocket",
"Rev": "db8e4de5b2d6653f66aea53094624468caad15d2"
},
{
"ImportPath": "gopkg.in/dancannon/gorethink.v1",
"Comment": "v1.x.x",
@ -224,11 +267,6 @@
"ImportPath": "gopkg.in/mgo.v2",
"Comment": "r2015.06.03-3-g3569c88",
"Rev": "3569c88678d88179dcbd68d02ab081cbca3cd4d0"
},
{
"ImportPath": "git.eclipse.org/gitroot/paho/org.eclipse.paho.mqtt.golang.git",
"Comment": "v0.9.1-14-g546c47a",
"Rev": "546c47a6d0e9492e77f6f37473d59c36a708e08b"
}
]
}

67
Godeps/_workspace/src/github.com/influxdb/influxdb/.gitignore generated vendored
View File

@ -1,67 +0,0 @@
*~
src/
config.json
/bin/
/pkg/
TAGS
# vim temp files
*.swp
*.test
/query/a.out*
.DS_Store
# ignore generated files.
cmd/influxd/version.go
# executables
influxd
**/influxd
!**/influxd/
influx
**/influx
!**/influx/
influxdb
**/influxdb
!**/influxdb/
/benchmark-tool
/main
/benchmark-storage
godef
gosym
gocode
inspect-raft
# dependencies
out_rpm/
packages/
# autconf
autom4te.cache/
config.log
config.status
Makefile
# log file
influxdb.log
benchmark.log
# config file
config.toml
# test data files
integration/migration_data/
# goide project files
.idea
# goconvey config files
*.goconvey

1607
Godeps/_workspace/src/github.com/influxdb/influxdb/CHANGELOG.md generated vendored
View File

File diff suppressed because it is too large Load Diff

231
Godeps/_workspace/src/github.com/influxdb/influxdb/CONTRIBUTING.md generated vendored
View File

@ -1,231 +0,0 @@
Contributing to InfluxDB
========================
Bug reports
---------------
Before you file an issue, please search existing issues in case it has already been filed, or perhaps even fixed. If you file an issue, please include the following.
* Full details of your operating system (or distribution) e.g. 64-bit Ubuntu 14.04.
* The version of InfluxDB you are running
* Whether you installed it using a pre-built package, or built it from source.
* A small test case, if applicable, that demonstrates the issues.
Remember the golden rule of bug reports: **The easier you make it for us to reproduce the problem, the faster it will get fixed.**
If you have never written a bug report before, or if you want to brush up on your bug reporting skills, we recommend reading [Simon Tatham's essay "How to Report Bugs Effectively."](http://www.chiark.greenend.org.uk/~sgtatham/bugs.html)
Test cases should be in the form of `curl` commands. For example:
```
# create database
curl -G http://localhost:8086/query --data-urlencode "q=CREATE DATABASE mydb"
# create retention policy
curl -G http://localhost:8086/query --data-urlencode "q=CREATE RETENTION POLICY myrp ON mydb DURATION 365d REPLICATION 1 DEFAULT"
# write data
curl -X POST http://localhost:8086/write --data-urlencode "db=mydb" --data-binary "cpu,region=useast,host=server_1,service=redis value=61"
# Delete a Measurement
curl -G http://localhost:8086/query --data-urlencode 'db=mydb' --data-urlencode 'q=DROP MEASUREMENT cpu'
# Query the Measurement
# Bug: expected it to return no data, but data comes back.
curl -G http://localhost:8086/query --data-urlencode 'db=mydb' --data-urlencode 'q=SELECT * from cpu'
```
**If you don't include a clear test case like this, your issue may not be investigated, and may even be closed**. If writing the data is too difficult, please zip up your data directory and include a link to it in your bug report.
Please note that issues are *not the place to file general questions* such as "how do I use collectd with InfluxDB?" Questions of this nature should be sent to the [Google Group](https://groups.google.com/forum/#!forum/influxdb), not filed as issues. Issues like this will be closed.
Feature requests
---------------
We really like to receive feature requests, as it helps us prioritize our work. Please be clear about your requirements, as incomplete feature requests may simply be closed if we don't understand what you would like to see added to InfluxDB.
Contributing to the source code
---------------
InfluxDB follows standard Go project structure. This means that all
your go development are done in `$GOPATH/src`. GOPATH can be any
directory under which InfluxDB and all its dependencies will be
cloned. For more details on recommended go project's structure, see
[How to Write Go Code](http://golang.org/doc/code.html) and
[Go: Best Practices for Production Environments](http://peter.bourgon.org/go-in-production/), or you can just follow
the steps below.
Submitting a pull request
------------
To submit a pull request you should fork the InfluxDB repository, and make your change on a feature branch of your fork. Then generate a pull request from your branch against *master* of the InfluxDB repository. Include in your pull request details of your change -- the why *and* the how -- as well as the testing your performed. Also, be sure to run the test suite with your change in place. Changes that cause tests to fail cannot be merged.
There will usually be some back and forth as we finalize the change, but once that completes it may be merged.
To assist in review for the PR, please add the following to your pull request comment:
```md
- [ ] CHANGELOG.md updated
- [ ] Rebased/mergable
- [ ] Tests pass
- [ ] Sign [CLA](http://influxdb.com/community/cla.html) (if not already signed)
```
Use of third-party packages
------------
A third-party package is defined as one that is not part of the standard Go distribution. Generally speaking we prefer to minimize our use of third-party packages, and avoid them unless absolutely necessarly. We'll often write a little bit of code rather than pull in a third-party package. Of course, we do use some third-party packages -- most importantly we use [BoltDB](https://github.com/boltdb/bolt) as the storage engine. So to maximise the chance your change will be accepted by us, use only the standard libaries, or the third-party packages we have decided to use.
For rationale, check out the post [The Case Against Third Party Libraries](http://blog.gopheracademy.com/advent-2014/case-against-3pl/).
Signing the CLA
---------------
If you are going to be contributing back to InfluxDB please take a
second to sign our CLA, which can be found
[on our website](http://influxdb.com/community/cla.html).
Installing Go
-------------
InfluxDB requires Go 1.4 or greater.
At InfluxDB we find gvm, a Go version manager, useful for installing Go. For instructions
on how to install it see [the gvm page on github](https://github.com/moovweb/gvm).
After installing gvm you can install and set the default go version by
running the following:
gvm install go1.4
gvm use go1.4 --default
Revision Control Systems
------
Go has the ability to import remote packages via revision control systems with the `go get` command. To ensure that you can retrieve any remote package, be sure to install the following rcs software to your system.
Currently the project only depends on `git` and `mercurial`.
* [Install Git](http://git-scm.com/book/en/Getting-Started-Installing-Git)
* [Install Mercurial](http://mercurial.selenic.com/wiki/Download)
Project structure
-----------------
First you need to setup the project structure:
export GOPATH=$HOME/gocodez
mkdir -p $GOPATH/src/github.com/influxdb
cd $GOPATH/src/github.com/influxdb
git clone git@github.com:influxdb/influxdb
You can add the line `export GOPATH=$HOME/gocodez` to your bash/zsh
file to be set for every shell instead of having to manually run it
everytime.
We have a pre commit hook to make sure code is formatted properly
and vetted before you commit any changes. We strongly recommend using the pre
commit hook to guard against accidentally committing unformatted
code. To use the pre-commit hook, run the following:
cd $GOPATH/src/github.com/influxdb/influxdb
cp .hooks/pre-commit .git/hooks/
In case the commit is rejected because it's not formatted you can run
the following to format the code:
```
go fmt ./...
go vet ./...
```
To install go vet, run the following command:
```
go get golang.org/x/tools/cmd/vet
```
NOTE: If you have not installed mercurial, the above command will fail. See [Revision Control Systems](#revision-control-systems) above.
For more information on `go vet`, [read the GoDoc](https://godoc.org/golang.org/x/tools/cmd/vet).
Build and Test
-----
Make sure you have Go installed and the project structure as shown above. To then build the project, execute the following commands:
```bash
cd $GOPATH/src/github.com/influxdb
go get -u -f -t ./...
go build ./...
```
To then install the binaries, run the following command. They can be found in `$GOPATH/bin`. Please note that the InfluxDB binary is named `influxd`, not `influxdb`.
```bash
go install ./...
```
To set the version and commit flags during the build pass the following to the build command:
```bash
-ldflags="-X main.version $VERSION -X main.branch $BRANCH -X main.commit $COMMIT"
```
where `$VERSION` is the version, `$BRANCH` is the branch, and `$COMMIT` is the git commit hash.
To run the tests, execute the following command:
```bash
cd $GOPATH/src/github.com/influxdb/influxdb
go test -v ./...
# run tests that match some pattern
go test -run=TestDatabase . -v
# run tests and show coverage
go test -coverprofile /tmp/cover . && go tool cover -html /tmp/cover
```
To install go cover, run the following command:
```
go get golang.org/x/tools/cmd/cover
```
Generated Google Protobuf code
-----------------
Most changes to the source do not require that the generated protocol buffer code be changed. But if you need to modify the protocol buffer code, you'll first need to install the protocol buffers toolchain.
First install the [protocol buffer compiler](https://developers.google.com/protocol-buffers/
) 2.6.1 or later for your OS:
Then install the go plugins:
```bash
go get github.com/gogo/protobuf/proto
go get github.com/gogo/protobuf/protoc-gen-gogo
go get github.com/gogo/protobuf/gogoproto
```
Finally run, `go generate` after updating any `*.proto` file:
```bash
go generate ./...
```
**Trouleshooting**
If generating the protobuf code is failing for you, check each of the following:
* Ensure the protobuf library can be found. Make sure that `LD_LIBRRARY_PATH` includes the directory in which the library `libprotoc.so` has been installed.
* Ensure the command `protoc-gen-gogo`, found in `GOPATH/bin`, is on your path. This can be done by adding `GOPATH/bin` to `PATH`.
Profiling
-----
When troubleshooting problems with CPU or memory the Go toolchain can be helpful. You can start InfluxDB with CPU or memory profiling turned on. For example:
```sh
# start influx with profiling
./influxd -cpuprofile influxd.prof
# run queries, writes, whatever you're testing
# open up pprof
go tool pprof influxd influxd.prof
# once inside run "web", opens up browser with the CPU graph
# can also run "web <function name>" to zoom in. Or "list <function name>" to see specific lines
```
Continuous Integration testing
-----
InfluxDB uses CirceCI for continuous integration testing. To see how the code is built and tested, check out [this file](https://github.com/influxdb/influxdb/blob/master/circle-test.sh). It closely follows the build and test process outlined above. You can see the exact version of Go InfluxDB uses for testing by consulting that file.
Useful links
------------
- [Useful techniques in Go](http://arslan.io/ten-useful-techniques-in-go)
- [Go in production](http://peter.bourgon.org/go-in-production/)
- [Principles of designing Go APIs with channels](https://inconshreveable.com/07-08-2014/principles-of-designing-go-apis-with-channels/)
- [Common mistakes in Golang](http://soryy.com/blog/2014/common-mistakes-with-go-lang/). Especially this section `Loops, Closures, and Local Variables`

44
Godeps/_workspace/src/github.com/influxdb/influxdb/DOCKER.md generated vendored
View File

@ -1,44 +0,0 @@
# Docker Setup
========================
This document describes how to build and run a minimal InfluxDB container under Docker. Currently, it has only been tested for local development and assumes that you have a working docker environment.
## Building Image
To build a docker image for InfluxDB from your current checkout, run the following:
```
$ ./build-docker.sh
```
This script uses the `golang:1.5` image to build a fully static binary of `influxd` and then adds it to a minimal `scratch` image.
To build the image using a different version of go:
```
$ GO_VER=1.4.2 ./build-docker.sh
```
Available version can be found [here](https://hub.docker.com/_/golang/).
## Single Node Container
This will start an interactive, single-node, that publishes the containers port `8086` and `8088` to the hosts ports `8086` and `8088` respectively. This is identical to starting `influxd` manually.
```
$ docker run -it -p 8086:8086 -p 8088:8088 influxdb
```
## Multi-Node Cluster
This will create a simple 3-node cluster. The data is stored within the container and will be lost when the container is removed. This is only useful for test clusters.
The `HOST_IP` env variable should be your host IP if running under linux or the virtualbox VM IP if running under OSX. On OSX, this would be something like: `$(docker-machine ip dev)` or `$(boot2docker ip)` depending on which docker tool you are using.
```
$ export HOST_IP=<your host/VM IP>
$ docker run -it -p 8086:8088 -p 8088:8088 influxdb -hostname $HOST_IP:8088
$ docker run -it -p 8186:8088 -p 8188:8088 influxdb -hostname $HOST_IP:8188 -join $HOST_IP:8088
$ docker run -it -p 8286:8088 -p 8288:8088 influxdb -hostname $HOST_IP:8288 -join $HOST_IP:8088
```

24
Godeps/_workspace/src/github.com/influxdb/influxdb/Dockerfile generated vendored
View File

@ -1,24 +0,0 @@
FROM busybox:ubuntu-14.04
MAINTAINER Jason Wilder "<jason@influxdb.com>"
# admin, http, udp, cluster, graphite, opentsdb, collectd
EXPOSE 8083 8086 8086/udp 8088 2003 4242 25826
WORKDIR /app
# copy binary into image
COPY influxd /app/
# Add influxd to the PATH
ENV PATH=/app:$PATH
# Generate a default config
RUN influxd config > /etc/influxdb.toml
# Use /data for all disk storage
RUN sed -i 's/dir = "\/.*influxdb/dir = "\/data/' /etc/influxdb.toml
VOLUME ["/data"]
ENTRYPOINT ["influxd", "--config", "/etc/influxdb.toml"]

20
Godeps/_workspace/src/github.com/influxdb/influxdb/LICENSE generated vendored
View File

@ -1,20 +0,0 @@
The MIT License (MIT)
Copyright (c) 2013-2015 Errplane Inc.
Permission is hereby granted, free of charge, to any person obtaining a copy of
this software and associated documentation files (the "Software"), to deal in
the Software without restriction, including without limitation the rights to
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
the Software, and to permit persons to whom the Software is furnished to do so,
subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

180
Godeps/_workspace/src/github.com/influxdb/influxdb/QUERIES.md generated vendored
View File

@ -1,180 +0,0 @@
The top level name is called a measurement. These names can contain any characters. Then there are field names, field values, tag keys and tag values, which can also contain any characters. However, if the measurement, field, or tag contains any character other than [A-Z,a-z,0-9,_], or if it starts with a digit, it must be double-quoted. Therefore anywhere a measurement name, field name, field value, tag name, or tag value appears it should be wrapped in double quotes.
# Databases & retention policies
```sql
-- create a database
CREATE DATABASE <name>
-- create a retention policy
CREATE RETENTION POLICY <rp-name> ON <db-name> DURATION <duration> REPLICATION <n> [DEFAULT]
-- alter retention policy
ALTER RETENTION POLICY <rp-name> ON <db-name> (DURATION <duration> | REPLICATION <n> | DEFAULT)+
-- drop a database
DROP DATABASE <name>
-- drop a retention policy
DROP RETENTION POLICY <rp-name> ON <db-name>
```
where `<duration>` is either `INF` for infinite retention, or an integer followed by the desired unit of time: u,ms,s,m,h,d,w for microseconds, milliseconds, seconds, minutes, hours, days, or weeks, respectively. `<replication>` must be an integer.
If present, `DEFAULT` sets the retention policy as the default retention policy for writes and reads.
# Users and permissions
```sql
-- create user
CREATE USER <name> WITH PASSWORD '<password>'
-- grant privilege on a database
GRANT <privilege> ON <db> TO <user>
-- grant cluster admin privileges
GRANT ALL [PRIVILEGES] TO <user>
-- revoke privilege
REVOKE <privilege> ON <db> FROM <user>
-- revoke all privileges for a DB
REVOKE ALL [PRIVILEGES] ON <db> FROM <user>
-- revoke all privileges including cluster admin
REVOKE ALL [PRIVILEGES] FROM <user>
-- combine db creation with privilege assignment (user must already exist)
CREATE DATABASE <name> GRANT <privilege> TO <user>
CREATE DATABASE <name> REVOKE <privilege> FROM <user>
-- delete a user
DROP USER <name>
```
where `<privilege> := READ | WRITE | All `.
Authentication must be enabled in the influxdb.conf file for user permissions to be in effect.
By default, newly created users have no privileges to any databases.
Cluster administration privileges automatically grant full read and write permissions to all databases, regardless of subsequent database-specific privilege revocation statements.
# Select
```sql
SELECT mean(value) from cpu WHERE host = 'serverA' AND time > now() - 4h GROUP BY time(5m)
SELECT mean(value) from cpu WHERE time > now() - 4h GROUP BY time(5m), region
```
## Group By
# Delete
# Series
## Destroy
```sql
DROP MEASUREMENT <name>
DROP MEASUREMENT cpu WHERE region = 'uswest'
```
## Show
Show series queries are for pulling out individual series from measurement names and tag data. They're useful for discovery.
```sql
-- show all databases
SHOW DATABASES
-- show measurement names
SHOW MEASUREMENTS
SHOW MEASUREMENTS LIMIT 15
SHOW MEASUREMENTS LIMIT 10 OFFSET 40
SHOW MEASUREMENTS WHERE service = 'redis'
-- LIMIT and OFFSET can be applied to any of the SHOW type queries
-- show all series across all measurements/tagsets
SHOW SERIES
-- get a show of all series for any measurements where tag key region = tak value 'uswest'
SHOW SERIES WHERE region = 'uswest'
SHOW SERIES FROM cpu_load WHERE region = 'uswest' LIMIT 10
-- returns the 100 - 109 rows in the result. In the case of SHOW SERIES, which returns
-- series split into measurements. Each series counts as a row. So you could see only a
-- single measurement returned, but 10 series within it.
SHOW SERIES FROM cpu_load WHERE region = 'uswest' LIMIT 10 OFFSET 100
-- show all retention policies on a database
SHOW RETENTION POLICIES ON mydb
-- get a show of all tag keys across all measurements
SHOW TAG KEYS
-- show all the tag keys for a given measurement
SHOW TAG KEYS FROM cpu
SHOW TAG KEYS FROM temperature, wind_speed
-- show all the tag values. note that a single WHERE TAG KEY = '...' clause is required
SHOW TAG VALUES WITH TAG KEY = 'region'
SHOW TAG VALUES FROM cpu WHERE region = 'uswest' WITH TAG KEY = 'host'
-- and you can do stuff against fields
SHOW FIELD KEYS FROM cpu
-- but you can't do this
SHOW FIELD VALUES
-- we don't index field values, so this query should be invalid.
-- show all users
SHOW USERS
```
Note that `FROM` and `WHERE` are optional clauses in most of the show series queries.
And the show series output looks like this:
```json
[
{
"name": "cpu",
"columns": ["id", "region", "host"],
"values": [
1, "uswest", "servera",
2, "uswest", "serverb"
]
},
{
"name": "reponse_time",
"columns": ["id", "application", "host"],
"values": [
3, "myRailsApp", "servera"
]
}
]
```
# Continuous Queries
Continous queries are going to be inspired by MySQL `TRIGGER` syntax:
http://dev.mysql.com/doc/refman/5.0/en/trigger-syntax.html
Instead of having automatically-assigned ids, named continuous queries allows for some level of duplication prevention,
particularly in the case where creation is scripted.
## Create
CREATE CONTINUOUS QUERY <name> AS SELECT ... FROM ...
## Destroy
DROP CONTINUOUS QUERY <name>
## List
SHOW CONTINUOUS QUERIES

71
Godeps/_workspace/src/github.com/influxdb/influxdb/README.md generated vendored
View File

@ -1,71 +0,0 @@
# InfluxDB [![Circle CI](https://circleci.com/gh/influxdb/influxdb/tree/master.svg?style=svg)](https://circleci.com/gh/influxdb/influxdb/tree/master)
## An Open-Source, Distributed, Time Series Database
> InfluxDB v0.9.0 is now out. Going forward, the 0.9.x series of releases will not make breaking API changes or breaking changes to the underlying data storage. However, 0.9.0 clustering should be considered an alpha release.
InfluxDB is an open source **distributed time series database** with
**no external dependencies**. It's useful for recording metrics,
events, and performing analytics.
## Features
* Built-in [HTTP API](http://influxdb.com/docs/v0.9/concepts/reading_and_writing_data.html) so you don't have to write any server side code to get up and running.
* Data can be tagged, allowing very flexible querying.
* SQL-like query language.
* Clustering is supported out of the box, so that you can scale horizontally to handle your data.
* Simple to install and manage, and fast to get data in and out.
* It aims to answer queries in real-time. That means every data point is
indexed as it comes in and is immediately available in queries that
should return in < 100ms.
## Getting Started
*The following directions apply only to the 0.9.0 release or building from the source on master.*
### Building
You don't need to build the project to use it - you can use any of our
[pre-built packages](http://influxdb.com/download/index.html) to install InfluxDB. That's
the recommended way to get it running. However, if you want to contribute to the core of InfluxDB, you'll need to build.
For those adventurous enough, you can
[follow along on our docs](http://github.com/influxdb/influxdb/blob/master/CONTRIBUTING.md).
### Starting InfluxDB
* `service influxdb start` if you have installed InfluxDB using an official Debian or RPM package.
* `$GOPATH/bin/influxd` if you have built InfluxDB from source.
### Creating your first database
```
curl -G 'http://localhost:8086/query' --data-urlencode "q=CREATE DATABASE mydb"
```
### Insert some data
```
curl -XPOST 'http://localhost:8086/write?db=mydb' \
-d 'cpu,host=server01,region=uswest load=42 1434055562000000000'
curl -XPOST 'http://localhost:8086/write?db=mydb' \
-d 'cpu,host=server02,region=uswest load=78 1434055562000000000'
curl -XPOST 'http://localhost:8086/write?db=mydb' \
-d 'cpu,host=server03,region=useast load=15.4 1434055562000000000'
```
### Query for the data
```JSON
curl -G http://localhost:8086/query?pretty=true --data-urlencode "db=mydb" \
--data-urlencode "q=SELECT * FROM cpu WHERE host='server01' AND time < now - 1d"
```
### Analyze the data
```JSON
curl -G http://localhost:8086/query?pretty=true --data-urlencode "db=mydb" \
--data-urlencode "q=SELECT mean(load) FROM cpu WHERE region='uswest'"
```
## Helpful Links
* Understand the [design goals and motivations of the project](http://influxdb.com/docs/v0.9/introduction/overview.html).
* Follow the [getting started guide](http://influxdb.com/docs/v0.9/introduction/getting_started.html) to find out how to install InfluxDB, start writing more data, and issue more queries - in just a few minutes.
* See the [HTTP API documentation to start writing a library for your favorite language](http://influxdb.com/docs/v0.9/concepts/reading_and_writing_data.html).

78
Godeps/_workspace/src/github.com/influxdb/influxdb/balancer.go generated vendored
View File

@ -1,78 +0,0 @@
package influxdb
import (
"math/rand"
"github.com/influxdb/influxdb/meta"
)
// Balancer represents a load-balancing algorithm for a set of nodes
type Balancer interface {
// Next returns the next Node according to the balancing method
// or nil if there are no nodes available
Next() *meta.NodeInfo
}
type nodeBalancer struct {
nodes []meta.NodeInfo // data nodes to balance between
p int // current node index
}
// NewNodeBalancer create a shuffled, round-robin balancer so that
// multiple instances will return nodes in randomized order and each
// each returned node will be repeated in a cycle
func NewNodeBalancer(nodes []meta.NodeInfo) Balancer {
// make a copy of the node slice so we can randomize it
// without affecting the original instance as well as ensure
// that each Balancer returns nodes in a different order
b := &nodeBalancer{}
b.nodes = make([]meta.NodeInfo, len(nodes))
copy(b.nodes, nodes)
b.shuffle()
return b
}
// shuffle randomizes the ordering the balancers available nodes
func (b *nodeBalancer) shuffle() {
for i := range b.nodes {
j := rand.Intn(i + 1)
b.nodes[i], b.nodes[j] = b.nodes[j], b.nodes[i]
}
}
// online returns a slice of the nodes that are online
func (b *nodeBalancer) online() []meta.NodeInfo {
return b.nodes
// now := time.Now().UTC()
// up := []meta.NodeInfo{}
// for _, n := range b.nodes {
// if n.OfflineUntil.After(now) {
// continue
// }
// up = append(up, n)
// }
// return up
}
// Next returns the next available nodes
func (b *nodeBalancer) Next() *meta.NodeInfo {
// only use online nodes
up := b.online()
// no nodes online
if len(up) == 0 {
return nil
}
// rollover back to the beginning
if b.p >= len(up) {
b.p = 0
}
d := &up[b.p]
b.p += 1
return d
}

115
Godeps/_workspace/src/github.com/influxdb/influxdb/balancer_test.go generated vendored
View File

@ -1,115 +0,0 @@
package influxdb_test
import (
"fmt"
"testing"
"github.com/influxdb/influxdb"
"github.com/influxdb/influxdb/meta"
)
func NewNodes() []meta.NodeInfo {
var nodes []meta.NodeInfo
for i := 1; i <= 2; i++ {
nodes = append(nodes, meta.NodeInfo{
ID: uint64(i),
Host: fmt.Sprintf("localhost:999%d", i),
})
}
return nodes
}
func TestBalancerEmptyNodes(t *testing.T) {
b := influxdb.NewNodeBalancer([]meta.NodeInfo{})
got := b.Next()
if got != nil {
t.Errorf("expected nil, got %v", got)
}
}
func TestBalancerUp(t *testing.T) {
nodes := NewNodes()
b := influxdb.NewNodeBalancer(nodes)
// First node in randomized round-robin order
first := b.Next()
if first == nil {
t.Errorf("expected datanode, got %v", first)
}
// Second node in randomized round-robin order
second := b.Next()
if second == nil {
t.Errorf("expected datanode, got %v", second)
}
// Should never get the same node in order twice
if first.ID == second.ID {
t.Errorf("expected first != second. got %v = %v", first.ID, second.ID)
}
}
/*
func TestBalancerDown(t *testing.T) {
nodes := NewNodes()
b := influxdb.NewNodeBalancer(nodes)
nodes[0].Down()
// First node in randomized round-robin order
first := b.Next()
if first == nil {
t.Errorf("expected datanode, got %v", first)
}
// Second node should rollover to the first up node
second := b.Next()
if second == nil {
t.Errorf("expected datanode, got %v", second)
}
// Health node should be returned each time
if first.ID != 2 && first.ID != second.ID {
t.Errorf("expected first != second. got %v = %v", first.ID, second.ID)
}
}
*/
/*
func TestBalancerBackUp(t *testing.T) {
nodes := newDataNodes()
b := influxdb.NewNodeBalancer(nodes)
nodes[0].Down()
for i := 0; i < 3; i++ {
got := b.Next()
if got == nil {
t.Errorf("expected datanode, got %v", got)
}
if exp := uint64(2); got.ID != exp {
t.Errorf("wrong node id: exp %v, got %v", exp, got.ID)
}
}
nodes[0].Up()
// First node in randomized round-robin order
first := b.Next()
if first == nil {
t.Errorf("expected datanode, got %v", first)
}
// Second node should rollover to the first up node
second := b.Next()
if second == nil {
t.Errorf("expected datanode, got %v", second)
}
// Should get both nodes returned
if first.ID == second.ID {
t.Errorf("expected first != second. got %v = %v", first.ID, second.ID)
}
}
*/

7
Godeps/_workspace/src/github.com/influxdb/influxdb/build-docker.sh generated vendored
View File

@ -1,7 +0,0 @@
#!/bin/sh -x -e
GO_VER=${GO_VER:-1.5}
docker run -it -v "$GOPATH":/gopath -v "$(pwd)":/app -e "GOPATH=/gopath" -w /app golang:$GO_VER sh -c 'CGO_ENABLED=0 go build -a --installsuffix cgo --ldflags="-s" -o influxd ./cmd/influxd'
docker build -t influxdb .

63
Godeps/_workspace/src/github.com/influxdb/influxdb/circle-test.sh generated vendored
View File

@ -1,63 +0,0 @@
#!/bin/bash
#
# This is the InfluxDB CircleCI test script. Using this script allows total control
# the environment in which the build and test is run, and matches the official
# build process for InfluxDB.
BUILD_DIR=$HOME/influxdb-build
GO_VERSION=go1.4.2
PARALLELISM="-parallel 256"
TIMEOUT="-timeout 480s"
# Executes the given statement, and exits if the command returns a non-zero code.
function exit_if_fail {
command=$@
echo "Executing '$command'"
$command
rc=$?
if [ $rc -ne 0 ]; then
echo "'$command' returned $rc."
exit $rc
fi
}
source $HOME/.gvm/scripts/gvm
exit_if_fail gvm use $GO_VERSION
# Set up the build directory, and then GOPATH.
exit_if_fail mkdir $BUILD_DIR
export GOPATH=$BUILD_DIR
exit_if_fail mkdir -p $GOPATH/src/github.com/influxdb
# Dump some test config to the log.
echo "Test configuration"
echo "========================================"
echo "\$HOME: $HOME"
echo "\$GOPATH: $GOPATH"
echo "\$CIRCLE_BRANCH: $CIRCLE_BRANCH"
# Move the checked-out source to a better location.
exit_if_fail mv $HOME/influxdb $GOPATH/src/github.com/influxdb
exit_if_fail cd $GOPATH/src/github.com/influxdb/influxdb
exit_if_fail git branch --set-upstream-to=origin/$CIRCLE_BRANCH $CIRCLE_BRANCH
# Install the code.
exit_if_fail cd $GOPATH/src/github.com/influxdb/influxdb
exit_if_fail go get -t -d -v ./...
exit_if_fail git checkout $CIRCLE_BRANCH # 'go get' switches to master. Who knew? Switch back.
exit_if_fail go build -v ./...
# Run the tests.
exit_if_fail go tool vet --composites=false .
case $CIRCLE_NODE_INDEX in
0)
go test $PARALLELISM $TIMEOUT -v ./... 2>&1 | tee $CIRCLE_ARTIFACTS/test_logs.txt
rc=${PIPESTATUS[0]}
;;
1)
GORACE="halt_on_error=1" go test $PARALLELISM $TIMEOUT -v -race ./... 2>&1 | tee $CIRCLE_ARTIFACTS/test_logs_race.txt
rc=${PIPESTATUS[0]}
;;
esac
exit $rc

12
Godeps/_workspace/src/github.com/influxdb/influxdb/circle.yml generated vendored
View File

@ -1,12 +0,0 @@
machine:
pre:
- bash < <(curl -s -S -L https://raw.githubusercontent.com/moovweb/gvm/master/binscripts/gvm-installer)
- source $HOME/.gvm/scripts/gvm; gvm install go1.4.2 --binary
dependencies:
override:
- echo "Dummy override, so no Circle dependencies execute"
test:
override:
- bash circle-test.sh:
parallel: true

57
Godeps/_workspace/src/github.com/influxdb/influxdb/cluster/client_pool.go generated vendored
View File

@ -1,57 +0,0 @@
package cluster
import (
"net"
"sync"
"gopkg.in/fatih/pool.v2"
)
type clientPool struct {
mu sync.RWMutex
pool map[uint64]pool.Pool
}
func newClientPool() *clientPool {
return &clientPool{
pool: make(map[uint64]pool.Pool),
}
}
func (c *clientPool) setPool(nodeID uint64, p pool.Pool) {
c.mu.Lock()
c.pool[nodeID] = p
c.mu.Unlock()
}
func (c *clientPool) getPool(nodeID uint64) (pool.Pool, bool) {
c.mu.RLock()
p, ok := c.pool[nodeID]
c.mu.RUnlock()
return p, ok
}
func (c *clientPool) size() int {
c.mu.RLock()
var size int
for _, p := range c.pool {
size += p.Len()
}
c.mu.RUnlock()
return size
}
func (c *clientPool) conn(nodeID uint64) (net.Conn, error) {
c.mu.RLock()
conn, err := c.pool[nodeID].Get()
c.mu.RUnlock()
return conn, err
}
func (c *clientPool) close() {
c.mu.Lock()
for _, p := range c.pool {
p.Close()
}
c.mu.Unlock()
}

35
Godeps/_workspace/src/github.com/influxdb/influxdb/cluster/config.go generated vendored
View File

@ -1,35 +0,0 @@
package cluster
import (
"time"
"github.com/influxdb/influxdb/toml"
)
const (
// DefaultWriteTimeout is the default timeout for a complete write to succeed.
DefaultWriteTimeout = 5 * time.Second
// DefaultShardWriterTimeout is the default timeout set on shard writers.
DefaultShardWriterTimeout = 5 * time.Second
// DefaultShardMapperTimeout is the default timeout set on shard mappers.
DefaultShardMapperTimeout = 5 * time.Second
)
// Config represents the configuration for the clustering service.
type Config struct {
ForceRemoteShardMapping bool `toml:"force-remote-mapping"`
WriteTimeout toml.Duration `toml:"write-timeout"`
ShardWriterTimeout toml.Duration `toml:"shard-writer-timeout"`
ShardMapperTimeout toml.Duration `toml:"shard-mapper-timeout"`
}
// NewConfig returns an instance of Config with defaults.
func NewConfig() Config {
return Config{
WriteTimeout: toml.Duration(DefaultWriteTimeout),
ShardWriterTimeout: toml.Duration(DefaultShardWriterTimeout),
ShardMapperTimeout: toml.Duration(DefaultShardMapperTimeout),
}
}

27
Godeps/_workspace/src/github.com/influxdb/influxdb/cluster/config_test.go generated vendored
View File

@ -1,27 +0,0 @@
package cluster_test
import (
"testing"
"time"
"github.com/BurntSushi/toml"
"github.com/influxdb/influxdb/cluster"
)
func TestConfig_Parse(t *testing.T) {
// Parse configuration.
var c cluster.Config
if _, err := toml.Decode(`
shard-writer-timeout = "10s"
write-timeout = "20s"
`, &c); err != nil {
t.Fatal(err)
}
// Validate configuration.
if time.Duration(c.ShardWriterTimeout) != 10*time.Second {
t.Fatalf("unexpected shard-writer timeout: %s", c.ShardWriterTimeout)
} else if time.Duration(c.WriteTimeout) != 20*time.Second {
t.Fatalf("unexpected write timeout s: %s", c.WriteTimeout)
}
}

286
Godeps/_workspace/src/github.com/influxdb/influxdb/cluster/internal/data.pb.go generated vendored
View File

@ -1,286 +0,0 @@
// Code generated by protoc-gen-gogo.
// source: internal/data.proto
// DO NOT EDIT!
/*
Package internal is a generated protocol buffer package.
It is generated from these files:
internal/data.proto
It has these top-level messages:
WriteShardRequest
Field
Tag
Point
WriteShardResponse
MapShardRequest
MapShardResponse
*/
package internal
import proto "github.com/gogo/protobuf/proto"
import math "math"
// Reference imports to suppress errors if they are not otherwise used.
var _ = proto.Marshal
var _ = math.Inf
type WriteShardRequest struct {
ShardID *uint64 `protobuf:"varint,1,req" json:"ShardID,omitempty"`
Points []*Point `protobuf:"bytes,2,rep" json:"Points,omitempty"`
XXX_unrecognized []byte `json:"-"`
}
func (m *WriteShardRequest) Reset() { *m = WriteShardRequest{} }
func (m *WriteShardRequest) String() string { return proto.CompactTextString(m) }
func (*WriteShardRequest) ProtoMessage() {}
func (m *WriteShardRequest) GetShardID() uint64 {
if m != nil && m.ShardID != nil {
return *m.ShardID
}
return 0
}
func (m *WriteShardRequest) GetPoints() []*Point {
if m != nil {
return m.Points
}
return nil
}
type Field struct {
Name *string `protobuf:"bytes,1,req" json:"Name,omitempty"`
Int32 *int32 `protobuf:"varint,2,opt" json:"Int32,omitempty"`
Int64 *int64 `protobuf:"varint,3,opt" json:"Int64,omitempty"`
Float64 *float64 `protobuf:"fixed64,4,opt" json:"Float64,omitempty"`
Bool *bool `protobuf:"varint,5,opt" json:"Bool,omitempty"`
String_ *string `protobuf:"bytes,6,opt" json:"String,omitempty"`
Bytes []byte `protobuf:"bytes,7,opt" json:"Bytes,omitempty"`
XXX_unrecognized []byte `json:"-"`
}
func (m *Field) Reset() { *m = Field{} }
func (m *Field) String() string { return proto.CompactTextString(m) }
func (*Field) ProtoMessage() {}
func (m *Field) GetName() string {
if m != nil && m.Name != nil {
return *m.Name
}
return ""
}
func (m *Field) GetInt32() int32 {
if m != nil && m.Int32 != nil {
return *m.Int32
}
return 0
}
func (m *Field) GetInt64() int64 {
if m != nil && m.Int64 != nil {
return *m.Int64
}
return 0
}
func (m *Field) GetFloat64() float64 {
if m != nil && m.Float64 != nil {
return *m.Float64
}
return 0
}
func (m *Field) GetBool() bool {
if m != nil && m.Bool != nil {
return *m.Bool
}
return false
}
func (m *Field) GetString_() string {
if m != nil && m.String_ != nil {
return *m.String_
}
return ""
}
func (m *Field) GetBytes() []byte {
if m != nil {
return m.Bytes
}
return nil
}
type Tag struct {
Key *string `protobuf:"bytes,1,req" json:"Key,omitempty"`
Value *string `protobuf:"bytes,2,req" json:"Value,omitempty"`
XXX_unrecognized []byte `json:"-"`
}
func (m *Tag) Reset() { *m = Tag{} }
func (m *Tag) String() string { return proto.CompactTextString(m) }
func (*Tag) ProtoMessage() {}
func (m *Tag) GetKey() string {
if m != nil && m.Key != nil {
return *m.Key
}
return ""
}
func (m *Tag) GetValue() string {
if m != nil && m.Value != nil {
return *m.Value
}
return ""
}
type Point struct {
Name *string `protobuf:"bytes,1,req" json:"Name,omitempty"`
Time *int64 `protobuf:"varint,2,req" json:"Time,omitempty"`
Fields []*Field `protobuf:"bytes,3,rep" json:"Fields,omitempty"`
Tags []*Tag `protobuf:"bytes,4,rep" json:"Tags,omitempty"`
XXX_unrecognized []byte `json:"-"`
}
func (m *Point) Reset() { *m = Point{} }
func (m *Point) String() string { return proto.CompactTextString(m) }
func (*Point) ProtoMessage() {}
func (m *Point) GetName() string {
if m != nil && m.Name != nil {
return *m.Name
}
return ""
}
func (m *Point) GetTime() int64 {
if m != nil && m.Time != nil {
return *m.Time
}
return 0
}
func (m *Point) GetFields() []*Field {
if m != nil {
return m.Fields
}
return nil
}
func (m *Point) GetTags() []*Tag {
if m != nil {
return m.Tags
}
return nil
}
type WriteShardResponse struct {
Code *int32 `protobuf:"varint,1,req" json:"Code,omitempty"`
Message *string `protobuf:"bytes,2,opt" json:"Message,omitempty"`
XXX_unrecognized []byte `json:"-"`
}
func (m *WriteShardResponse) Reset() { *m = WriteShardResponse{} }
func (m *WriteShardResponse) String() string { return proto.CompactTextString(m) }
func (*WriteShardResponse) ProtoMessage() {}
func (m *WriteShardResponse) GetCode() int32 {
if m != nil && m.Code != nil {
return *m.Code
}
return 0
}
func (m *WriteShardResponse) GetMessage() string {
if m != nil && m.Message != nil {
return *m.Message
}
return ""
}
type MapShardRequest struct {
ShardID *uint64 `protobuf:"varint,1,req" json:"ShardID,omitempty"`
Query *string `protobuf:"bytes,2,req" json:"Query,omitempty"`
ChunkSize *int32 `protobuf:"varint,3,req" json:"ChunkSize,omitempty"`
XXX_unrecognized []byte `json:"-"`
}
func (m *MapShardRequest) Reset() { *m = MapShardRequest{} }
func (m *MapShardRequest) String() string { return proto.CompactTextString(m) }
func (*MapShardRequest) ProtoMessage() {}
func (m *MapShardRequest) GetShardID() uint64 {
if m != nil && m.ShardID != nil {
return *m.ShardID
}
return 0
}
func (m *MapShardRequest) GetQuery() string {
if m != nil && m.Query != nil {
return *m.Query
}
return ""
}
func (m *MapShardRequest) GetChunkSize() int32 {
if m != nil && m.ChunkSize != nil {
return *m.ChunkSize
}
return 0
}
type MapShardResponse struct {
Code *int32 `protobuf:"varint,1,req" json:"Code,omitempty"`
Message *string `protobuf:"bytes,2,opt" json:"Message,omitempty"`
Data []byte `protobuf:"bytes,3,opt" json:"Data,omitempty"`
TagSets []string `protobuf:"bytes,4,rep" json:"TagSets,omitempty"`
Fields []string `protobuf:"bytes,5,rep" json:"Fields,omitempty"`
XXX_unrecognized []byte `json:"-"`
}
func (m *MapShardResponse) Reset() { *m = MapShardResponse{} }
func (m *MapShardResponse) String() string { return proto.CompactTextString(m) }
func (*MapShardResponse) ProtoMessage() {}
func (m *MapShardResponse) GetCode() int32 {
if m != nil && m.Code != nil {
return *m.Code
}
return 0
}
func (m *MapShardResponse) GetMessage() string {
if m != nil && m.Message != nil {
return *m.Message
}
return ""
}
func (m *MapShardResponse) GetData() []byte {
if m != nil {
return m.Data
}
return nil
}
func (m *MapShardResponse) GetTagSets() []string {
if m != nil {
return m.TagSets
}
return nil
}
func (m *MapShardResponse) GetFields() []string {
if m != nil {
return m.Fields
}
return nil
}
func init() {
}

49
Godeps/_workspace/src/github.com/influxdb/influxdb/cluster/internal/data.proto generated vendored
View File

@ -1,49 +0,0 @@
package internal;
message WriteShardRequest {
required uint64 ShardID = 1;
repeated Point Points = 2;
}
message Field {
required string Name = 1;
oneof Value {
int32 Int32 = 2;
int64 Int64 = 3;
double Float64 = 4;
bool Bool = 5;
string String = 6;
bytes Bytes = 7;
}
}
message Tag {
required string Key = 1;
required string Value = 2;
}
message Point {
required string Name = 1;
required int64 Time = 2;
repeated Field Fields = 3;
repeated Tag Tags = 4;
}
message WriteShardResponse {
required int32 Code = 1;
optional string Message = 2;
}
message MapShardRequest {
required uint64 ShardID = 1;
required string Query = 2;
required int32 ChunkSize = 3;
}
message MapShardResponse {
required int32 Code = 1;
optional string Message = 2;
optional bytes Data = 3;
repeated string TagSets = 4;
repeated string Fields = 5;
}

314
Godeps/_workspace/src/github.com/influxdb/influxdb/cluster/points_writer.go generated vendored
View File

@ -1,314 +0,0 @@
package cluster
import (
"errors"
"fmt"
"log"
"os"
"strings"
"sync"
"time"
"github.com/influxdb/influxdb"
"github.com/influxdb/influxdb/meta"
"github.com/influxdb/influxdb/tsdb"
)
// ConsistencyLevel represent a required replication criteria before a write can
// be returned as successful
type ConsistencyLevel int
const (
// ConsistencyLevelAny allows for hinted hand off, potentially no write happened yet
ConsistencyLevelAny ConsistencyLevel = iota
// ConsistencyLevelOne requires at least one data node acknowledged a write
ConsistencyLevelOne
// ConsistencyLevelOne requires a quorum of data nodes to acknowledge a write
ConsistencyLevelQuorum
// ConsistencyLevelAll requires all data nodes to acknowledge a write
ConsistencyLevelAll
)
var (
// ErrTimeout is returned when a write times out.
ErrTimeout = errors.New("timeout")
// ErrPartialWrite is returned when a write partially succeeds but does
// not meet the requested consistency level.
ErrPartialWrite = errors.New("partial write")
// ErrWriteFailed is returned when no writes succeeded.
ErrWriteFailed = errors.New("write failed")
// ErrInvalidConsistencyLevel is returned when parsing the string version
// of a consistency level.
ErrInvalidConsistencyLevel = errors.New("invalid consistency level")
)
func ParseConsistencyLevel(level string) (ConsistencyLevel, error) {
switch strings.ToLower(level) {
case "any":
return ConsistencyLevelAny, nil
case "one":
return ConsistencyLevelOne, nil
case "quorum":
return ConsistencyLevelQuorum, nil
case "all":
return ConsistencyLevelAll, nil
default:
return 0, ErrInvalidConsistencyLevel
}
}
// PointsWriter handles writes across multiple local and remote data nodes.
type PointsWriter struct {
mu sync.RWMutex
closing chan struct{}
WriteTimeout time.Duration
Logger *log.Logger
MetaStore interface {
NodeID() uint64
Database(name string) (di *meta.DatabaseInfo, err error)
RetentionPolicy(database, policy string) (*meta.RetentionPolicyInfo, error)
CreateShardGroupIfNotExists(database, policy string, timestamp time.Time) (*meta.ShardGroupInfo, error)
ShardOwner(shardID uint64) (string, string, *meta.ShardGroupInfo)
}
TSDBStore interface {
CreateShard(database, retentionPolicy string, shardID uint64) error
WriteToShard(shardID uint64, points []tsdb.Point) error
}
ShardWriter interface {
WriteShard(shardID, ownerID uint64, points []tsdb.Point) error
}
HintedHandoff interface {
WriteShard(shardID, ownerID uint64, points []tsdb.Point) error
}
}
// NewPointsWriter returns a new instance of PointsWriter for a node.
func NewPointsWriter() *PointsWriter {
return &PointsWriter{
closing: make(chan struct{}),
WriteTimeout: DefaultWriteTimeout,
Logger: log.New(os.Stderr, "[write] ", log.LstdFlags),
}
}
// ShardMapping contains a mapping of a shards to a points.
type ShardMapping struct {
Points map[uint64][]tsdb.Point // The points associated with a shard ID
Shards map[uint64]*meta.ShardInfo // The shards that have been mapped, keyed by shard ID
}
// NewShardMapping creates an empty ShardMapping
func NewShardMapping() *ShardMapping {
return &ShardMapping{
Points: map[uint64][]tsdb.Point{},
Shards: map[uint64]*meta.ShardInfo{},
}
}
// MapPoint maps a point to shard
func (s *ShardMapping) MapPoint(shardInfo *meta.ShardInfo, p tsdb.Point) {
points, ok := s.Points[shardInfo.ID]
if !ok {
s.Points[shardInfo.ID] = []tsdb.Point{p}
} else {
s.Points[shardInfo.ID] = append(points, p)
}
s.Shards[shardInfo.ID] = shardInfo
}
func (w *PointsWriter) Open() error {
w.mu.Lock()
defer w.mu.Unlock()
if w.closing == nil {
w.closing = make(chan struct{})
}
return nil
}
func (w *PointsWriter) Close() error {
w.mu.Lock()
defer w.mu.Unlock()
if w.closing != nil {
close(w.closing)
w.closing = nil
}
return nil
}
// MapShards maps the points contained in wp to a ShardMapping. If a point
// maps to a shard group or shard that does not currently exist, it will be
// created before returning the mapping.
func (w *PointsWriter) MapShards(wp *WritePointsRequest) (*ShardMapping, error) {
// holds the start time ranges for required shard groups
timeRanges := map[time.Time]*meta.ShardGroupInfo{}
rp, err := w.MetaStore.RetentionPolicy(wp.Database, wp.RetentionPolicy)
if err != nil {
return nil, err
}
for _, p := range wp.Points {
timeRanges[p.Time().Truncate(rp.ShardGroupDuration)] = nil
}
// holds all the shard groups and shards that are required for writes
for t := range timeRanges {
sg, err := w.MetaStore.CreateShardGroupIfNotExists(wp.Database, wp.RetentionPolicy, t)
if err != nil {
return nil, err
}
timeRanges[t] = sg
}
mapping := NewShardMapping()
for _, p := range wp.Points {
sg := timeRanges[p.Time().Truncate(rp.ShardGroupDuration)]
sh := sg.ShardFor(p.HashID())
mapping.MapPoint(&sh, p)
}
return mapping, nil
}
// WritePoints writes across multiple local and remote data nodes according the consistency level.
func (w *PointsWriter) WritePoints(p *WritePointsRequest) error {
if p.RetentionPolicy == "" {
db, err := w.MetaStore.Database(p.Database)
if err != nil {
return err
} else if db == nil {
return influxdb.ErrDatabaseNotFound(p.Database)
}
p.RetentionPolicy = db.DefaultRetentionPolicy
}
shardMappings, err := w.MapShards(p)
if err != nil {
return err
}
// Write each shard in it's own goroutine and return as soon
// as one fails.
ch := make(chan error, len(shardMappings.Points))
for shardID, points := range shardMappings.Points {
go func(shard *meta.ShardInfo, database, retentionPolicy string, points []tsdb.Point) {
ch <- w.writeToShard(shard, p.Database, p.RetentionPolicy, p.ConsistencyLevel, points)
}(shardMappings.Shards[shardID], p.Database, p.RetentionPolicy, points)
}
for range shardMappings.Points {
select {
case <-w.closing:
return ErrWriteFailed
case err := <-ch:
if err != nil {
return err
}
}
}
return nil
}
// writeToShards writes points to a shard and ensures a write consistency level has been met. If the write
// partially succeeds, ErrPartialWrite is returned.
func (w *PointsWriter) writeToShard(shard *meta.ShardInfo, database, retentionPolicy string,
consistency ConsistencyLevel, points []tsdb.Point) error {
// The required number of writes to achieve the requested consistency level
required := len(shard.OwnerIDs)
switch consistency {
case ConsistencyLevelAny, ConsistencyLevelOne:
required = 1
case ConsistencyLevelQuorum:
required = required/2 + 1
}
// response channel for each shard writer go routine
ch := make(chan error, len(shard.OwnerIDs))
for _, nodeID := range shard.OwnerIDs {
go func(shardID, nodeID uint64, points []tsdb.Point) {
if w.MetaStore.NodeID() == nodeID {
err := w.TSDBStore.WriteToShard(shardID, points)
// If we've written to shard that should exist on the current node, but the store has
// not actually created this shard, tell it to create it and retry the write
if err == tsdb.ErrShardNotFound {
err = w.TSDBStore.CreateShard(database, retentionPolicy, shardID)
if err != nil {
ch <- err
return
}
err = w.TSDBStore.WriteToShard(shardID, points)
}
ch <- err
return
}
err := w.ShardWriter.WriteShard(shardID, nodeID, points)
if err != nil && tsdb.IsRetryable(err) {
// The remote write failed so queue it via hinted handoff
hherr := w.HintedHandoff.WriteShard(shardID, nodeID, points)
// If the write consistency level is ANY, then a successful hinted handoff can
// be considered a successful write so send nil to the response channel
// otherwise, let the original error propogate to the response channel
if hherr == nil && consistency == ConsistencyLevelAny {
ch <- nil
return
}
}
ch <- err
}(shard.ID, nodeID, points)
}
var wrote int
timeout := time.After(w.WriteTimeout)
var writeError error
for _, nodeID := range shard.OwnerIDs {
select {
case <-w.closing:
return ErrWriteFailed
case <-timeout:
// return timeout error to caller
return ErrTimeout
case err := <-ch:
// If the write returned an error, continue to the next response
if err != nil {
w.Logger.Printf("write failed for shard %d on node %d: %v", shard.ID, nodeID, err)
// Keep track of the first error we see to return back to the client
if writeError == nil {
writeError = err
}
continue
}
wrote += 1
}
}
// We wrote the required consistency level
if wrote >= required {
return nil
}
if wrote > 0 {
return ErrPartialWrite
}
if writeError != nil {
return fmt.Errorf("write failed: %v", writeError)
}
return ErrWriteFailed
}

436
Godeps/_workspace/src/github.com/influxdb/influxdb/cluster/points_writer_test.go generated vendored
View File

@ -1,436 +0,0 @@
package cluster_test
import (
"fmt"
"sync"
"sync/atomic"
"testing"
"time"
"github.com/influxdb/influxdb/cluster"
"github.com/influxdb/influxdb/meta"
"github.com/influxdb/influxdb/tsdb"
)
// Ensures the points writer maps a single point to a single shard.
func TestPointsWriter_MapShards_One(t *testing.T) {
ms := MetaStore{}
rp := NewRetentionPolicy("myp", time.Hour, 3)
ms.NodeIDFn = func() uint64 { return 1 }
ms.RetentionPolicyFn = func(db, retentionPolicy string) (*meta.RetentionPolicyInfo, error) {
return rp, nil
}
ms.CreateShardGroupIfNotExistsFn = func(database, policy string, timestamp time.Time) (*meta.ShardGroupInfo, error) {
return &rp.ShardGroups[0], nil
}
c := cluster.PointsWriter{MetaStore: ms}
pr := &cluster.WritePointsRequest{
Database: "mydb",
RetentionPolicy: "myrp",
ConsistencyLevel: cluster.ConsistencyLevelOne,
}
pr.AddPoint("cpu", 1.0, time.Now(), nil)
var (
shardMappings *cluster.ShardMapping
err error
)
if shardMappings, err = c.MapShards(pr); err != nil {
t.Fatalf("unexpected an error: %v", err)
}
if exp := 1; len(shardMappings.Points) != exp {
t.Errorf("MapShards() len mismatch. got %v, exp %v", len(shardMappings.Points), exp)
}
}
// Ensures the points writer maps a multiple points across shard group boundaries.
func TestPointsWriter_MapShards_Multiple(t *testing.T) {
ms := MetaStore{}
rp := NewRetentionPolicy("myp", time.Hour, 3)
AttachShardGroupInfo(rp, []uint64{1, 2, 3})
AttachShardGroupInfo(rp, []uint64{1, 2, 3})
ms.NodeIDFn = func() uint64 { return 1 }
ms.RetentionPolicyFn = func(db, retentionPolicy string) (*meta.RetentionPolicyInfo, error) {
return rp, nil
}
ms.CreateShardGroupIfNotExistsFn = func(database, policy string, timestamp time.Time) (*meta.ShardGroupInfo, error) {
for i, sg := range rp.ShardGroups {
if timestamp.Equal(sg.StartTime) || timestamp.After(sg.StartTime) && timestamp.Before(sg.EndTime) {
return &rp.ShardGroups[i], nil
}
}
panic("should not get here")
}
c := cluster.PointsWriter{MetaStore: ms}
pr := &cluster.WritePointsRequest{
Database: "mydb",
RetentionPolicy: "myrp",
ConsistencyLevel: cluster.ConsistencyLevelOne,
}
// Three points that range over the shardGroup duration (1h) and should map to two
// distinct shards
pr.AddPoint("cpu", 1.0, time.Unix(0, 0), nil)
pr.AddPoint("cpu", 2.0, time.Unix(0, 0).Add(time.Hour), nil)
pr.AddPoint("cpu", 3.0, time.Unix(0, 0).Add(time.Hour+time.Second), nil)
var (
shardMappings *cluster.ShardMapping
err error
)
if shardMappings, err = c.MapShards(pr); err != nil {
t.Fatalf("unexpected an error: %v", err)
}
if exp := 2; len(shardMappings.Points) != exp {
t.Errorf("MapShards() len mismatch. got %v, exp %v", len(shardMappings.Points), exp)
}
for _, points := range shardMappings.Points {
// First shard shoud have 1 point w/ first point added
if len(points) == 1 && points[0].Time() != pr.Points[0].Time() {
t.Fatalf("MapShards() value mismatch. got %v, exp %v", points[0].Time(), pr.Points[0].Time())
}
// Second shard shoud have the last two points added
if len(points) == 2 && points[0].Time() != pr.Points[1].Time() {
t.Fatalf("MapShards() value mismatch. got %v, exp %v", points[0].Time(), pr.Points[1].Time())
}
if len(points) == 2 && points[1].Time() != pr.Points[2].Time() {
t.Fatalf("MapShards() value mismatch. got %v, exp %v", points[1].Time(), pr.Points[2].Time())
}
}
}
func TestPointsWriter_WritePoints(t *testing.T) {
tests := []struct {
name string
database string
retentionPolicy string
consistency cluster.ConsistencyLevel
// the responses returned by each shard write call. node ID 1 = pos 0
err []error
expErr error
}{
// Consistency one
{
name: "write one success",
database: "mydb",
retentionPolicy: "myrp",
consistency: cluster.ConsistencyLevelOne,
err: []error{nil, nil, nil},
expErr: nil,
},
{
name: "write one error",
database: "mydb",
retentionPolicy: "myrp",
consistency: cluster.ConsistencyLevelOne,
err: []error{fmt.Errorf("a failure"), fmt.Errorf("a failure"), fmt.Errorf("a failure")},
expErr: fmt.Errorf("write failed: a failure"),
},
// Consistency any
{
name: "write any success",
database: "mydb",
retentionPolicy: "myrp",
consistency: cluster.ConsistencyLevelAny,
err: []error{fmt.Errorf("a failure"), nil, fmt.Errorf("a failure")},
expErr: nil,
},
// Consistency all
{
name: "write all success",
database: "mydb",
retentionPolicy: "myrp",
consistency: cluster.ConsistencyLevelAll,
err: []error{nil, nil, nil},
expErr: nil,
},
{
name: "write all, 2/3, partial write",
database: "mydb",
retentionPolicy: "myrp",
consistency: cluster.ConsistencyLevelAll,
err: []error{nil, fmt.Errorf("a failure"), nil},
expErr: cluster.ErrPartialWrite,
},
{
name: "write all, 1/3 (failure)",
database: "mydb",
retentionPolicy: "myrp",
consistency: cluster.ConsistencyLevelAll,
err: []error{nil, fmt.Errorf("a failure"), fmt.Errorf("a failure")},
expErr: cluster.ErrPartialWrite,
},
// Consistency quorum
{
name: "write quorum, 1/3 failure",
consistency: cluster.ConsistencyLevelQuorum,
database: "mydb",
retentionPolicy: "myrp",
err: []error{fmt.Errorf("a failure"), fmt.Errorf("a failure"), nil},
expErr: cluster.ErrPartialWrite,
},
{
name: "write quorum, 2/3 success",
database: "mydb",
retentionPolicy: "myrp",
consistency: cluster.ConsistencyLevelQuorum,
err: []error{nil, nil, fmt.Errorf("a failure")},
expErr: nil,
},
{
name: "write quorum, 3/3 success",
database: "mydb",
retentionPolicy: "myrp",
consistency: cluster.ConsistencyLevelQuorum,
err: []error{nil, nil, nil},
expErr: nil,
},
// Error write error
{
name: "no writes succeed",
database: "mydb",
retentionPolicy: "myrp",
consistency: cluster.ConsistencyLevelOne,
err: []error{fmt.Errorf("a failure"), fmt.Errorf("a failure"), fmt.Errorf("a failure")},
expErr: fmt.Errorf("write failed: a failure"),
},
// Hinted handoff w/ ANY
{
name: "hinted handoff write succeed",
database: "mydb",
retentionPolicy: "myrp",
consistency: cluster.ConsistencyLevelAny,
err: []error{fmt.Errorf("a failure"), fmt.Errorf("a failure"), fmt.Errorf("a failure")},
expErr: nil,
},
// Write to non-existant database
{
name: "write to non-existant database",
database: "doesnt_exist",
retentionPolicy: "",
consistency: cluster.ConsistencyLevelAny,
err: []error{nil, nil, nil},
expErr: fmt.Errorf("database not found: doesnt_exist"),
},
}
for _, test := range tests {
pr := &cluster.WritePointsRequest{
Database: test.database,
RetentionPolicy: test.retentionPolicy,
ConsistencyLevel: test.consistency,
}
// Three points that range over the shardGroup duration (1h) and should map to two
// distinct shards
pr.AddPoint("cpu", 1.0, time.Unix(0, 0), nil)
pr.AddPoint("cpu", 2.0, time.Unix(0, 0).Add(time.Hour), nil)
pr.AddPoint("cpu", 3.0, time.Unix(0, 0).Add(time.Hour+time.Second), nil)
// copy to prevent data race
theTest := test
sm := cluster.NewShardMapping()
sm.MapPoint(
&meta.ShardInfo{ID: uint64(1), OwnerIDs: []uint64{uint64(1), uint64(2), uint64(3)}},
pr.Points[0])
sm.MapPoint(
&meta.ShardInfo{ID: uint64(2), OwnerIDs: []uint64{uint64(1), uint64(2), uint64(3)}},
pr.Points[1])
sm.MapPoint(
&meta.ShardInfo{ID: uint64(2), OwnerIDs: []uint64{uint64(1), uint64(2), uint64(3)}},
pr.Points[2])
// Local cluster.Node ShardWriter
// lock on the write increment since these functions get called in parallel
var mu sync.Mutex
sw := &fakeShardWriter{
ShardWriteFn: func(shardID, nodeID uint64, points []tsdb.Point) error {
mu.Lock()
defer mu.Unlock()
return theTest.err[int(nodeID)-1]
},
}
store := &fakeStore{
WriteFn: func(shardID uint64, points []tsdb.Point) error {
mu.Lock()
defer mu.Unlock()
return theTest.err[0]
},
}
hh := &fakeShardWriter{
ShardWriteFn: func(shardID, nodeID uint64, points []tsdb.Point) error {
return nil
},
}
ms := NewMetaStore()
ms.DatabaseFn = func(database string) (*meta.DatabaseInfo, error) {
return nil, nil
}
ms.NodeIDFn = func() uint64 { return 1 }
c := cluster.NewPointsWriter()
c.MetaStore = ms
c.ShardWriter = sw
c.TSDBStore = store
c.HintedHandoff = hh
err := c.WritePoints(pr)
if err == nil && test.expErr != nil {
t.Errorf("PointsWriter.WritePoints(): '%s' error: got %v, exp %v", test.name, err, test.expErr)
}
if err != nil && test.expErr == nil {
t.Errorf("PointsWriter.WritePoints(): '%s' error: got %v, exp %v", test.name, err, test.expErr)
}
if err != nil && test.expErr != nil && err.Error() != test.expErr.Error() {
t.Errorf("PointsWriter.WritePoints(): '%s' error: got %v, exp %v", test.name, err, test.expErr)
}
}
}
var shardID uint64
type fakeShardWriter struct {
ShardWriteFn func(shardID, nodeID uint64, points []tsdb.Point) error
}
func (f *fakeShardWriter) WriteShard(shardID, nodeID uint64, points []tsdb.Point) error {
return f.ShardWriteFn(shardID, nodeID, points)
}
type fakeStore struct {
WriteFn func(shardID uint64, points []tsdb.Point) error
CreateShardfn func(database, retentionPolicy string, shardID uint64) error
}
func (f *fakeStore) WriteToShard(shardID uint64, points []tsdb.Point) error {
return f.WriteFn(shardID, points)
}
func (f *fakeStore) CreateShard(database, retentionPolicy string, shardID uint64) error {
return f.CreateShardfn(database, retentionPolicy, shardID)
}
func NewMetaStore() *MetaStore {
ms := &MetaStore{}
rp := NewRetentionPolicy("myp", time.Hour, 3)
AttachShardGroupInfo(rp, []uint64{1, 2, 3})
AttachShardGroupInfo(rp, []uint64{1, 2, 3})
ms.RetentionPolicyFn = func(db, retentionPolicy string) (*meta.RetentionPolicyInfo, error) {
return rp, nil
}
ms.CreateShardGroupIfNotExistsFn = func(database, policy string, timestamp time.Time) (*meta.ShardGroupInfo, error) {
for i, sg := range rp.ShardGroups {
if timestamp.Equal(sg.StartTime) || timestamp.After(sg.StartTime) && timestamp.Before(sg.EndTime) {
return &rp.ShardGroups[i], nil
}
}
panic("should not get here")
}
return ms
}
type MetaStore struct {
NodeIDFn func() uint64
RetentionPolicyFn func(database, name string) (*meta.RetentionPolicyInfo, error)
CreateShardGroupIfNotExistsFn func(database, policy string, timestamp time.Time) (*meta.ShardGroupInfo, error)
DatabaseFn func(database string) (*meta.DatabaseInfo, error)
ShardOwnerFn func(shardID uint64) (string, string, *meta.ShardGroupInfo)
}
func (m MetaStore) NodeID() uint64 { return m.NodeIDFn() }
func (m MetaStore) RetentionPolicy(database, name string) (*meta.RetentionPolicyInfo, error) {
return m.RetentionPolicyFn(database, name)
}
func (m MetaStore) CreateShardGroupIfNotExists(database, policy string, timestamp time.Time) (*meta.ShardGroupInfo, error) {
return m.CreateShardGroupIfNotExistsFn(database, policy, timestamp)
}
func (m MetaStore) Database(database string) (*meta.DatabaseInfo, error) {
return m.DatabaseFn(database)
}
func (m MetaStore) ShardOwner(shardID uint64) (string, string, *meta.ShardGroupInfo) {
return m.ShardOwnerFn(shardID)
}
func NewRetentionPolicy(name string, duration time.Duration, nodeCount int) *meta.RetentionPolicyInfo {
shards := []meta.ShardInfo{}
ownerIDs := []uint64{}
for i := 1; i <= nodeCount; i++ {
ownerIDs = append(ownerIDs, uint64(i))
}
// each node is fully replicated with each other
shards = append(shards, meta.ShardInfo{
ID: nextShardID(),
OwnerIDs: ownerIDs,
})
rp := &meta.RetentionPolicyInfo{
Name: "myrp",
ReplicaN: nodeCount,
Duration: duration,
ShardGroupDuration: duration,
ShardGroups: []meta.ShardGroupInfo{
meta.ShardGroupInfo{
ID: nextShardID(),
StartTime: time.Unix(0, 0),
EndTime: time.Unix(0, 0).Add(duration).Add(-1),
Shards: shards,
},
},
}
return rp
}
func AttachShardGroupInfo(rp *meta.RetentionPolicyInfo, ownerIDs []uint64) {
var startTime, endTime time.Time
if len(rp.ShardGroups) == 0 {
startTime = time.Unix(0, 0)
} else {
startTime = rp.ShardGroups[len(rp.ShardGroups)-1].StartTime.Add(rp.ShardGroupDuration)
}
endTime = startTime.Add(rp.ShardGroupDuration).Add(-1)
sh := meta.ShardGroupInfo{
ID: uint64(len(rp.ShardGroups) + 1),
StartTime: startTime,
EndTime: endTime,
Shards: []meta.ShardInfo{
meta.ShardInfo{
ID: nextShardID(),
OwnerIDs: ownerIDs,
},
},
}
rp.ShardGroups = append(rp.ShardGroups, sh)
}
func nextShardID() uint64 {
return atomic.AddUint64(&shardID, 1)
}

229
Godeps/_workspace/src/github.com/influxdb/influxdb/cluster/rpc.go generated vendored
View File

@ -1,229 +0,0 @@
package cluster
import (
"time"
"github.com/gogo/protobuf/proto"
"github.com/influxdb/influxdb/cluster/internal"
"github.com/influxdb/influxdb/tsdb"
)
//go:generate protoc --gogo_out=. internal/data.proto
// MapShardRequest represents the request to map a remote shard for a query.
type MapShardRequest struct {
pb internal.MapShardRequest
}
func (m *MapShardRequest) ShardID() uint64 { return m.pb.GetShardID() }
func (m *MapShardRequest) Query() string { return m.pb.GetQuery() }
func (m *MapShardRequest) ChunkSize() int32 { return m.pb.GetChunkSize() }
func (m *MapShardRequest) SetShardID(id uint64) { m.pb.ShardID = &id }
func (m *MapShardRequest) SetQuery(query string) { m.pb.Query = &query }
func (m *MapShardRequest) SetChunkSize(chunkSize int32) { m.pb.ChunkSize = &chunkSize }
// MarshalBinary encodes the object to a binary format.
func (m *MapShardRequest) MarshalBinary() ([]byte, error) {
return proto.Marshal(&m.pb)
}
// UnmarshalBinary populates MapShardRequest from a binary format.
func (m *MapShardRequest) UnmarshalBinary(buf []byte) error {
if err := proto.Unmarshal(buf, &m.pb); err != nil {
return err
}
return nil
}
// MapShardResponse represents the response returned from a remote MapShardRequest call
type MapShardResponse struct {
pb internal.MapShardResponse
}
func NewMapShardResponse(code int, message string) *MapShardResponse {
m := &MapShardResponse{}
m.SetCode(code)
m.SetMessage(message)
return m
}
func (r *MapShardResponse) Code() int { return int(r.pb.GetCode()) }
func (r *MapShardResponse) Message() string { return r.pb.GetMessage() }
func (r *MapShardResponse) TagSets() []string { return r.pb.GetTagSets() }
func (r *MapShardResponse) Fields() []string { return r.pb.GetFields() }
func (r *MapShardResponse) Data() []byte { return r.pb.GetData() }
func (r *MapShardResponse) SetCode(code int) { r.pb.Code = proto.Int32(int32(code)) }
func (r *MapShardResponse) SetMessage(message string) { r.pb.Message = &message }
func (r *MapShardResponse) SetTagSets(tagsets []string) { r.pb.TagSets = tagsets }
func (r *MapShardResponse) SetFields(fields []string) { r.pb.Fields = fields }
func (r *MapShardResponse) SetData(data []byte) { r.pb.Data = data }
// MarshalBinary encodes the object to a binary format.
func (r *MapShardResponse) MarshalBinary() ([]byte, error) {
return proto.Marshal(&r.pb)
}
// UnmarshalBinary populates WritePointRequest from a binary format.
func (r *MapShardResponse) UnmarshalBinary(buf []byte) error {
if err := proto.Unmarshal(buf, &r.pb); err != nil {
return err
}
return nil
}
// WritePointsRequest represents a request to write point data to the cluster
type WritePointsRequest struct {
Database string
RetentionPolicy string
ConsistencyLevel ConsistencyLevel
Points []tsdb.Point
}
// AddPoint adds a point to the WritePointRequest with field name 'value'
func (w *WritePointsRequest) AddPoint(name string, value interface{}, timestamp time.Time, tags map[string]string) {
w.Points = append(w.Points, tsdb.NewPoint(
name, tags, map[string]interface{}{"value": value}, timestamp,
))
}
// WriteShardRequest represents the a request to write a slice of points to a shard
type WriteShardRequest struct {
pb internal.WriteShardRequest
}
// WriteShardResponse represents the response returned from a remote WriteShardRequest call
type WriteShardResponse struct {
pb internal.WriteShardResponse
}
func (w *WriteShardRequest) SetShardID(id uint64) { w.pb.ShardID = &id }
func (w *WriteShardRequest) ShardID() uint64 { return w.pb.GetShardID() }
func (w *WriteShardRequest) Points() []tsdb.Point { return w.unmarshalPoints() }
func (w *WriteShardRequest) AddPoint(name string, value interface{}, timestamp time.Time, tags map[string]string) {
w.AddPoints([]tsdb.Point{tsdb.NewPoint(
name, tags, map[string]interface{}{"value": value}, timestamp,
)})
}
func (w *WriteShardRequest) AddPoints(points []tsdb.Point) {
w.pb.Points = append(w.pb.Points, w.marshalPoints(points)...)
}
// MarshalBinary encodes the object to a binary format.
func (w *WriteShardRequest) MarshalBinary() ([]byte, error) {
return proto.Marshal(&w.pb)
}
func (w *WriteShardRequest) marshalPoints(points []tsdb.Point) []*internal.Point {
pts := make([]*internal.Point, len(points))
for i, p := range points {
fields := []*internal.Field{}
for k, v := range p.Fields() {
name := k
f := &internal.Field{
Name: &name,
}
switch t := v.(type) {
case int:
f.Int64 = proto.Int64(int64(t))
case int32:
f.Int32 = proto.Int32(t)
case int64:
f.Int64 = proto.Int64(t)
case float64:
f.Float64 = proto.Float64(t)
case bool:
f.Bool = proto.Bool(t)
case string:
f.String_ = proto.String(t)
case []byte:
f.Bytes = t
}
fields = append(fields, f)
}
tags := []*internal.Tag{}
for k, v := range p.Tags() {
key := k
value := v
tags = append(tags, &internal.Tag{
Key: &key,
Value: &value,
})
}
name := p.Name()
pts[i] = &internal.Point{
Name: &name,
Time: proto.Int64(p.Time().UnixNano()),
Fields: fields,
Tags: tags,
}
}
return pts
}
// UnmarshalBinary populates WritePointRequest from a binary format.
func (w *WriteShardRequest) UnmarshalBinary(buf []byte) error {
if err := proto.Unmarshal(buf, &w.pb); err != nil {
return err
}
return nil
}
func (w *WriteShardRequest) unmarshalPoints() []tsdb.Point {
points := make([]tsdb.Point, len(w.pb.GetPoints()))
for i, p := range w.pb.GetPoints() {
pt := tsdb.NewPoint(
p.GetName(), map[string]string{},
map[string]interface{}{}, time.Unix(0, p.GetTime()))
for _, f := range p.GetFields() {
n := f.GetName()
if f.Int32 != nil {
pt.AddField(n, f.GetInt32())
} else if f.Int64 != nil {
pt.AddField(n, f.GetInt64())
} else if f.Float64 != nil {
pt.AddField(n, f.GetFloat64())
} else if f.Bool != nil {
pt.AddField(n, f.GetBool())
} else if f.String_ != nil {
pt.AddField(n, f.GetString_())
} else {
pt.AddField(n, f.GetBytes())
}
}
tags := tsdb.Tags{}
for _, t := range p.GetTags() {
tags[t.GetKey()] = t.GetValue()
}
pt.SetTags(tags)
points[i] = pt
}
return points
}
func (w *WriteShardResponse) SetCode(code int) { w.pb.Code = proto.Int32(int32(code)) }
func (w *WriteShardResponse) SetMessage(message string) { w.pb.Message = &message }
func (w *WriteShardResponse) Code() int { return int(w.pb.GetCode()) }
func (w *WriteShardResponse) Message() string { return w.pb.GetMessage() }
// MarshalBinary encodes the object to a binary format.
func (w *WriteShardResponse) MarshalBinary() ([]byte, error) {
return proto.Marshal(&w.pb)
}
// UnmarshalBinary populates WritePointRequest from a binary format.
func (w *WriteShardResponse) UnmarshalBinary(buf []byte) error {
if err := proto.Unmarshal(buf, &w.pb); err != nil {
return err
}
return nil
}

110
Godeps/_workspace/src/github.com/influxdb/influxdb/cluster/rpc_test.go generated vendored
View File

@ -1,110 +0,0 @@
package cluster
import (
"testing"
"time"
)
func TestWriteShardRequestBinary(t *testing.T) {
sr := &WriteShardRequest{}
sr.SetShardID(uint64(1))
if exp := uint64(1); sr.ShardID() != exp {
t.Fatalf("ShardID mismatch: got %v, exp %v", sr.ShardID(), exp)
}
sr.AddPoint("cpu", 1.0, time.Unix(0, 0), map[string]string{"host": "serverA"})
sr.AddPoint("cpu", 2.0, time.Unix(0, 0).Add(time.Hour), nil)
sr.AddPoint("cpu_load", 3.0, time.Unix(0, 0).Add(time.Hour+time.Second), nil)
b, err := sr.MarshalBinary()
if err != nil {
t.Fatalf("WritePointsRequest.MarshalBinary() failed: %v", err)
}
if len(b) == 0 {
t.Fatalf("WritePointsRequest.MarshalBinary() returned 0 bytes")
}
got := &WriteShardRequest{}
if err := got.UnmarshalBinary(b); err != nil {
t.Fatalf("WritePointsRequest.UnmarshalMarshalBinary() failed: %v", err)
}
if got.ShardID() != sr.ShardID() {
t.Errorf("ShardID mismatch: got %v, exp %v", got.ShardID(), sr.ShardID())
}
if len(got.Points()) != len(sr.Points()) {
t.Errorf("Points count mismatch: got %v, exp %v", len(got.Points()), len(sr.Points()))
}
srPoints := sr.Points()
gotPoints := got.Points()
for i, p := range srPoints {
g := gotPoints[i]
if g.Name() != p.Name() {
t.Errorf("Point %d name mismatch: got %v, exp %v", i, g.Name(), p.Name())
}
if !g.Time().Equal(p.Time()) {
t.Errorf("Point %d time mismatch: got %v, exp %v", i, g.Time(), p.Time())
}
if g.HashID() != p.HashID() {
t.Errorf("Point #%d HashID() mismatch: got %v, exp %v", i, g.HashID(), p.HashID())
}
for k, v := range p.Tags() {
if g.Tags()[k] != v {
t.Errorf("Point #%d tag mismatch: got %v, exp %v", i, k, v)
}
}
if len(p.Fields()) != len(g.Fields()) {
t.Errorf("Point %d field count mismatch: got %v, exp %v", i, len(g.Fields()), len(p.Fields()))
}
for j, f := range p.Fields() {
if g.Fields()[j] != f {
t.Errorf("Point %d field mismatch: got %v, exp %v", i, g.Fields()[j], f)
}
}
}
}
func TestWriteShardResponseBinary(t *testing.T) {
sr := &WriteShardResponse{}
sr.SetCode(10)
sr.SetMessage("foo")
b, err := sr.MarshalBinary()
if exp := 10; sr.Code() != exp {
t.Fatalf("Code mismatch: got %v, exp %v", sr.Code(), exp)
}
if exp := "foo"; sr.Message() != exp {
t.Fatalf("Message mismatch: got %v, exp %v", sr.Message(), exp)
}
if err != nil {
t.Fatalf("WritePointsResponse.MarshalBinary() failed: %v", err)
}
if len(b) == 0 {
t.Fatalf("WritePointsResponse.MarshalBinary() returned 0 bytes")
}
got := &WriteShardResponse{}
if err := got.UnmarshalBinary(b); err != nil {
t.Fatalf("WritePointsResponse.UnmarshalMarshalBinary() failed: %v", err)
}
if got.Code() != sr.Code() {
t.Errorf("Code mismatch: got %v, exp %v", got.Code(), sr.Code())
}
if got.Message() != sr.Message() {
t.Errorf("Message mismatch: got %v, exp %v", got.Message(), sr.Message())
}
}

338
Godeps/_workspace/src/github.com/influxdb/influxdb/cluster/service.go generated vendored
View File

@ -1,338 +0,0 @@
package cluster
import (
"encoding/binary"
"encoding/json"
"fmt"
"io"
"log"
"net"
"os"
"strings"
"sync"
"github.com/influxdb/influxdb/meta"
"github.com/influxdb/influxdb/tsdb"
)
// MaxMessageSize defines how large a message can be before we reject it
const MaxMessageSize = 1024 * 1024 * 1024 // 1GB
// MuxHeader is the header byte used in the TCP mux.
const MuxHeader = 2
// Service processes data received over raw TCP connections.
type Service struct {
mu sync.RWMutex
wg sync.WaitGroup
closing chan struct{}
Listener net.Listener
MetaStore interface {
ShardOwner(shardID uint64) (string, string, *meta.ShardGroupInfo)
}
TSDBStore interface {
CreateShard(database, policy string, shardID uint64) error
WriteToShard(shardID uint64, points []tsdb.Point) error
CreateMapper(shardID uint64, query string, chunkSize int) (tsdb.Mapper, error)
}
Logger *log.Logger
}
// NewService returns a new instance of Service.
func NewService(c Config) *Service {
return &Service{
closing: make(chan struct{}),
Logger: log.New(os.Stderr, "[tcp] ", log.LstdFlags),
}
}
// Open opens the network listener and begins serving requests.
func (s *Service) Open() error {
s.Logger.Println("Starting cluster service")
// Begin serving conections.
s.wg.Add(1)
go s.serve()
return nil
}
// SetLogger sets the internal logger to the logger passed in.
func (s *Service) SetLogger(l *log.Logger) {
s.Logger = l
}
// serve accepts connections from the listener and handles them.
func (s *Service) serve() {
defer s.wg.Done()
for {
// Check if the service is shutting down.
select {
case <-s.closing:
return
default:
}
// Accept the next connection.
conn, err := s.Listener.Accept()
if err != nil {
if strings.Contains(err.Error(), "connection closed") {
s.Logger.Printf("cluster service accept error: %s", err)
return
}
s.Logger.Printf("accept error: %s", err)
continue
}
// Delegate connection handling to a separate goroutine.
s.wg.Add(1)
go func() {
defer s.wg.Done()
s.handleConn(conn)
}()
}
}
// Close shuts down the listener and waits for all connections to finish.
func (s *Service) Close() error {
if s.Listener != nil {
s.Listener.Close()
}
// Shut down all handlers.
close(s.closing)
// s.wg.Wait() // FIXME(benbjohnson)
return nil
}
// handleConn services an individual TCP connection.
func (s *Service) handleConn(conn net.Conn) {
// Ensure connection is closed when service is closed.
closing := make(chan struct{})
defer close(closing)
go func() {
select {
case <-closing:
case <-s.closing:
}
conn.Close()
}()
s.Logger.Printf("accept remote write connection from %v\n", conn.RemoteAddr())
defer func() {
s.Logger.Printf("close remote write connection from %v\n", conn.RemoteAddr())
}()
for {
// Read type-length-value.
typ, buf, err := ReadTLV(conn)
if err != nil {
if strings.HasSuffix(err.Error(), "EOF") {
return
}
s.Logger.Printf("unable to read type-length-value %s", err)
return
}
// Delegate message processing by type.
switch typ {
case writeShardRequestMessage:
err := s.processWriteShardRequest(buf)
if err != nil {
s.Logger.Printf("process write shard error: %s", err)
}
s.writeShardResponse(conn, err)
case mapShardRequestMessage:
err := s.processMapShardRequest(conn, buf)
if err != nil {
s.Logger.Printf("process map shard error: %s", err)
if err := writeMapShardResponseMessage(conn, NewMapShardResponse(1, err.Error())); err != nil {
s.Logger.Printf("process map shard error writing response: %s", err.Error())
}
}
default:
s.Logger.Printf("cluster service message type not found: %d", typ)
}
}
}
func (s *Service) processWriteShardRequest(buf []byte) error {
// Build request
var req WriteShardRequest
if err := req.UnmarshalBinary(buf); err != nil {
return err
}
err := s.TSDBStore.WriteToShard(req.ShardID(), req.Points())
// We may have received a write for a shard that we don't have locally because the
// sending node may have just created the shard (via the metastore) and the write
// arrived before the local store could create the shard. In this case, we need
// to check the metastore to determine what database and retention policy this
// shard should reside within.
if err == tsdb.ErrShardNotFound {
// Query the metastore for the owner of this shard
database, retentionPolicy, sgi := s.MetaStore.ShardOwner(req.ShardID())
if sgi == nil {
// If we can't find it, then we need to drop this request
// as it is no longer valid. This could happen if writes were queued via
// hinted handoff and delivered after a shard group was deleted.
s.Logger.Printf("drop write request: shard=%d", req.ShardID())
return nil
}
err = s.TSDBStore.CreateShard(database, retentionPolicy, req.ShardID())
if err != nil {
return err
}
return s.TSDBStore.WriteToShard(req.ShardID(), req.Points())
}
if err != nil {
return fmt.Errorf("write shard %d: %s", req.ShardID(), err)
}
return nil
}
func (s *Service) writeShardResponse(w io.Writer, e error) {
// Build response.
var resp WriteShardResponse
if e != nil {
resp.SetCode(1)
resp.SetMessage(e.Error())
} else {
resp.SetCode(0)
}
// Marshal response to binary.
buf, err := resp.MarshalBinary()
if err != nil {
s.Logger.Printf("error marshalling shard response: %s", err)
return
}
// Write to connection.
if err := WriteTLV(w, writeShardResponseMessage, buf); err != nil {
s.Logger.Printf("write shard response error: %s", err)
}
}
func (s *Service) processMapShardRequest(w io.Writer, buf []byte) error {
// Decode request
var req MapShardRequest
if err := req.UnmarshalBinary(buf); err != nil {
return err
}
m, err := s.TSDBStore.CreateMapper(req.ShardID(), req.Query(), int(req.ChunkSize()))
if err != nil {
return fmt.Errorf("create mapper: %s", err)
}
if m == nil {
return writeMapShardResponseMessage(w, NewMapShardResponse(0, ""))
}
if err := m.Open(); err != nil {
return fmt.Errorf("mapper open: %s", err)
}
defer m.Close()
var metaSent bool
for {
var resp MapShardResponse
if !metaSent {
resp.SetTagSets(m.TagSets())
resp.SetFields(m.Fields())
metaSent = true
}
chunk, err := m.NextChunk()
if err != nil {
return fmt.Errorf("next chunk: %s", err)
}
if chunk != nil {
b, err := json.Marshal(chunk)
if err != nil {
return fmt.Errorf("encoding: %s", err)
}
resp.SetData(b)
}
// Write to connection.
resp.SetCode(0)
if err := writeMapShardResponseMessage(w, &resp); err != nil {
return err
}
if chunk == nil {
// All mapper data sent.
return nil
}
}
}
func writeMapShardResponseMessage(w io.Writer, msg *MapShardResponse) error {
buf, err := msg.MarshalBinary()
if err != nil {
return err
}
return WriteTLV(w, mapShardResponseMessage, buf)
}
// ReadTLV reads a type-length-value record from r.
func ReadTLV(r io.Reader) (byte, []byte, error) {
var typ [1]byte
if _, err := io.ReadFull(r, typ[:]); err != nil {
return 0, nil, fmt.Errorf("read message type: %s", err)
}
// Read the size of the message.
var sz int64
if err := binary.Read(r, binary.BigEndian, &sz); err != nil {
return 0, nil, fmt.Errorf("read message size: %s", err)
}
if sz == 0 {
return 0, nil, fmt.Errorf("invalid message size: %d", sz)
}
if sz >= MaxMessageSize {
return 0, nil, fmt.Errorf("max message size of %d exceeded: %d", MaxMessageSize, sz)
}
// Read the value.
buf := make([]byte, sz)
if _, err := io.ReadFull(r, buf); err != nil {
return 0, nil, fmt.Errorf("read message value: %s", err)
}
return typ[0], buf, nil
}
// WriteTLV writes a type-length-value record to w.
func WriteTLV(w io.Writer, typ byte, buf []byte) error {
if _, err := w.Write([]byte{typ}); err != nil {
return fmt.Errorf("write message type: %s", err)
}
// Write the size of the message.
if err := binary.Write(w, binary.BigEndian, int64(len(buf))); err != nil {
return fmt.Errorf("write message size: %s", err)
}
// Write the value.
if _, err := w.Write(buf); err != nil {
return fmt.Errorf("write message value: %s", err)
}
return nil
}

103
Godeps/_workspace/src/github.com/influxdb/influxdb/cluster/service_test.go generated vendored
View File

@ -1,103 +0,0 @@
package cluster_test
import (
"fmt"
"net"
"time"
"github.com/influxdb/influxdb/cluster"
"github.com/influxdb/influxdb/meta"
"github.com/influxdb/influxdb/tcp"
"github.com/influxdb/influxdb/tsdb"
)
type metaStore struct {
host string
}
func (m *metaStore) Node(nodeID uint64) (*meta.NodeInfo, error) {
return &meta.NodeInfo{
ID: nodeID,
Host: m.host,
}, nil
}
type testService struct {
nodeID uint64
ln net.Listener
muxln net.Listener
writeShardFunc func(shardID uint64, points []tsdb.Point) error
createShardFunc func(database, policy string, shardID uint64) error
createMapperFunc func(shardID uint64, query string, chunkSize int) (tsdb.Mapper, error)
}
func newTestWriteService(f func(shardID uint64, points []tsdb.Point) error) testService {
ln, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
panic(err)
}
mux := tcp.NewMux()
muxln := mux.Listen(cluster.MuxHeader)
go mux.Serve(ln)
return testService{
writeShardFunc: f,
ln: ln,
muxln: muxln,
}
}
func (ts *testService) Close() {
if ts.ln != nil {
ts.ln.Close()
}
}
type serviceResponses []serviceResponse
type serviceResponse struct {
shardID uint64
ownerID uint64
points []tsdb.Point
}
func (t testService) WriteToShard(shardID uint64, points []tsdb.Point) error {
return t.writeShardFunc(shardID, points)
}
func (t testService) CreateShard(database, policy string, shardID uint64) error {
return t.createShardFunc(database, policy, shardID)
}
func (t testService) CreateMapper(shardID uint64, query string, chunkSize int) (tsdb.Mapper, error) {
return t.createMapperFunc(shardID, query, chunkSize)
}
func writeShardSuccess(shardID uint64, points []tsdb.Point) error {
responses <- &serviceResponse{
shardID: shardID,
points: points,
}
return nil
}
func writeShardFail(shardID uint64, points []tsdb.Point) error {
return fmt.Errorf("failed to write")
}
var responses = make(chan *serviceResponse, 1024)
func (testService) ResponseN(n int) ([]*serviceResponse, error) {
var a []*serviceResponse
for {
select {
case r := <-responses:
a = append(a, r)
if len(a) == n {
return a, nil
}
case <-time.After(time.Second):
return a, fmt.Errorf("unexpected response count: expected: %d, actual: %d", n, len(a))
}
}
}

207
Godeps/_workspace/src/github.com/influxdb/influxdb/cluster/shard_mapper.go generated vendored
View File

@ -1,207 +0,0 @@
package cluster
import (
"encoding/json"
"fmt"
"io"
"math/rand"
"net"
"time"
"github.com/influxdb/influxdb/meta"
"github.com/influxdb/influxdb/tsdb"
"gopkg.in/fatih/pool.v2"
)
// ShardMapper is responsible for providing mappers for requested shards. It is
// responsible for creating those mappers from the local store, or reaching
// out to another node on the cluster.
type ShardMapper struct {
ForceRemoteMapping bool // All shards treated as remote. Useful for testing.
MetaStore interface {
NodeID() uint64
Node(id uint64) (ni *meta.NodeInfo, err error)
}
TSDBStore interface {
CreateMapper(shardID uint64, query string, chunkSize int) (tsdb.Mapper, error)
}
timeout time.Duration
pool *clientPool
}
// NewShardMapper returns a mapper of local and remote shards.
func NewShardMapper(timeout time.Duration) *ShardMapper {
return &ShardMapper{
pool: newClientPool(),
timeout: timeout,
}
}
// CreateMapper returns a Mapper for the given shard ID.
func (s *ShardMapper) CreateMapper(sh meta.ShardInfo, stmt string, chunkSize int) (tsdb.Mapper, error) {
var err error
var m tsdb.Mapper
if sh.OwnedBy(s.MetaStore.NodeID()) && !s.ForceRemoteMapping {
m, err = s.TSDBStore.CreateMapper(sh.ID, stmt, chunkSize)
if err != nil {
return nil, err
}
} else {
// Pick a node in a pseudo-random manner.
conn, err := s.dial(sh.OwnerIDs[rand.Intn(len(sh.OwnerIDs))])
if err != nil {
return nil, err
}
conn.SetDeadline(time.Now().Add(s.timeout))
rm := NewRemoteMapper(conn.(*pool.PoolConn), sh.ID, stmt, chunkSize)
m = rm
}
return m, nil
}
func (s *ShardMapper) dial(nodeID uint64) (net.Conn, error) {
// If we don't have a connection pool for that addr yet, create one
_, ok := s.pool.getPool(nodeID)
if !ok {
factory := &connFactory{nodeID: nodeID, clientPool: s.pool, timeout: s.timeout}
factory.metaStore = s.MetaStore
p, err := pool.NewChannelPool(1, 3, factory.dial)
if err != nil {
return nil, err
}
s.pool.setPool(nodeID, p)
}
return s.pool.conn(nodeID)
}
type remoteShardConn interface {
io.ReadWriter
Close() error
MarkUnusable()
}
// RemoteMapper implements the tsdb.Mapper interface. It connects to a remote node,
// sends a query, and interprets the stream of data that comes back.
type RemoteMapper struct {
shardID uint64
stmt string
chunkSize int
tagsets []string
fields []string
conn remoteShardConn
bufferedResponse *MapShardResponse
}
// NewRemoteMapper returns a new remote mapper using the given connection.
func NewRemoteMapper(c remoteShardConn, shardID uint64, stmt string, chunkSize int) *RemoteMapper {
return &RemoteMapper{
conn: c,
shardID: shardID,
stmt: stmt,
chunkSize: chunkSize,
}
}
// Open connects to the remote node and starts receiving data.
func (r *RemoteMapper) Open() (err error) {
defer func() {
if err != nil {
r.conn.Close()
}
}()
// Build Map request.
var request MapShardRequest
request.SetShardID(r.shardID)
request.SetQuery(r.stmt)
request.SetChunkSize(int32(r.chunkSize))
// Marshal into protocol buffers.
buf, err := request.MarshalBinary()
if err != nil {
return err
}
// Write request.
if err := WriteTLV(r.conn, mapShardRequestMessage, buf); err != nil {
r.conn.MarkUnusable()
return err
}
// Read the response.
_, buf, err = ReadTLV(r.conn)
if err != nil {
r.conn.MarkUnusable()
return err
}
// Unmarshal response.
r.bufferedResponse = &MapShardResponse{}
if err := r.bufferedResponse.UnmarshalBinary(buf); err != nil {
return err
}
if r.bufferedResponse.Code() != 0 {
return fmt.Errorf("error code %d: %s", r.bufferedResponse.Code(), r.bufferedResponse.Message())
}
// Decode the first response to get the TagSets.
r.tagsets = r.bufferedResponse.TagSets()
return nil
}
func (r *RemoteMapper) TagSets() []string {
return r.tagsets
}
func (r *RemoteMapper) Fields() []string {
return r.fields
}
// NextChunk returns the next chunk read from the remote node to the client.
func (r *RemoteMapper) NextChunk() (chunk interface{}, err error) {
output := &tsdb.MapperOutput{}
var response *MapShardResponse
if r.bufferedResponse != nil {
response = r.bufferedResponse
r.bufferedResponse = nil
} else {
response = &MapShardResponse{}
// Read the response.
_, buf, err := ReadTLV(r.conn)
if err != nil {
r.conn.MarkUnusable()
return nil, err
}
// Unmarshal response.
if err := response.UnmarshalBinary(buf); err != nil {
return nil, err
}
if response.Code() != 0 {
return nil, fmt.Errorf("error code %d: %s", response.Code(), response.Message())
}
}
if response.Data() == nil {
return nil, nil
}
err = json.Unmarshal(response.Data(), output)
return output, err
}
// Close the Mapper
func (r *RemoteMapper) Close() {
r.conn.Close()
}

96
Godeps/_workspace/src/github.com/influxdb/influxdb/cluster/shard_mapper_test.go generated vendored
View File

@ -1,96 +0,0 @@
package cluster
import (
"bytes"
"encoding/json"
"io"
"testing"
"github.com/influxdb/influxdb/tsdb"
)
// remoteShardResponder implements the remoteShardConn interface.
type remoteShardResponder struct {
t *testing.T
rxBytes []byte
buffer *bytes.Buffer
}
func newRemoteShardResponder(outputs []*tsdb.MapperOutput, tagsets []string) *remoteShardResponder {
r := &remoteShardResponder{}
a := make([]byte, 0, 1024)
r.buffer = bytes.NewBuffer(a)
// Pump the outputs in the buffer for later reading.
for _, o := range outputs {
resp := &MapShardResponse{}
resp.SetCode(0)
if o != nil {
d, _ := json.Marshal(o)
resp.SetData(d)
resp.SetTagSets(tagsets)
}
g, _ := resp.MarshalBinary()
WriteTLV(r.buffer, mapShardResponseMessage, g)
}
return r
}
func (r remoteShardResponder) MarkUnusable() { return }
func (r remoteShardResponder) Close() error { return nil }
func (r remoteShardResponder) Read(p []byte) (n int, err error) {
return io.ReadFull(r.buffer, p)
}
func (r remoteShardResponder) Write(p []byte) (n int, err error) {
if r.rxBytes == nil {
r.rxBytes = make([]byte, 0)
}
r.rxBytes = append(r.rxBytes, p...)
return len(p), nil
}
// Ensure a RemoteMapper can process valid responses from a remote shard.
func TestShardWriter_RemoteMapper_Success(t *testing.T) {
expTagSets := []string{"tagsetA"}
expOutput := &tsdb.MapperOutput{
Name: "cpu",
Tags: map[string]string{"host": "serverA"},
}
c := newRemoteShardResponder([]*tsdb.MapperOutput{expOutput, nil}, expTagSets)
r := NewRemoteMapper(c, 1234, "SELECT * FROM CPU", 10)
if err := r.Open(); err != nil {
t.Fatalf("failed to open remote mapper: %s", err.Error())
}
if r.TagSets()[0] != expTagSets[0] {
t.Fatalf("incorrect tagsets received, exp %v, got %v", expTagSets, r.TagSets())
}
// Get first chunk from mapper.
chunk, err := r.NextChunk()
if err != nil {
t.Fatalf("failed to get next chunk from mapper: %s", err.Error())
}
output, ok := chunk.(*tsdb.MapperOutput)
if !ok {
t.Fatal("chunk is not of expected type")
}
if output.Name != "cpu" {
t.Fatalf("received output incorrect, exp: %v, got %v", expOutput, output)
}
// Next chunk should be nil, indicating no more data.
chunk, err = r.NextChunk()
if err != nil {
t.Fatalf("failed to get next chunk from mapper: %s", err.Error())
}
if chunk != nil {
t.Fatal("received more chunks when none expected")
}
}

163
Godeps/_workspace/src/github.com/influxdb/influxdb/cluster/shard_writer.go generated vendored
View File

@ -1,163 +0,0 @@
package cluster
import (
"fmt"
"net"
"time"
"github.com/influxdb/influxdb/meta"
"github.com/influxdb/influxdb/tsdb"
"gopkg.in/fatih/pool.v2"
)
const (
writeShardRequestMessage byte = iota + 1
writeShardResponseMessage
mapShardRequestMessage
mapShardResponseMessage
)
// ShardWriter writes a set of points to a shard.
type ShardWriter struct {
pool *clientPool
timeout time.Duration
MetaStore interface {
Node(id uint64) (ni *meta.NodeInfo, err error)
}
}
// NewShardWriter returns a new instance of ShardWriter.
func NewShardWriter(timeout time.Duration) *ShardWriter {
return &ShardWriter{
pool: newClientPool(),
timeout: timeout,
}
}
func (w *ShardWriter) WriteShard(shardID, ownerID uint64, points []tsdb.Point) error {
c, err := w.dial(ownerID)
if err != nil {
return err
}
conn, ok := c.(*pool.PoolConn)
if !ok {
panic("wrong connection type")
}
defer func(conn net.Conn) {
conn.Close() // return to pool
}(conn)
// Build write request.
var request WriteShardRequest
request.SetShardID(shardID)
request.AddPoints(points)
// Marshal into protocol buffers.
buf, err := request.MarshalBinary()
if err != nil {
return err
}
// Write request.
conn.SetWriteDeadline(time.Now().Add(w.timeout))
if err := WriteTLV(conn, writeShardRequestMessage, buf); err != nil {
conn.MarkUnusable()
return err
}
// Read the response.
conn.SetReadDeadline(time.Now().Add(w.timeout))
_, buf, err = ReadTLV(conn)
if err != nil {
conn.MarkUnusable()
return err
}
// Unmarshal response.
var response WriteShardResponse
if err := response.UnmarshalBinary(buf); err != nil {
return err
}
if response.Code() != 0 {
return fmt.Errorf("error code %d: %s", response.Code(), response.Message())
}
return nil
}
func (c *ShardWriter) dial(nodeID uint64) (net.Conn, error) {
// If we don't have a connection pool for that addr yet, create one
_, ok := c.pool.getPool(nodeID)
if !ok {
factory := &connFactory{nodeID: nodeID, clientPool: c.pool, timeout: c.timeout}
factory.metaStore = c.MetaStore
p, err := pool.NewChannelPool(1, 3, factory.dial)
if err != nil {
return nil, err
}
c.pool.setPool(nodeID, p)
}
return c.pool.conn(nodeID)
}
func (w *ShardWriter) Close() error {
if w.pool == nil {
return fmt.Errorf("client already closed")
}
w.pool.close()
w.pool = nil
return nil
}
const (
maxConnections = 500
maxRetries = 3
)
var errMaxConnectionsExceeded = fmt.Errorf("can not exceed max connections of %d", maxConnections)
type connFactory struct {
nodeID uint64
timeout time.Duration
clientPool interface {
size() int
}
metaStore interface {
Node(id uint64) (ni *meta.NodeInfo, err error)
}
}
func (c *connFactory) dial() (net.Conn, error) {
if c.clientPool.size() > maxConnections {
return nil, errMaxConnectionsExceeded
}
ni, err := c.metaStore.Node(c.nodeID)
if err != nil {
return nil, err
}
if ni == nil {
return nil, fmt.Errorf("node %d does not exist", c.nodeID)
}
conn, err := net.DialTimeout("tcp", ni.Host, c.timeout)
if err != nil {
return nil, err
}
// Write a marker byte for cluster messages.
_, err = conn.Write([]byte{MuxHeader})
if err != nil {
conn.Close()
return nil, err
}
return conn, nil
}

186
Godeps/_workspace/src/github.com/influxdb/influxdb/cluster/shard_writer_test.go generated vendored
View File

@ -1,186 +0,0 @@
package cluster_test
import (
"net"
"strings"
"testing"
"time"
"github.com/influxdb/influxdb/cluster"
"github.com/influxdb/influxdb/tsdb"
)
// Ensure the shard writer can successful write a single request.
func TestShardWriter_WriteShard_Success(t *testing.T) {
ts := newTestWriteService(writeShardSuccess)
s := cluster.NewService(cluster.Config{})
s.Listener = ts.muxln
s.TSDBStore = ts
if err := s.Open(); err != nil {
t.Fatal(err)
}
defer s.Close()
defer ts.Close()
w := cluster.NewShardWriter(time.Minute)
w.MetaStore = &metaStore{host: ts.ln.Addr().String()}
// Build a single point.
now := time.Now()
var points []tsdb.Point
points = append(points, tsdb.NewPoint("cpu", tsdb.Tags{"host": "server01"}, map[string]interface{}{"value": int64(100)}, now))
// Write to shard and close.
if err := w.WriteShard(1, 2, points); err != nil {
t.Fatal(err)
} else if err := w.Close(); err != nil {
t.Fatal(err)
}
// Validate response.
responses, err := ts.ResponseN(1)
if err != nil {
t.Fatal(err)
} else if responses[0].shardID != 1 {
t.Fatalf("unexpected shard id: %d", responses[0].shardID)
}
// Validate point.
if p := responses[0].points[0]; p.Name() != "cpu" {
t.Fatalf("unexpected name: %s", p.Name())
} else if p.Fields()["value"] != int64(100) {
t.Fatalf("unexpected 'value' field: %d", p.Fields()["value"])
} else if p.Tags()["host"] != "server01" {
t.Fatalf("unexpected 'host' tag: %s", p.Tags()["host"])
} else if p.Time().UnixNano() != now.UnixNano() {
t.Fatalf("unexpected time: %s", p.Time())
}
}
// Ensure the shard writer can successful write a multiple requests.
func TestShardWriter_WriteShard_Multiple(t *testing.T) {
ts := newTestWriteService(writeShardSuccess)
s := cluster.NewService(cluster.Config{})
s.Listener = ts.muxln
s.TSDBStore = ts
if err := s.Open(); err != nil {
t.Fatal(err)
}
defer s.Close()
defer ts.Close()
w := cluster.NewShardWriter(time.Minute)
w.MetaStore = &metaStore{host: ts.ln.Addr().String()}
// Build a single point.
now := time.Now()
var points []tsdb.Point
points = append(points, tsdb.NewPoint("cpu", tsdb.Tags{"host": "server01"}, map[string]interface{}{"value": int64(100)}, now))
// Write to shard twice and close.
if err := w.WriteShard(1, 2, points); err != nil {
t.Fatal(err)
} else if err := w.WriteShard(1, 2, points); err != nil {
t.Fatal(err)
} else if err := w.Close(); err != nil {
t.Fatal(err)
}
// Validate response.
responses, err := ts.ResponseN(1)
if err != nil {
t.Fatal(err)
} else if responses[0].shardID != 1 {
t.Fatalf("unexpected shard id: %d", responses[0].shardID)
}
// Validate point.
if p := responses[0].points[0]; p.Name() != "cpu" {
t.Fatalf("unexpected name: %s", p.Name())
} else if p.Fields()["value"] != int64(100) {
t.Fatalf("unexpected 'value' field: %d", p.Fields()["value"])
} else if p.Tags()["host"] != "server01" {
t.Fatalf("unexpected 'host' tag: %s", p.Tags()["host"])
} else if p.Time().UnixNano() != now.UnixNano() {
t.Fatalf("unexpected time: %s", p.Time())
}
}
// Ensure the shard writer returns an error when the server fails to accept the write.
func TestShardWriter_WriteShard_Error(t *testing.T) {
ts := newTestWriteService(writeShardFail)
s := cluster.NewService(cluster.Config{})
s.Listener = ts.muxln
s.TSDBStore = ts
if err := s.Open(); err != nil {
t.Fatal(err)
}
defer s.Close()
defer ts.Close()
w := cluster.NewShardWriter(time.Minute)
w.MetaStore = &metaStore{host: ts.ln.Addr().String()}
now := time.Now()
shardID := uint64(1)
ownerID := uint64(2)
var points []tsdb.Point
points = append(points, tsdb.NewPoint(
"cpu", tsdb.Tags{"host": "server01"}, map[string]interface{}{"value": int64(100)}, now,
))
if err := w.WriteShard(shardID, ownerID, points); err == nil || err.Error() != "error code 1: write shard 1: failed to write" {
t.Fatalf("unexpected error: %v", err)
}
}
// Ensure the shard writer returns an error when dialing times out.
func TestShardWriter_Write_ErrDialTimeout(t *testing.T) {
ts := newTestWriteService(writeShardSuccess)
s := cluster.NewService(cluster.Config{})
s.Listener = ts.muxln
s.TSDBStore = ts
if err := s.Open(); err != nil {
t.Fatal(err)
}
defer s.Close()
defer ts.Close()
w := cluster.NewShardWriter(time.Nanosecond)
w.MetaStore = &metaStore{host: ts.ln.Addr().String()}
now := time.Now()
shardID := uint64(1)
ownerID := uint64(2)
var points []tsdb.Point
points = append(points, tsdb.NewPoint(
"cpu", tsdb.Tags{"host": "server01"}, map[string]interface{}{"value": int64(100)}, now,
))
if err, exp := w.WriteShard(shardID, ownerID, points), "i/o timeout"; err == nil || !strings.Contains(err.Error(), exp) {
t.Fatalf("expected error %v, to contain %s", err, exp)
}
}
// Ensure the shard writer returns an error when reading times out.
func TestShardWriter_Write_ErrReadTimeout(t *testing.T) {
ln, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
w := cluster.NewShardWriter(time.Millisecond)
w.MetaStore = &metaStore{host: ln.Addr().String()}
now := time.Now()
shardID := uint64(1)
ownerID := uint64(2)
var points []tsdb.Point
points = append(points, tsdb.NewPoint(
"cpu", tsdb.Tags{"host": "server01"}, map[string]interface{}{"value": int64(100)}, now,
))
if err := w.WriteShard(shardID, ownerID, points); err == nil || !strings.Contains(err.Error(), "i/o timeout") {
t.Fatalf("unexpected error: %s", err)
}
}

724
Godeps/_workspace/src/github.com/influxdb/influxdb/cmd/influx/main.go generated vendored
View File

@ -1,724 +0,0 @@
package main
import (
"encoding/csv"
"encoding/json"
"flag"
"fmt"
"io"
"net"
"net/url"
"os"
"os/user"
"path/filepath"
"sort"
"strconv"
"strings"
"text/tabwriter"
"github.com/influxdb/influxdb/client"
"github.com/influxdb/influxdb/importer/v8"
"github.com/peterh/liner"
)
// These variables are populated via the Go linker.
var (
version string = "0.9"
)
const (
// defaultFormat is the default format of the results when issuing queries
defaultFormat = "column"
// defaultPPS is the default points per second that the import will throttle at
// by default it's 0, which means it will not throttle
defaultPPS = 0
)
type CommandLine struct {
Client *client.Client
Line *liner.State
Host string
Port int
Username string
Password string
Database string
Ssl bool
RetentionPolicy string
Version string
Pretty bool // controls pretty print for json
Format string // controls the output format. Valid values are json, csv, or column
Execute string
ShowVersion bool
Import bool
PPS int // Controls how many points per second the import will allow via throttling
Path string
Compressed bool
}
func main() {
c := CommandLine{}
fs := flag.NewFlagSet("InfluxDB shell version "+version, flag.ExitOnError)
fs.StringVar(&c.Host, "host", client.DefaultHost, "Influxdb host to connect to.")
fs.IntVar(&c.Port, "port", client.DefaultPort, "Influxdb port to connect to.")
fs.StringVar(&c.Username, "username", c.Username, "Username to connect to the server.")
fs.StringVar(&c.Password, "password", c.Password, `Password to connect to the server. Leaving blank will prompt for password (--password="").`)
fs.StringVar(&c.Database, "database", c.Database, "Database to connect to the server.")
fs.BoolVar(&c.Ssl, "ssl", false, "Use https for connecting to cluster.")
fs.StringVar(&c.Format, "format", defaultFormat, "Format specifies the format of the server responses: json, csv, or column.")
fs.BoolVar(&c.Pretty, "pretty", false, "Turns on pretty print for the json format.")
fs.StringVar(&c.Execute, "execute", c.Execute, "Execute command and quit.")
fs.BoolVar(&c.ShowVersion, "version", false, "Displays the InfluxDB version.")
fs.BoolVar(&c.Import, "import", false, "Import a previous database.")
fs.IntVar(&c.PPS, "pps", defaultPPS, "How many points per second the import will allow. By default it is zero and will not throttle importing.")
fs.StringVar(&c.Path, "path", "", "path to the file to import")
fs.BoolVar(&c.Compressed, "compressed", false, "set to true if the import file is compressed")
// Define our own custom usage to print
fs.Usage = func() {
fmt.Println(`Usage of influx:
-version
Display the version and exit.
-host 'host name'
Host to connect to.
-port 'port #'
Port to connect to.
-database 'database name'
Database to connect to the server.
-password 'password'
Password to connect to the server. Leaving blank will prompt for password (--password '').
-username 'username'
Username to connect to the server.
-ssl
Use https for requests.
-execute 'command'
Execute command and quit.
-format 'json|csv|column'
Format specifies the format of the server responses: json, csv, or column.
-pretty
Turns on pretty print for the json format.
-import
Import a previous database export from file
-pps
How many points per second the import will allow. By default it is zero and will not throttle importing.
-path
Path to file to import
-compressed
Set to true if the import file is compressed
Examples:
# Use influx in a non-interactive mode to query the database "metrics" and pretty print json:
$ influx -database 'metrics' -execute 'select * from cpu' -format 'json' -pretty
# Connect to a specific database on startup and set database context:
$ influx -database 'metrics' -host 'localhost' -port '8086'
`)
}
fs.Parse(os.Args[1:])
if c.ShowVersion {
showVersion()
os.Exit(0)
}
var promptForPassword bool
// determine if they set the password flag but provided no value
for _, v := range os.Args {
v = strings.ToLower(v)
if (strings.HasPrefix(v, "-password") || strings.HasPrefix(v, "--password")) && c.Password == "" {
promptForPassword = true
break
}
}
c.Line = liner.NewLiner()
defer c.Line.Close()
if promptForPassword {
p, e := c.Line.PasswordPrompt("password: ")
if e != nil {
fmt.Println("Unable to parse password.")
} else {
c.Password = p
}
}
if err := c.connect(""); err != nil {
}
if c.Execute == "" && !c.Import {
fmt.Printf("Connected to %s version %s\n", c.Client.Addr(), c.Version)
}
if c.Execute != "" {
if err := c.ExecuteQuery(c.Execute); err != nil {
c.Line.Close()
os.Exit(1)
}
c.Line.Close()
os.Exit(0)
}
if c.Import {
path := net.JoinHostPort(c.Host, strconv.Itoa(c.Port))
u, e := client.ParseConnectionString(path, c.Ssl)
if e != nil {
fmt.Println(e)
return
}
config := v8.NewConfig()
config.Username = c.Username
config.Password = c.Password
config.Precision = "ns"
config.WriteConsistency = "any"
config.Path = c.Path
config.Version = version
config.URL = u
config.Compressed = c.Compressed
config.PPS = c.PPS
i := v8.NewImporter(config)
if err := i.Import(); err != nil {
fmt.Printf("ERROR: %s\n", err)
c.Line.Close()
os.Exit(1)
}
c.Line.Close()
os.Exit(0)
}
showVersion()
var historyFile string
usr, err := user.Current()
// Only load history if we can get the user
if err == nil {
historyFile = filepath.Join(usr.HomeDir, ".influx_history")
if f, err := os.Open(historyFile); err == nil {
c.Line.ReadHistory(f)
f.Close()
}
}
for {
l, e := c.Line.Prompt("> ")
if e != nil {
break
}
if c.ParseCommand(l) {
// write out the history
if len(historyFile) > 0 {
c.Line.AppendHistory(l)
if f, err := os.Create(historyFile); err == nil {
c.Line.WriteHistory(f)
f.Close()
}
}
} else {
break // exit main loop
}
}
}
func showVersion() {
fmt.Println("InfluxDB shell " + version)
}
func (c *CommandLine) ParseCommand(cmd string) bool {
lcmd := strings.TrimSpace(strings.ToLower(cmd))
switch {
case strings.HasPrefix(lcmd, "exit"):
// signal the program to exit
return false
case strings.HasPrefix(lcmd, "gopher"):
c.gopher()
case strings.HasPrefix(lcmd, "connect"):
c.connect(cmd)
case strings.HasPrefix(lcmd, "auth"):
c.SetAuth(cmd)
case strings.HasPrefix(lcmd, "help"):
c.help()
case strings.HasPrefix(lcmd, "format"):
c.SetFormat(cmd)
case strings.HasPrefix(lcmd, "settings"):
c.Settings()
case strings.HasPrefix(lcmd, "pretty"):
c.Pretty = !c.Pretty
if c.Pretty {
fmt.Println("Pretty print enabled")
} else {
fmt.Println("Pretty print disabled")
}
case strings.HasPrefix(lcmd, "use"):
c.use(cmd)
case strings.HasPrefix(lcmd, "insert"):
c.Insert(cmd)
case lcmd == "":
break
default:
c.ExecuteQuery(cmd)
}
return true
}
func (c *CommandLine) connect(cmd string) error {
var cl *client.Client
var u url.URL
// Remove the "connect" keyword if it exists
path := strings.TrimSpace(strings.Replace(cmd, "connect", "", -1))
// If they didn't provide a connection string, use the current settings
if path == "" {
path = net.JoinHostPort(c.Host, strconv.Itoa(c.Port))
}
var e error
u, e = client.ParseConnectionString(path, c.Ssl)
if e != nil {
return e
}
config := client.NewConfig()
config.URL = u
config.Username = c.Username
config.Password = c.Password
config.UserAgent = "InfluxDBShell/" + version
cl, err := client.NewClient(config)
if err != nil {
return fmt.Errorf("Could not create client %s", err)
}
c.Client = cl
if _, v, e := c.Client.Ping(); e != nil {
return fmt.Errorf("Failed to connect to %s\n", c.Client.Addr())
} else {
c.Version = v
}
return nil
}
func (c *CommandLine) SetAuth(cmd string) {
// If they pass in the entire command, we should parse it
// auth <username> <password>
args := strings.Fields(cmd)
if len(args) == 3 {
args = args[1:]
} else {
args = []string{}
}
if len(args) == 2 {
c.Username = args[0]
c.Password = args[1]
} else {
u, e := c.Line.Prompt("username: ")
if e != nil {
fmt.Printf("Unable to process input: %s", e)
return
}
c.Username = strings.TrimSpace(u)
p, e := c.Line.PasswordPrompt("password: ")
if e != nil {
fmt.Printf("Unable to process input: %s", e)
return
}
c.Password = p
}
// Update the client as well
c.Client.SetAuth(c.Username, c.Password)
}
func (c *CommandLine) use(cmd string) {
args := strings.Split(strings.TrimSuffix(strings.TrimSpace(cmd), ";"), " ")
if len(args) != 2 {
fmt.Printf("Could not parse database name from %q.\n", cmd)
return
}
d := args[1]
c.Database = d
fmt.Printf("Using database %s\n", d)
}
func (c *CommandLine) SetFormat(cmd string) {
// Remove the "format" keyword if it exists
cmd = strings.TrimSpace(strings.Replace(cmd, "format", "", -1))
// normalize cmd
cmd = strings.ToLower(cmd)
switch cmd {
case "json", "csv", "column":
c.Format = cmd
default:
fmt.Printf("Unknown format %q. Please use json, csv, or column.\n", cmd)
}
}
// isWhitespace returns true if the rune is a space, tab, or newline.
func isWhitespace(ch rune) bool { return ch == ' ' || ch == '\t' || ch == '\n' }
// isLetter returns true if the rune is a letter.
func isLetter(ch rune) bool { return (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') }
// isDigit returns true if the rune is a digit.
func isDigit(ch rune) bool { return (ch >= '0' && ch <= '9') }
// isIdentFirstChar returns true if the rune can be used as the first char in an unquoted identifer.
func isIdentFirstChar(ch rune) bool { return isLetter(ch) || ch == '_' }
// isIdentChar returns true if the rune can be used in an unquoted identifier.
func isNotIdentChar(ch rune) bool { return !(isLetter(ch) || isDigit(ch) || ch == '_') }
func parseUnquotedIdentifier(stmt string) (string, string) {
if fields := strings.FieldsFunc(stmt, isNotIdentChar); len(fields) > 0 {
return fields[0], strings.TrimPrefix(stmt, fields[0])
}
return "", stmt
}
func parseDoubleQuotedIdentifier(stmt string) (string, string) {
escapeNext := false
fields := strings.FieldsFunc(stmt, func(ch rune) bool {
if ch == '\\' {
escapeNext = true
} else if ch == '"' {
if !escapeNext {
return true
}
escapeNext = false
}
return false
})
if len(fields) > 0 {
return fields[0], strings.TrimPrefix(stmt, "\""+fields[0]+"\"")
}
return "", stmt
}
func parseNextIdentifier(stmt string) (ident, remainder string) {
if len(stmt) > 0 {
switch {
case isWhitespace(rune(stmt[0])):
return parseNextIdentifier(stmt[1:])
case isIdentFirstChar(rune(stmt[0])):
return parseUnquotedIdentifier(stmt)
case stmt[0] == '"':
return parseDoubleQuotedIdentifier(stmt)
}
}
return "", stmt
}
func (c *CommandLine) parseInto(stmt string) string {
ident, stmt := parseNextIdentifier(stmt)
if strings.HasPrefix(stmt, ".") {
c.Database = ident
fmt.Printf("Using database %s\n", c.Database)
ident, stmt = parseNextIdentifier(stmt[1:])
}
if strings.HasPrefix(stmt, " ") {
c.RetentionPolicy = ident
fmt.Printf("Using retention policy %s\n", c.RetentionPolicy)
return stmt[1:]
}
return stmt
}
func (c *CommandLine) Insert(stmt string) error {
i, point := parseNextIdentifier(stmt)
if !strings.EqualFold(i, "insert") {
fmt.Printf("ERR: found %s, expected INSERT\n", i)
return nil
}
if i, r := parseNextIdentifier(point); strings.EqualFold(i, "into") {
point = c.parseInto(r)
}
_, err := c.Client.Write(client.BatchPoints{
Points: []client.Point{
client.Point{Raw: point},
},
Database: c.Database,
RetentionPolicy: c.RetentionPolicy,
Precision: "n",
WriteConsistency: client.ConsistencyAny,
})
if err != nil {
fmt.Printf("ERR: %s\n", err)
if c.Database == "" {
fmt.Println("Note: error may be due to not setting a database or retention policy.")
fmt.Println(`Please set a database with the command "use <database>" or`)
fmt.Println("INSERT INTO <database>.<retention-policy> <point>")
}
return err
}
return nil
}
func (c *CommandLine) ExecuteQuery(query string) error {
response, err := c.Client.Query(client.Query{Command: query, Database: c.Database})
if err != nil {
fmt.Printf("ERR: %s\n", err)
return err
}
c.FormatResponse(response, os.Stdout)
if err := response.Error(); err != nil {
fmt.Printf("ERR: %s\n", response.Error())
if c.Database == "" {
fmt.Println("Warning: It is possible this error is due to not setting a database.")
fmt.Println(`Please set a database with the command "use <database>".`)
}
return err
}
return nil
}
func (c *CommandLine) FormatResponse(response *client.Response, w io.Writer) {
switch c.Format {
case "json":
c.writeJSON(response, w)
case "csv":
c.writeCSV(response, w)
case "column":
c.writeColumns(response, w)
default:
fmt.Fprintf(w, "Unknown output format %q.\n", c.Format)
}
}
func (c *CommandLine) writeJSON(response *client.Response, w io.Writer) {
var data []byte
var err error
if c.Pretty {
data, err = json.MarshalIndent(response, "", " ")
} else {
data, err = json.Marshal(response)
}
if err != nil {
fmt.Fprintf(w, "Unable to parse json: %s\n", err)
return
}
fmt.Fprintln(w, string(data))
}
func (c *CommandLine) writeCSV(response *client.Response, w io.Writer) {
csvw := csv.NewWriter(w)
for _, result := range response.Results {
// Create a tabbed writer for each result as they won't always line up
rows := c.formatResults(result, "\t")
for _, r := range rows {
csvw.Write(strings.Split(r, "\t"))
}
csvw.Flush()
}
}
func (c *CommandLine) writeColumns(response *client.Response, w io.Writer) {
for _, result := range response.Results {
// Create a tabbed writer for each result a they won't always line up
w := new(tabwriter.Writer)
w.Init(os.Stdout, 0, 8, 1, '\t', 0)
csv := c.formatResults(result, "\t")
for _, r := range csv {
fmt.Fprintln(w, r)
}
w.Flush()
}
}
// formatResults will behave differently if you are formatting for columns or csv
func (c *CommandLine) formatResults(result client.Result, separator string) []string {
rows := []string{}
// Create a tabbed writer for each result a they won't always line up
for i, row := range result.Series {
// gather tags
tags := []string{}
for k, v := range row.Tags {
tags = append(tags, fmt.Sprintf("%s=%s", k, v))
sort.Strings(tags)
}
columnNames := []string{}
// Only put name/tags in a column if format is csv
if c.Format == "csv" {
if len(tags) > 0 {
columnNames = append([]string{"tags"}, columnNames...)
}
if row.Name != "" {
columnNames = append([]string{"name"}, columnNames...)
}
}
for _, column := range row.Columns {
columnNames = append(columnNames, column)
}
// Output a line separator if we have more than one set or results and format is column
if i > 0 && c.Format == "column" {
rows = append(rows, "")
}
// If we are column format, we break out the name/tag to seperate lines
if c.Format == "column" {
if row.Name != "" {
n := fmt.Sprintf("name: %s", row.Name)
rows = append(rows, n)
if len(tags) == 0 {
l := strings.Repeat("-", len(n))
rows = append(rows, l)
}
}
if len(tags) > 0 {
t := fmt.Sprintf("tags: %s", (strings.Join(tags, ", ")))
rows = append(rows, t)
}
}
rows = append(rows, strings.Join(columnNames, separator))
// if format is column, break tags to their own line/format
if c.Format == "column" && len(tags) > 0 {
lines := []string{}
for _, columnName := range columnNames {
lines = append(lines, strings.Repeat("-", len(columnName)))
}
rows = append(rows, strings.Join(lines, separator))
}
for _, v := range row.Values {
var values []string
if c.Format == "csv" {
if row.Name != "" {
values = append(values, row.Name)
}
if len(tags) > 0 {
values = append(values, strings.Join(tags, ","))
}
}
for _, vv := range v {
values = append(values, interfaceToString(vv))
}
rows = append(rows, strings.Join(values, separator))
}
// Outout a line separator if in column format
if c.Format == "column" {
rows = append(rows, "")
}
}
return rows
}
func interfaceToString(v interface{}) string {
switch t := v.(type) {
case nil:
return ""
case bool:
return fmt.Sprintf("%v", v)
case int, int8, int16, int32, int64, uint, uint8, uint16, uint32, uint64, uintptr:
return fmt.Sprintf("%d", t)
case float32, float64:
return fmt.Sprintf("%v", t)
default:
return fmt.Sprintf("%v", t)
}
}
func (c *CommandLine) Settings() {
w := new(tabwriter.Writer)
w.Init(os.Stdout, 0, 8, 1, '\t', 0)
if c.Port > 0 {
fmt.Fprintf(w, "Host\t%s:%d\n", c.Host, c.Port)
} else {
fmt.Fprintf(w, "Host\t%s\n", c.Host)
}
fmt.Fprintf(w, "Username\t%s\n", c.Username)
fmt.Fprintf(w, "Database\t%s\n", c.Database)
fmt.Fprintf(w, "Pretty\t%v\n", c.Pretty)
fmt.Fprintf(w, "Format\t%s\n", c.Format)
fmt.Fprintln(w)
w.Flush()
}
func (c *CommandLine) help() {
fmt.Println(`Usage:
connect <host:port> connect to another node
auth prompt for username and password
pretty toggle pretty print
use <db_name> set current databases
format <format> set the output format: json, csv, or column
settings output the current settings for the shell
exit quit the influx shell
show databases show database names
show series show series information
show measurements show measurement information
show tag keys show tag key information
show tag values show tag value information
a full list of influxql commands can be found at:
https://influxdb.com/docs/v0.9/query_language/spec.html
`)
}
func (c *CommandLine) gopher() {
fmt.Println(`
.-::-::://:-::- .:/++/'
'://:-''/oo+//++o+/.://o- ./+:
.:-. '++- .o/ '+yydhy' o-
.:/. .h: :osoys .smMN- :/
-/:.' s- /MMMymh. '/y/ s'
-+s:'''' d -mMMms// '-/o:
-/++/++/////:. o: '... s- :s.
:+-+s-' ':/' 's- /+ 'o:
'+-'o: /ydhsh. '//. '-o- o-
.y. o: .MMMdm+y ':+++:::/+:.' s:
.-h/ y- 'sdmds'h -+ydds:::-.' 'h.
.//-.d' o: '.' 'dsNMMMNh:.:++' :y
+y. 'd 's. .s:mddds: ++ o/
'N- odd 'o/. './o-s-' .---+++' o-
'N' yNd .://:/:::::. -s -+/s/./s' 'o/'
so' .h '''' ////s: '+. .s +y'
os/-.y' 's' 'y::+ +d'
'.:o/ -+:-:.' so.---.'
o' 'd-.''/s'
.s' :y.''.y
-s mo:::'
:: yh
// '''' /M'
o+ .s///:/. 'N:
:+ /: -s' ho
's- -/s/:+/.+h' +h
ys' ':' '-. -d
oh .h
/o .s
s. .h
-y .d
m/ -h
+d /o
'N- y:
h: m.
s- -d
o- s+
+- 'm'
s/ oo--.
y- /s ':+'
s' 'od--' .d:
-+ ':o: ':+-/+
y- .:+- '
//o- '.:+/.
.-:+/' ''-/+/.
./:' ''.:o+/-'
.+o:/:/+-' ''.-+ooo/-'
o: -h///++////-.
/: .o/
//+ 'y
./sooy.
`)
}

194
Godeps/_workspace/src/github.com/influxdb/influxdb/cmd/influx/main_test.go generated vendored
View File

@ -1,194 +0,0 @@
package main_test
import (
"encoding/json"
"net/http"
"net/http/httptest"
"net/url"
"testing"
"github.com/influxdb/influxdb/client"
main "github.com/influxdb/influxdb/cmd/influx"
)
func TestParseCommand_CommandsExist(t *testing.T) {
t.Parallel()
c := main.CommandLine{}
tests := []struct {
cmd string
}{
{cmd: "gopher"},
{cmd: "connect"},
{cmd: "help"},
{cmd: "pretty"},
{cmd: "use"},
{cmd: ""}, // test that a blank command just returns
}
for _, test := range tests {
if !c.ParseCommand(test.cmd) {
t.Fatalf(`Command failed for %q.`, test.cmd)
}
}
}
func TestParseCommand_TogglePretty(t *testing.T) {
t.Parallel()
c := main.CommandLine{}
if c.Pretty {
t.Fatalf(`Pretty should be false.`)
}
c.ParseCommand("pretty")
if !c.Pretty {
t.Fatalf(`Pretty should be true.`)
}
c.ParseCommand("pretty")
if c.Pretty {
t.Fatalf(`Pretty should be false.`)
}
}
func TestParseCommand_Exit(t *testing.T) {
t.Parallel()
c := main.CommandLine{}
tests := []struct {
cmd string
}{
{cmd: "exit"},
{cmd: " exit"},
{cmd: "exit "},
{cmd: "Exit "},
}
for _, test := range tests {
if c.ParseCommand(test.cmd) {
t.Fatalf(`Command "exit" failed for %q.`, test.cmd)
}
}
}
func TestParseCommand_Use(t *testing.T) {
t.Parallel()
c := main.CommandLine{}
tests := []struct {
cmd string
}{
{cmd: "use db"},
{cmd: " use db"},
{cmd: "use db "},
{cmd: "use db;"},
{cmd: "use db; "},
{cmd: "Use db"},
}
for _, test := range tests {
if !c.ParseCommand(test.cmd) {
t.Fatalf(`Command "use" failed for %q.`, test.cmd)
}
if c.Database != "db" {
t.Fatalf(`Command "use" changed database to %q. Expected db`, c.Database)
}
}
}
func TestParseCommand_Insert(t *testing.T) {
t.Parallel()
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
var data client.Response
w.WriteHeader(http.StatusNoContent)
_ = json.NewEncoder(w).Encode(data)
}))
defer ts.Close()
u, _ := url.Parse(ts.URL)
config := client.Config{URL: *u}
c, err := client.NewClient(config)
if err != nil {
t.Fatalf("unexpected error. expected %v, actual %v", nil, err)
}
m := main.CommandLine{Client: c}
tests := []struct {
cmd string
}{
{cmd: "INSERT cpu,host=serverA,region=us-west value=1.0"},
{cmd: " INSERT cpu,host=serverA,region=us-west value=1.0"},
{cmd: "INSERT cpu,host=serverA,region=us-west value=1.0"},
{cmd: "insert cpu,host=serverA,region=us-west value=1.0 "},
{cmd: "insert"},
{cmd: "Insert "},
{cmd: "insert c"},
{cmd: "insert int"},
}
for _, test := range tests {
if !m.ParseCommand(test.cmd) {
t.Fatalf(`Command "insert" failed for %q.`, test.cmd)
}
}
}
func TestParseCommand_InsertInto(t *testing.T) {
t.Parallel()
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
var data client.Response
w.WriteHeader(http.StatusNoContent)
_ = json.NewEncoder(w).Encode(data)
}))
defer ts.Close()
u, _ := url.Parse(ts.URL)
config := client.Config{URL: *u}
c, err := client.NewClient(config)
if err != nil {
t.Fatalf("unexpected error. expected %v, actual %v", nil, err)
}
m := main.CommandLine{Client: c}
tests := []struct {
cmd, db, rp string
}{
{
cmd: `INSERT INTO test cpu,host=serverA,region=us-west value=1.0`,
db: "",
rp: "test",
},
{
cmd: ` INSERT INTO .test cpu,host=serverA,region=us-west value=1.0`,
db: "",
rp: "test",
},
{
cmd: `INSERT INTO "test test" cpu,host=serverA,region=us-west value=1.0`,
db: "",
rp: "test test",
},
{
cmd: `Insert iNTO test.test cpu,host=serverA,region=us-west value=1.0`,
db: "test",
rp: "test",
},
{
cmd: `insert into "test test" cpu,host=serverA,region=us-west value=1.0`,
db: "test",
rp: "test test",
},
{
cmd: `insert into "d b"."test test" cpu,host=serverA,region=us-west value=1.0`,
db: "d b",
rp: "test test",
},
}
for _, test := range tests {
if !m.ParseCommand(test.cmd) {
t.Fatalf(`Command "insert into" failed for %q.`, test.cmd)
}
if m.Database != test.db {
t.Fatalf(`Command "insert into" db parsing failed, expected: %q, actual: %q`, test.db, m.Database)
}
if m.RetentionPolicy != test.rp {
t.Fatalf(`Command "insert into" rp parsing failed, expected: %q, actual: %q`, test.rp, m.RetentionPolicy)
}
}
}

154
Godeps/_workspace/src/github.com/influxdb/influxdb/cmd/influx_stress/influx_stress.go generated vendored
View File

@ -1,154 +0,0 @@
package main
import (
"flag"
"fmt"
"math/rand"
"net/url"
"runtime"
"sort"
"sync"
"time"
"github.com/influxdb/influxdb/client"
)
var (
batchSize = flag.Int("batchsize", 5000, "number of points per batch")
seriesCount = flag.Int("series", 100000, "number of unique series to create")
pointCount = flag.Int("points", 100, "number of points per series to create")
concurrency = flag.Int("concurrency", 10, "number of simultaneous writes to run")
batchInterval = flag.Duration("batchinterval", 0*time.Second, "duration between batches")
database = flag.String("database", "stress", "name of database")
address = flag.String("addr", "localhost:8086", "IP address and port of database (e.g., localhost:8086)")
)
func main() {
flag.Parse()
runtime.GOMAXPROCS(runtime.NumCPU())
startTime := time.Now()
counter := NewConcurrencyLimiter(*concurrency)
u, _ := url.Parse(fmt.Sprintf("http://%s", *address))
c, err := client.NewClient(client.Config{URL: *u})
if err != nil {
panic(err)
}
var mu sync.Mutex
var wg sync.WaitGroup
responseTimes := make([]int, 0)
totalPoints := 0
batch := &client.BatchPoints{
Database: *database,
WriteConsistency: "any",
Time: time.Now(),
Precision: "n",
}
for i := 1; i <= *pointCount; i++ {
for j := 1; j <= *seriesCount; j++ {
p := client.Point{
Measurement: "cpu",
Tags: map[string]string{"region": "uswest", "host": fmt.Sprintf("host-%d", j)},
Fields: map[string]interface{}{"value": rand.Float64()},
}
batch.Points = append(batch.Points, p)
if len(batch.Points) >= *batchSize {
wg.Add(1)
counter.Increment()
totalPoints += len(batch.Points)
go func(b *client.BatchPoints, total int) {
st := time.Now()
if _, err := c.Write(*b); err != nil {
fmt.Println("ERROR: ", err.Error())
} else {
mu.Lock()
responseTimes = append(responseTimes, int(time.Since(st).Nanoseconds()))
mu.Unlock()
}
wg.Done()
counter.Decrement()
if total%500000 == 0 {
fmt.Printf("%d total points. %d in %s\n", total, *batchSize, time.Since(st))
}
}(batch, totalPoints)
batch = &client.BatchPoints{
Database: *database,
WriteConsistency: "any",
Precision: "n",
Time: time.Now(),
}
}
}
}
wg.Wait()
sort.Sort(sort.Reverse(sort.IntSlice(responseTimes)))
total := int64(0)
for _, t := range responseTimes {
total += int64(t)
}
mean := total / int64(len(responseTimes))
fmt.Printf("Wrote %d points at average rate of %.0f\n", totalPoints, float64(totalPoints)/time.Since(startTime).Seconds())
fmt.Println("Average response time: ", time.Duration(mean))
fmt.Println("Slowest response times:")
for _, r := range responseTimes[:100] {
fmt.Println(time.Duration(r))
}
}
// ConcurrencyLimiter is a go routine safe struct that can be used to
// ensure that no more than a specifid max number of goroutines are
// executing.
type ConcurrencyLimiter struct {
inc chan chan struct{}
dec chan struct{}
max int
count int
}
// NewConcurrencyLimiter returns a configured limiter that will
// ensure that calls to Increment will block if the max is hit.
func NewConcurrencyLimiter(max int) *ConcurrencyLimiter {
c := &ConcurrencyLimiter{
inc: make(chan chan struct{}),
dec: make(chan struct{}, max),
max: max,
}
go c.handleLimits()
return c
}
// Increment will increase the count of running goroutines by 1.
// if the number is currently at the max, the call to Increment
// will block until another goroutine decrements.
func (c *ConcurrencyLimiter) Increment() {
r := make(chan struct{})
c.inc <- r
<-r
}
// Decrement will reduce the count of running goroutines by 1
func (c *ConcurrencyLimiter) Decrement() {
c.dec <- struct{}{}
}
// handleLimits runs in a goroutine to manage the count of
// running goroutines.
func (c *ConcurrencyLimiter) handleLimits() {
for {
r := <-c.inc
if c.count >= c.max {
<-c.dec
c.count--
}
c.count++
r <- struct{}{}
}
}

170
Godeps/_workspace/src/github.com/influxdb/influxdb/cmd/influxd/backup/backup.go generated vendored
View File

@ -1,170 +0,0 @@
package backup
import (
"encoding/json"
"errors"
"flag"
"fmt"
"io"
"log"
"net"
"os"
"github.com/influxdb/influxdb/services/snapshotter"
"github.com/influxdb/influxdb/snapshot"
)
// Suffix is a suffix added to the backup while it's in-process.
const Suffix = ".pending"
// Command represents the program execution for "influxd backup".
type Command struct {
// The logger passed to the ticker during execution.
Logger *log.Logger
// Standard input/output, overridden for testing.
Stderr io.Writer
}
// NewCommand returns a new instance of Command with default settings.
func NewCommand() *Command {
return &Command{
Stderr: os.Stderr,
}
}
// Run executes the program.
func (cmd *Command) Run(args ...string) error {
// Set up logger.
cmd.Logger = log.New(cmd.Stderr, "", log.LstdFlags)
cmd.Logger.Printf("influxdb backup")
// Parse command line arguments.
host, path, err := cmd.parseFlags(args)
if err != nil {
return err
}
// Retrieve snapshot from local file.
m, err := snapshot.ReadFileManifest(path)
if err != nil && !os.IsNotExist(err) {
return fmt.Errorf("read file snapshot: %s", err)
}
// Determine temporary path to download to.
tmppath := path + Suffix
// Calculate path of next backup file.
// This uses the path if it doesn't exist.
// Otherwise it appends an autoincrementing number.
path, err = cmd.nextPath(path)
if err != nil {
return fmt.Errorf("next path: %s", err)
}
// Retrieve snapshot.
if err := cmd.download(host, m, tmppath); err != nil {
return fmt.Errorf("download: %s", err)
}
// Rename temporary file to final path.
if err := os.Rename(tmppath, path); err != nil {
return fmt.Errorf("rename: %s", err)
}
// TODO: Check file integrity.
// Notify user of completion.
cmd.Logger.Println("backup complete")
return nil
}
// parseFlags parses and validates the command line arguments.
func (cmd *Command) parseFlags(args []string) (host string, path string, err error) {
fs := flag.NewFlagSet("", flag.ContinueOnError)
fs.StringVar(&host, "host", "localhost:8088", "")
fs.SetOutput(cmd.Stderr)
fs.Usage = cmd.printUsage
if err := fs.Parse(args); err != nil {
return "", "", err
}
// Ensure that only one arg is specified.
if fs.NArg() == 0 {
return "", "", errors.New("snapshot path required")
} else if fs.NArg() != 1 {
return "", "", errors.New("only one snapshot path allowed")
}
path = fs.Arg(0)
return host, path, nil
}
// nextPath returns the next file to write to.
func (cmd *Command) nextPath(path string) (string, error) {
// Use base path if it doesn't exist.
if _, err := os.Stat(path); os.IsNotExist(err) {
return path, nil
} else if err != nil {
return "", err
}
// Otherwise iterate through incremental files until one is available.
for i := 0; ; i++ {
s := fmt.Sprintf(path+".%d", i)
if _, err := os.Stat(s); os.IsNotExist(err) {
return s, nil
} else if err != nil {
return "", err
}
}
}
// download downloads a snapshot from a host to a given path.
func (cmd *Command) download(host string, m *snapshot.Manifest, path string) error {
// Create local file to write to.
f, err := os.Create(path)
if err != nil {
return fmt.Errorf("open temp file: %s", err)
}
defer f.Close()
// Connect to snapshotter service.
conn, err := net.Dial("tcp", host)
if err != nil {
return err
}
defer conn.Close()
// Send snapshotter marker byte.
if _, err := conn.Write([]byte{snapshotter.MuxHeader}); err != nil {
return fmt.Errorf("write snapshot header byte: %s", err)
}
// Write the manifest we currently have.
if err := json.NewEncoder(conn).Encode(m); err != nil {
return fmt.Errorf("encode snapshot manifest: %s", err)
}
// Read snapshot from the connection.
if _, err := io.Copy(f, conn); err != nil {
return fmt.Errorf("copy snapshot to file: %s", err)
}
// FIXME(benbjohnson): Verify integrity of snapshot.
return nil
}
// printUsage prints the usage message to STDERR.
func (cmd *Command) printUsage() {
fmt.Fprintf(cmd.Stderr, `usage: influxd backup [flags] PATH
backup downloads a snapshot of a data node and saves it to disk.
-host <host:port>
The host to connect to snapshot.
Defaults to 127.0.0.1:8088.
`)
}

125
Godeps/_workspace/src/github.com/influxdb/influxdb/cmd/influxd/backup/backup_test.go generated vendored
View File

@ -1,125 +0,0 @@
package backup_test
/*
import (
"bytes"
"net/http"
"net/http/httptest"
"os"
"strings"
"testing"
"github.com/influxdb/influxdb"
"github.com/influxdb/influxdb/cmd/influxd"
)
// Ensure the backup can download from the server and save to disk.
func TestBackupCommand(t *testing.T) {
// Mock the backup endpoint.
s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/data/snapshot" {
t.Fatalf("unexpected url path: %s", r.URL.Path)
}
// Write a simple snapshot to the buffer.
sw := influxdb.NewSnapshotWriter()
sw.Snapshot = &influxdb.Snapshot{Files: []influxdb.SnapshotFile{
{Name: "meta", Size: 5, Index: 10},
}}
sw.FileWriters["meta"] = influxdb.NopWriteToCloser(bytes.NewBufferString("55555"))
if _, err := sw.WriteTo(w); err != nil {
t.Fatal(err)
}
}))
defer s.Close()
// Create a temp path and remove incremental backups at the end.
path := tempfile()
defer os.Remove(path)
defer os.Remove(path + ".0")
defer os.Remove(path + ".1")
// Execute the backup against the mock server.
for i := 0; i < 3; i++ {
if err := NewBackupCommand().Run("-host", s.URL, path); err != nil {
t.Fatal(err)
}
}
// Verify snapshot and two incremental snapshots were written.
if _, err := os.Stat(path); err != nil {
t.Fatalf("snapshot not found: %s", err)
} else if _, err = os.Stat(path + ".0"); err != nil {
t.Fatalf("incremental snapshot(0) not found: %s", err)
} else if _, err = os.Stat(path + ".1"); err != nil {
t.Fatalf("incremental snapshot(1) not found: %s", err)
}
}
// Ensure the backup command returns an error if flags cannot be parsed.
func TestBackupCommand_ErrFlagParse(t *testing.T) {
cmd := NewBackupCommand()
if err := cmd.Run("-bad-flag"); err == nil || err.Error() != `flag provided but not defined: -bad-flag` {
t.Fatal(err)
} else if !strings.Contains(cmd.Stderr.String(), "usage") {
t.Fatal("usage message not displayed")
}
}
// Ensure the backup command returns an error if the host cannot be parsed.
func TestBackupCommand_ErrInvalidHostURL(t *testing.T) {
if err := NewBackupCommand().Run("-host", "http://%f"); err == nil || err.Error() != `parse host url: parse http://%f: hexadecimal escape in host` {
t.Fatal(err)
}
}
// Ensure the backup command returns an error if the output path is not specified.
func TestBackupCommand_ErrPathRequired(t *testing.T) {
if err := NewBackupCommand().Run("-host", "//localhost"); err == nil || err.Error() != `snapshot path required` {
t.Fatal(err)
}
}
// Ensure the backup returns an error if it cannot connect to the server.
func TestBackupCommand_ErrConnectionRefused(t *testing.T) {
// Start and immediately stop a server so we have a dead port.
s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}))
s.Close()
// Execute the backup command.
path := tempfile()
defer os.Remove(path)
if err := NewBackupCommand().Run("-host", s.URL, path); err == nil ||
!(strings.Contains(err.Error(), `connection refused`) || strings.Contains(err.Error(), `No connection could be made`)) {
t.Fatal(err)
}
}
// Ensure the backup returns any non-200 status codes.
func TestBackupCommand_ErrServerError(t *testing.T) {
s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusInternalServerError)
}))
defer s.Close()
// Execute the backup command.
path := tempfile()
defer os.Remove(path)
if err := NewBackupCommand().Run("-host", s.URL, path); err == nil || err.Error() != `download: snapshot error: status=500` {
t.Fatal(err)
}
}
// BackupCommand is a test wrapper for main.BackupCommand.
type BackupCommand struct {
*main.BackupCommand
Stderr bytes.Buffer
}
// NewBackupCommand returns a new instance of BackupCommand.
func NewBackupCommand() *BackupCommand {
cmd := &BackupCommand{BackupCommand: main.NewBackupCommand()}
cmd.BackupCommand.Stderr = &cmd.Stderr
return cmd
}
*/

46
Godeps/_workspace/src/github.com/influxdb/influxdb/cmd/influxd/help/help.go generated vendored
View File

@ -1,46 +0,0 @@
package help
import (
"fmt"
"io"
"os"
"strings"
)
// Command displays help for command-line sub-commands.
type Command struct {
Stdout io.Writer
}
// NewCommand returns a new instance of Command.
func NewCommand() *Command {
return &Command{
Stdout: os.Stdout,
}
}
// Run executes the command.
func (cmd *Command) Run(args ...string) error {
fmt.Fprintln(cmd.Stdout, strings.TrimSpace(usage))
return nil
}
const usage = `
Configure and start an InfluxDB server.
Usage:
influxd [[command] [arguments]]
The commands are:
backup downloads a snapshot of a data node and saves it to disk
config display the default configuration
restore uses a snapshot of a data node to rebuild a cluster
run run node with existing configuration
version displays the InfluxDB version
"run" is the default command.
Use "influxd help [command]" for more information about a command.
`

200
Godeps/_workspace/src/github.com/influxdb/influxdb/cmd/influxd/main.go generated vendored
View File

@ -1,200 +0,0 @@
package main
import (
"flag"
"fmt"
"io"
"log"
"math/rand"
"os"
"os/signal"
"strings"
"syscall"
"time"
"github.com/influxdb/influxdb/cmd/influxd/backup"
"github.com/influxdb/influxdb/cmd/influxd/help"
"github.com/influxdb/influxdb/cmd/influxd/restore"
"github.com/influxdb/influxdb/cmd/influxd/run"
)
// These variables are populated via the Go linker.
var (
version string = "0.9"
commit string
branch string
)
func init() {
// If commit or branch are not set, make that clear.
if commit == "" {
commit = "unknown"
}
if branch == "" {
branch = "unknown"
}
}
func main() {
rand.Seed(time.Now().UnixNano())
m := NewMain()
if err := m.Run(os.Args[1:]...); err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
}
// Main represents the program execution.
type Main struct {
Logger *log.Logger
Stdin io.Reader
Stdout io.Writer
Stderr io.Writer
}
// NewMain return a new instance of Main.
func NewMain() *Main {
return &Main{
Logger: log.New(os.Stderr, "[run] ", log.LstdFlags),
Stdin: os.Stdin,
Stdout: os.Stdout,
Stderr: os.Stderr,
}
}
// Run determines and runs the command specified by the CLI args.
func (m *Main) Run(args ...string) error {
name, args := ParseCommandName(args)
// Extract name from args.
switch name {
case "", "run":
cmd := run.NewCommand()
// Tell the server the build details.
cmd.Version = version
cmd.Commit = commit
cmd.Branch = branch
if err := cmd.Run(args...); err != nil {
return fmt.Errorf("run: %s", err)
}
signalCh := make(chan os.Signal, 1)
signal.Notify(signalCh, os.Interrupt, syscall.SIGTERM)
m.Logger.Println("Listening for signals")
// Block until one of the signals above is received
select {
case <-signalCh:
m.Logger.Println("Signal received, initializing clean shutdown...")
go func() {
cmd.Close()
}()
}
// Block again until another signal is received, a shutdown timeout elapses,
// or the Command is gracefully closed
m.Logger.Println("Waiting for clean shutdown...")
select {
case <-signalCh:
m.Logger.Println("second signal received, initializing hard shutdown")
case <-time.After(time.Second * 30):
m.Logger.Println("time limit reached, initializing hard shutdown")
case <-cmd.Closed:
m.Logger.Println("server shutdown completed")
}
// goodbye.
case "backup":
name := backup.NewCommand()
if err := name.Run(args...); err != nil {
return fmt.Errorf("backup: %s", err)
}
case "restore":
name := restore.NewCommand()
if err := name.Run(args...); err != nil {
return fmt.Errorf("restore: %s", err)
}
case "config":
if err := run.NewPrintConfigCommand().Run(args...); err != nil {
return fmt.Errorf("config: %s", err)
}
case "version":
if err := NewVersionCommand().Run(args...); err != nil {
return fmt.Errorf("version: %s", err)
}
case "help":
if err := help.NewCommand().Run(args...); err != nil {
return fmt.Errorf("help: %s", err)
}
default:
return fmt.Errorf(`unknown command "%s"`+"\n"+`Run 'influxd help' for usage`+"\n\n", name)
}
return nil
}
// ParseCommandName extracts the command name and args from the args list.
func ParseCommandName(args []string) (string, []string) {
// Retrieve command name as first argument.
var name string
if len(args) > 0 && !strings.HasPrefix(args[0], "-") {
name = args[0]
}
// Special case -h immediately following binary name
if len(args) > 0 && args[0] == "-h" {
name = "help"
}
// If command is "help" and has an argument then rewrite args to use "-h".
if name == "help" && len(args) > 1 {
args[0], args[1] = args[1], "-h"
name = args[0]
}
// If a named command is specified then return it with its arguments.
if name != "" {
return name, args[1:]
}
return "", args
}
// Command represents the command executed by "influxd version".
type VersionCommand struct {
Stdout io.Writer
Stderr io.Writer
}
// NewVersionCommand return a new instance of VersionCommand.
func NewVersionCommand() *VersionCommand {
return &VersionCommand{
Stdout: os.Stdout,
Stderr: os.Stderr,
}
}
// Run prints the current version and commit info.
func (cmd *VersionCommand) Run(args ...string) error {
// Parse flags in case -h is specified.
fs := flag.NewFlagSet("", flag.ContinueOnError)
fs.Usage = func() { fmt.Fprintln(cmd.Stderr, strings.TrimSpace(versionUsage)) }
if err := fs.Parse(args); err != nil {
return err
}
// Print version info.
fmt.Fprintf(cmd.Stdout, "InfluxDB v%s (git: %s %s)\n", version, branch, commit)
return nil
}
var versionUsage = `
usage: version
version displays the InfluxDB version, build branch and git commit hash
`

250
Godeps/_workspace/src/github.com/influxdb/influxdb/cmd/influxd/restore/restore.go generated vendored
View File

@ -1,250 +0,0 @@
package restore
import (
"bytes"
"errors"
"flag"
"fmt"
"io"
"net"
"os"
"path/filepath"
"github.com/BurntSushi/toml"
"github.com/influxdb/influxdb/meta"
"github.com/influxdb/influxdb/snapshot"
"github.com/influxdb/influxdb/tsdb"
)
// Command represents the program execution for "influxd restore".
type Command struct {
Stdout io.Writer
Stderr io.Writer
}
// NewCommand returns a new instance of Command with default settings.
func NewCommand() *Command {
return &Command{
Stdout: os.Stdout,
Stderr: os.Stderr,
}
}
// Run executes the program.
func (cmd *Command) Run(args ...string) error {
config, path, err := cmd.parseFlags(args)
if err != nil {
return err
}
return cmd.Restore(config, path)
}
func (cmd *Command) Restore(config *Config, path string) error {
// Remove meta and data directories.
if err := os.RemoveAll(config.Meta.Dir); err != nil {
return fmt.Errorf("remove meta dir: %s", err)
} else if err := os.RemoveAll(config.Data.Dir); err != nil {
return fmt.Errorf("remove data dir: %s", err)
}
// Open snapshot file and all incremental backups.
mr, files, err := snapshot.OpenFileMultiReader(path)
if err != nil {
return fmt.Errorf("open multireader: %s", err)
}
defer closeAll(files)
// Unpack files from archive.
if err := cmd.unpack(mr, config); err != nil {
return fmt.Errorf("unpack: %s", err)
}
// Notify user of completion.
fmt.Fprintf(os.Stdout, "restore complete using %s", path)
return nil
}
// parseFlags parses and validates the command line arguments.
func (cmd *Command) parseFlags(args []string) (*Config, string, error) {
fs := flag.NewFlagSet("", flag.ContinueOnError)
configPath := fs.String("config", "", "")
fs.SetOutput(cmd.Stderr)
fs.Usage = cmd.printUsage
if err := fs.Parse(args); err != nil {
return nil, "", err
}
// Parse configuration file from disk.
if *configPath == "" {
return nil, "", fmt.Errorf("config required")
}
// Parse config.
config := Config{
Meta: meta.NewConfig(),
Data: tsdb.NewConfig(),
}
if _, err := toml.DecodeFile(*configPath, &config); err != nil {
return nil, "", err
}
// Require output path.
path := fs.Arg(0)
if path == "" {
return nil, "", fmt.Errorf("snapshot path required")
}
return &config, path, nil
}
func closeAll(a []io.Closer) {
for _, c := range a {
_ = c.Close()
}
}
// unpack expands the files in the snapshot archive into a directory.
func (cmd *Command) unpack(mr *snapshot.MultiReader, config *Config) error {
// Loop over files and extract.
for {
// Read entry header.
sf, err := mr.Next()
if err == io.EOF {
break
} else if err != nil {
return fmt.Errorf("next: entry=%s, err=%s", sf.Name, err)
}
// Log progress.
fmt.Fprintf(os.Stdout, "unpacking: %s (%d bytes)\n", sf.Name, sf.Size)
// Handle meta and tsdb files separately.
switch sf.Name {
case "meta":
if err := cmd.unpackMeta(mr, sf, config); err != nil {
return fmt.Errorf("meta: %s", err)
}
default:
if err := cmd.unpackData(mr, sf, config); err != nil {
return fmt.Errorf("data: %s", err)
}
}
}
return nil
}
// unpackMeta reads the metadata from the snapshot and initializes a raft
// cluster and replaces the root metadata.
func (cmd *Command) unpackMeta(mr *snapshot.MultiReader, sf snapshot.File, config *Config) error {
// Read meta into buffer.
var buf bytes.Buffer
if _, err := io.CopyN(&buf, mr, sf.Size); err != nil {
return fmt.Errorf("copy: %s", err)
}
// Unpack into metadata.
var data meta.Data
if err := data.UnmarshalBinary(buf.Bytes()); err != nil {
return fmt.Errorf("unmarshal: %s", err)
}
// Copy meta config and remove peers so it starts in single mode.
c := config.Meta
c.Peers = nil
// Initialize meta store.
store := meta.NewStore(config.Meta)
store.RaftListener = newNopListener()
store.ExecListener = newNopListener()
// Determine advertised address.
_, port, err := net.SplitHostPort(config.Meta.BindAddress)
if err != nil {
return fmt.Errorf("split bind address: %s", err)
}
hostport := net.JoinHostPort(config.Meta.Hostname, port)
// Resolve address.
addr, err := net.ResolveTCPAddr("tcp", hostport)
if err != nil {
return fmt.Errorf("resolve tcp: addr=%s, err=%s", hostport, err)
}
store.Addr = addr
// Open the meta store.
if err := store.Open(); err != nil {
return fmt.Errorf("open store: %s", err)
}
defer store.Close()
// Wait for the store to be ready or error.
select {
case <-store.Ready():
case err := <-store.Err():
return err
}
// Force set the full metadata.
if err := store.SetData(&data); err != nil {
return fmt.Errorf("set data: %s", err)
}
return nil
}
func (cmd *Command) unpackData(mr *snapshot.MultiReader, sf snapshot.File, config *Config) error {
path := filepath.Join(config.Data.Dir, sf.Name)
// Create parent directory for output file.
if err := os.MkdirAll(filepath.Dir(path), 0777); err != nil {
return fmt.Errorf("mkdir: entry=%s, err=%s", sf.Name, err)
}
// Create output file.
f, err := os.Create(path)
if err != nil {
return fmt.Errorf("create: entry=%s, err=%s", sf.Name, err)
}
defer f.Close()
// Copy contents from reader.
if _, err := io.CopyN(f, mr, sf.Size); err != nil {
return fmt.Errorf("copy: entry=%s, err=%s", sf.Name, err)
}
return nil
}
// printUsage prints the usage message to STDERR.
func (cmd *Command) printUsage() {
fmt.Fprintf(cmd.Stderr, `usage: influxd restore [flags] PATH
restore uses a snapshot of a data node to rebuild a cluster.
-config <path>
Set the path to the configuration file.
`)
}
// Config represents a partial config for rebuilding the server.
type Config struct {
Meta *meta.Config `toml:"meta"`
Data tsdb.Config `toml:"data"`
}
type nopListener struct {
closing chan struct{}
}
func newNopListener() *nopListener {
return &nopListener{make(chan struct{})}
}
func (ln *nopListener) Accept() (net.Conn, error) {
<-ln.closing
return nil, errors.New("listener closing")
}
func (ln *nopListener) Close() error { close(ln.closing); return nil }
func (ln *nopListener) Addr() net.Addr { return nil }

155
Godeps/_workspace/src/github.com/influxdb/influxdb/cmd/influxd/restore/restore_test.go generated vendored
View File

@ -1,155 +0,0 @@
package restore_test
/*
import (
"bytes"
"io/ioutil"
"os"
"path/filepath"
"reflect"
"testing"
"time"
main "github.com/influxdb/influxdb/cmd/influxd"
"github.com/influxdb/influxdb/tsdb"
)
func newConfig(path string, port int) main.Config {
config := main.NewConfig()
config.Port = port
config.Broker.Enabled = true
config.Broker.Dir = filepath.Join(path, "broker")
config.Data.Enabled = true
config.Data.Dir = filepath.Join(path, "data")
return *config
}
// Ensure the restore command can expand a snapshot and bootstrap a broker.
func TestRestoreCommand(t *testing.T) {
if testing.Short() {
t.Skip("skipping TestRestoreCommand")
}
now := time.Now()
// Create root path to server.
path := tempfile()
defer os.Remove(path)
// Parse configuration.
config := newConfig(path, 8900)
// Start server.
cmd := main.NewRunCommand()
node := cmd.Open(&config, "")
if node.Broker == nil {
t.Fatal("cannot run broker")
} else if node.DataNode == nil {
t.Fatal("cannot run server")
}
b := node.Broker
s := node.DataNode
// Create data.
if err := s.CreateDatabase("db"); err != nil {
t.Fatalf("cannot create database: %s", err)
}
if index, err := s.WriteSeries("db", "default", []tsdb.Point{tsdb.NewPoint("cpu", nil, map[string]interface{}{"value": float64(100)}, now)}); err != nil {
t.Fatalf("cannot write series: %s", err)
} else if err = s.Sync(1, index); err != nil {
t.Fatalf("shard sync: %s", err)
}
// Create snapshot writer.
sw, err := s.CreateSnapshotWriter()
if err != nil {
t.Fatalf("create snapshot writer: %s", err)
}
// Snapshot to file.
sspath := tempfile()
f, err := os.Create(sspath)
if err != nil {
t.Fatal(err)
}
sw.WriteTo(f)
f.Close()
// Stop server.
node.Close()
// Remove data & broker directories.
if err := os.RemoveAll(path); err != nil {
t.Fatalf("remove: %s", err)
}
// Execute the restore.
if err := NewRestoreCommand().Restore(&config, sspath); err != nil {
t.Fatal(err)
}
// Rewrite config to a new port and re-parse.
config = newConfig(path, 8910)
// Restart server.
cmd = main.NewRunCommand()
node = cmd.Open(&config, "")
if b == nil {
t.Fatal("cannot run broker")
} else if s == nil {
t.Fatal("cannot run server")
}
b = node.Broker
s = node.DataNode
// Write new data.
if err := s.CreateDatabase("newdb"); err != nil {
t.Fatalf("cannot create new database: %s", err)
}
if index, err := s.WriteSeries("newdb", "default", []tsdb.Point{tsdb.NewPoint("mem", nil, map[string]interface{}{"value": float64(1000)}, now)}); err != nil {
t.Fatalf("cannot write new series: %s", err)
} else if err = s.Sync(2, index); err != nil {
t.Fatalf("shard sync: %s", err)
}
// Read series data.
if v, err := s.ReadSeries("db", "default", "cpu", nil, now); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(v, map[string]interface{}{"value": float64(100)}) {
t.Fatalf("read series(0) mismatch: %#v", v)
}
// Read new series data.
if v, err := s.ReadSeries("newdb", "default", "mem", nil, now); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(v, map[string]interface{}{"value": float64(1000)}) {
t.Fatalf("read series(1) mismatch: %#v", v)
}
// Stop server.
node.Close()
}
// RestoreCommand is a test wrapper for main.RestoreCommand.
type RestoreCommand struct {
*main.RestoreCommand
Stderr bytes.Buffer
}
// NewRestoreCommand returns a new instance of RestoreCommand.
func NewRestoreCommand() *RestoreCommand {
cmd := &RestoreCommand{RestoreCommand: main.NewRestoreCommand()}
cmd.RestoreCommand.Stderr = &cmd.Stderr
return cmd
}
// MustReadFile reads data from a file. Panic on error.
func MustReadFile(filename string) []byte {
b, err := ioutil.ReadFile(filename)
if err != nil {
panic(err.Error())
}
return b
}
*/

235
Godeps/_workspace/src/github.com/influxdb/influxdb/cmd/influxd/run/command.go generated vendored
View File

@ -1,235 +0,0 @@
package run
import (
"flag"
"fmt"
"io"
"io/ioutil"
"log"
"os"
"path/filepath"
"runtime"
"strconv"
"strings"
"time"
"github.com/BurntSushi/toml"
)
const logo = `
8888888 .d888 888 8888888b. 888888b.
888 d88P" 888 888 "Y88b 888 "88b
888 888 888 888 888 888 .88P
888 88888b. 888888 888 888 888 888 888 888 888 8888888K.
888 888 "88b 888 888 888 888 Y8bd8P' 888 888 888 "Y88b
888 888 888 888 888 888 888 X88K 888 888 888 888
888 888 888 888 888 Y88b 888 .d8""8b. 888 .d88P 888 d88P
8888888 888 888 888 888 "Y88888 888 888 8888888P" 8888888P"
`
// Command represents the command executed by "influxd run".
type Command struct {
Version string
Branch string
Commit string
closing chan struct{}
Closed chan struct{}
Stdin io.Reader
Stdout io.Writer
Stderr io.Writer
Server *Server
}
// NewCommand return a new instance of Command.
func NewCommand() *Command {
return &Command{
closing: make(chan struct{}),
Closed: make(chan struct{}),
Stdin: os.Stdin,
Stdout: os.Stdout,
Stderr: os.Stderr,
}
}
// Run parses the config from args and runs the server.
func (cmd *Command) Run(args ...string) error {
// Parse the command line flags.
options, err := cmd.ParseFlags(args...)
if err != nil {
return err
}
// Print sweet InfluxDB logo.
fmt.Print(logo)
// Write the PID file.
if err := cmd.writePIDFile(options.PIDFile); err != nil {
return fmt.Errorf("write pid file: %s", err)
}
// Set parallelism.
runtime.GOMAXPROCS(runtime.NumCPU())
// Turn on block profiling to debug stuck databases
runtime.SetBlockProfileRate(int(1 * time.Second))
// Parse config
config, err := cmd.ParseConfig(options.ConfigPath)
if err != nil {
return fmt.Errorf("parse config: %s", err)
}
// Apply any environment variables on top of the parsed config
if err := config.ApplyEnvOverrides(); err != nil {
return fmt.Errorf("apply env config: %v", err)
}
// Override config hostname if specified in the command line args.
if options.Hostname != "" {
config.Meta.Hostname = options.Hostname
}
if options.Join != "" {
config.Meta.Peers = strings.Split(options.Join, ",")
}
// Validate the configuration.
if err := config.Validate(); err != nil {
return fmt.Errorf("%s. To generate a valid configuration file run `influxd config > influxdb.generated.conf`.", err)
}
// Create server from config and start it.
s, err := NewServer(config, cmd.Version)
if err != nil {
return fmt.Errorf("create server: %s", err)
}
s.CPUProfile = options.CPUProfile
s.MemProfile = options.MemProfile
if err := s.Open(); err != nil {
return fmt.Errorf("open server: %s", err)
}
cmd.Server = s
// Mark start-up in log.
log.Printf("InfluxDB starting, version %s, branch %s, commit %s", cmd.Version, cmd.Branch, cmd.Commit)
log.Println("GOMAXPROCS set to", runtime.GOMAXPROCS(0))
// Begin monitoring the server's error channel.
go cmd.monitorServerErrors()
return nil
}
// Close shuts down the server.
func (cmd *Command) Close() error {
defer close(cmd.Closed)
close(cmd.closing)
if cmd.Server != nil {
return cmd.Server.Close()
}
return nil
}
func (cmd *Command) monitorServerErrors() {
logger := log.New(cmd.Stderr, "", log.LstdFlags)
for {
select {
case err := <-cmd.Server.Err():
logger.Println(err)
case <-cmd.closing:
return
}
}
}
// ParseFlags parses the command line flags from args and returns an options set.
func (cmd *Command) ParseFlags(args ...string) (Options, error) {
var options Options
fs := flag.NewFlagSet("", flag.ContinueOnError)
fs.StringVar(&options.ConfigPath, "config", "", "")
fs.StringVar(&options.PIDFile, "pidfile", "", "")
fs.StringVar(&options.Hostname, "hostname", "", "")
fs.StringVar(&options.Join, "join", "", "")
fs.StringVar(&options.CPUProfile, "cpuprofile", "", "")
fs.StringVar(&options.MemProfile, "memprofile", "", "")
fs.Usage = func() { fmt.Fprintln(cmd.Stderr, usage) }
if err := fs.Parse(args); err != nil {
return Options{}, err
}
return options, nil
}
// writePIDFile writes the process ID to path.
func (cmd *Command) writePIDFile(path string) error {
// Ignore if path is not set.
if path == "" {
return nil
}
// Ensure the required directory structure exists.
err := os.MkdirAll(filepath.Dir(path), 0777)
if err != nil {
return fmt.Errorf("mkdir: %s", err)
}
// Retrieve the PID and write it.
pid := strconv.Itoa(os.Getpid())
if err := ioutil.WriteFile(path, []byte(pid), 0666); err != nil {
return fmt.Errorf("write file: %s", err)
}
return nil
}
// ParseConfig parses the config at path.
// Returns a demo configuration if path is blank.
func (cmd *Command) ParseConfig(path string) (*Config, error) {
// Use demo configuration if no config path is specified.
if path == "" {
fmt.Fprintln(cmd.Stdout, "no configuration provided, using default settings")
return NewDemoConfig()
}
fmt.Fprintf(cmd.Stdout, "Using configuration at: %s\n", path)
config := NewConfig()
if _, err := toml.DecodeFile(path, &config); err != nil {
return nil, err
}
return config, nil
}
var usage = `usage: run [flags]
run starts the broker and data node server. If this is the first time running
the command then a new cluster will be initialized unless the -join argument
is used.
-config <path>
Set the path to the configuration file.
-hostname <name>
Override the hostname, the 'hostname' configuration
option will be overridden.
-join <url>
Joins the server to an existing cluster.
-pidfile <path>
Write process ID to a file.
`
// Options represents the command line options that can be parsed.
type Options struct {
ConfigPath string
PIDFile string
Hostname string
Join string
CPUProfile string
MemProfile string
}

227
Godeps/_workspace/src/github.com/influxdb/influxdb/cmd/influxd/run/config.go generated vendored
View File

@ -1,227 +0,0 @@
package run
import (
"errors"
"fmt"
"os"
"os/user"
"path/filepath"
"reflect"
"strconv"
"strings"
"time"
"github.com/influxdb/influxdb/cluster"
"github.com/influxdb/influxdb/meta"
"github.com/influxdb/influxdb/services/admin"
"github.com/influxdb/influxdb/services/collectd"
"github.com/influxdb/influxdb/services/continuous_querier"
"github.com/influxdb/influxdb/services/graphite"
"github.com/influxdb/influxdb/services/hh"
"github.com/influxdb/influxdb/services/httpd"
"github.com/influxdb/influxdb/services/monitor"
"github.com/influxdb/influxdb/services/opentsdb"
"github.com/influxdb/influxdb/services/precreator"
"github.com/influxdb/influxdb/services/retention"
"github.com/influxdb/influxdb/services/udp"
"github.com/influxdb/influxdb/tsdb"
)
// Config represents the configuration format for the influxd binary.
type Config struct {
Meta *meta.Config `toml:"meta"`
Data tsdb.Config `toml:"data"`
Cluster cluster.Config `toml:"cluster"`
Retention retention.Config `toml:"retention"`
Precreator precreator.Config `toml:"shard-precreation"`
Admin admin.Config `toml:"admin"`
HTTPD httpd.Config `toml:"http"`
Graphites []graphite.Config `toml:"graphite"`
Collectd collectd.Config `toml:"collectd"`
OpenTSDB opentsdb.Config `toml:"opentsdb"`
UDPs []udp.Config `toml:"udp"`
// Snapshot SnapshotConfig `toml:"snapshot"`
Monitoring monitor.Config `toml:"monitoring"`
ContinuousQuery continuous_querier.Config `toml:"continuous_queries"`
HintedHandoff hh.Config `toml:"hinted-handoff"`
// Server reporting
ReportingDisabled bool `toml:"reporting-disabled"`
}
// NewConfig returns an instance of Config with reasonable defaults.
func NewConfig() *Config {
c := &Config{}
c.Meta = meta.NewConfig()
c.Data = tsdb.NewConfig()
c.Cluster = cluster.NewConfig()
c.Precreator = precreator.NewConfig()
c.Admin = admin.NewConfig()
c.HTTPD = httpd.NewConfig()
c.Collectd = collectd.NewConfig()
c.OpenTSDB = opentsdb.NewConfig()
c.Graphites = append(c.Graphites, graphite.NewConfig())
c.Monitoring = monitor.NewConfig()
c.ContinuousQuery = continuous_querier.NewConfig()
c.Retention = retention.NewConfig()
c.HintedHandoff = hh.NewConfig()
return c
}
// NewDemoConfig returns the config that runs when no config is specified.
func NewDemoConfig() (*Config, error) {
c := NewConfig()
var homeDir string
// By default, store meta and data files in current users home directory
u, err := user.Current()
if err == nil {
homeDir = u.HomeDir
} else if os.Getenv("HOME") != "" {
homeDir = os.Getenv("HOME")
} else {
return nil, fmt.Errorf("failed to determine current user for storage")
}
c.Meta.Dir = filepath.Join(homeDir, ".influxdb/meta")
c.Data.Dir = filepath.Join(homeDir, ".influxdb/data")
c.HintedHandoff.Dir = filepath.Join(homeDir, ".influxdb/hh")
c.Data.WALDir = filepath.Join(homeDir, ".influxdb/wal")
c.Admin.Enabled = true
c.Monitoring.Enabled = false
return c, nil
}
// Validate returns an error if the config is invalid.
func (c *Config) Validate() error {
if c.Meta.Dir == "" {
return errors.New("Meta.Dir must be specified")
} else if c.Data.Dir == "" {
return errors.New("Data.Dir must be specified")
} else if c.HintedHandoff.Dir == "" {
return errors.New("HintedHandoff.Dir must be specified")
} else if c.Data.WALDir == "" {
return errors.New("Data.WALDir must be specified")
}
for _, g := range c.Graphites {
if err := g.Validate(); err != nil {
return fmt.Errorf("invalid graphite config: %v", err)
}
}
return nil
}
func (c *Config) ApplyEnvOverrides() error {
return c.applyEnvOverrides("INFLUXDB", reflect.ValueOf(c))
}
func (c *Config) applyEnvOverrides(prefix string, spec reflect.Value) error {
// If we have a pointer, dereference it
s := spec
if spec.Kind() == reflect.Ptr {
s = spec.Elem()
}
// Make sure we have struct
if s.Kind() != reflect.Struct {
return nil
}
typeOfSpec := s.Type()
for i := 0; i < s.NumField(); i++ {
f := s.Field(i)
// Get the toml tag to determine what env var name to use
configName := typeOfSpec.Field(i).Tag.Get("toml")
// Replace hyphens with underscores to avoid issues with shells
configName = strings.Replace(configName, "-", "_", -1)
fieldName := typeOfSpec.Field(i).Name
// Skip any fields that we cannot set
if f.CanSet() || f.Kind() == reflect.Slice {
// Use the upper-case prefix and toml name for the env var
key := strings.ToUpper(configName)
if prefix != "" {
key = strings.ToUpper(fmt.Sprintf("%s_%s", prefix, configName))
}
value := os.Getenv(key)
// If the type is s slice, apply to each using the index as a suffix
// e.g. GRAPHITE_0
if f.Kind() == reflect.Slice || f.Kind() == reflect.Array {
for i := 0; i < f.Len(); i++ {
if err := c.applyEnvOverrides(fmt.Sprintf("%s_%d", key, i), f.Index(i)); err != nil {
return err
}
}
continue
}
// If it's a sub-config, recursively apply
if f.Kind() == reflect.Struct || f.Kind() == reflect.Ptr {
if err := c.applyEnvOverrides(key, f); err != nil {
return err
}
continue
}
// Skip any fields we don't have a value to set
if value == "" {
continue
}
switch f.Kind() {
case reflect.String:
f.SetString(value)
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
var intValue int64
// Handle toml.Duration
if f.Type().Name() == "Duration" {
dur, err := time.ParseDuration(value)
if err != nil {
return fmt.Errorf("failed to apply %v to %v using type %v and value '%v'", key, fieldName, f.Type().String(), value)
}
intValue = dur.Nanoseconds()
} else {
var err error
intValue, err = strconv.ParseInt(value, 0, f.Type().Bits())
if err != nil {
return fmt.Errorf("failed to apply %v to %v using type %v and value '%v'", key, fieldName, f.Type().String(), value)
}
}
f.SetInt(intValue)
case reflect.Bool:
boolValue, err := strconv.ParseBool(value)
if err != nil {
return fmt.Errorf("failed to apply %v to %v using type %v and value '%v'", key, fieldName, f.Type().String(), value)
}
f.SetBool(boolValue)
case reflect.Float32, reflect.Float64:
floatValue, err := strconv.ParseFloat(value, f.Type().Bits())
if err != nil {
return fmt.Errorf("failed to apply %v to %v using type %v and value '%v'", key, fieldName, f.Type().String(), value)
}
f.SetFloat(floatValue)
default:
if err := c.applyEnvOverrides(key, f); err != nil {
return err
}
}
}
}
return nil
}

73
Godeps/_workspace/src/github.com/influxdb/influxdb/cmd/influxd/run/config_command.go generated vendored
View File

@ -1,73 +0,0 @@
package run
import (
"flag"
"fmt"
"io"
"os"
"github.com/BurntSushi/toml"
)
// PrintConfigCommand represents the command executed by "influxd config".
type PrintConfigCommand struct {
Stdin io.Reader
Stdout io.Writer
Stderr io.Writer
}
// NewPrintConfigCommand return a new instance of PrintConfigCommand.
func NewPrintConfigCommand() *PrintConfigCommand {
return &PrintConfigCommand{
Stdin: os.Stdin,
Stdout: os.Stdout,
Stderr: os.Stderr,
}
}
// Run parses and prints the current config loaded.
func (cmd *PrintConfigCommand) Run(args ...string) error {
// Parse command flags.
fs := flag.NewFlagSet("", flag.ContinueOnError)
configPath := fs.String("config", "", "")
hostname := fs.String("hostname", "", "")
fs.Usage = func() { fmt.Fprintln(cmd.Stderr, printConfigUsage) }
if err := fs.Parse(args); err != nil {
return err
}
// Parse config from path.
config, err := cmd.parseConfig(*configPath)
if err != nil {
return fmt.Errorf("parse config: %s", err)
}
// Override config properties.
if *hostname != "" {
config.Meta.Hostname = *hostname
}
toml.NewEncoder(cmd.Stdout).Encode(config)
fmt.Fprint(cmd.Stdout, "\n")
return nil
}
// ParseConfig parses the config at path.
// Returns a demo configuration if path is blank.
func (cmd *PrintConfigCommand) parseConfig(path string) (*Config, error) {
if path == "" {
return NewDemoConfig()
}
config := NewConfig()
if _, err := toml.DecodeFile(path, &config); err != nil {
return nil, err
}
return config, nil
}
var printConfigUsage = `usage: config
config displays the default configuration
`

144
Godeps/_workspace/src/github.com/influxdb/influxdb/cmd/influxd/run/config_test.go generated vendored
View File

@ -1,144 +0,0 @@
package run_test
import (
"os"
"testing"
"github.com/BurntSushi/toml"
"github.com/influxdb/influxdb/cmd/influxd/run"
)
// Ensure the configuration can be parsed.
func TestConfig_Parse(t *testing.T) {
// Parse configuration.
var c run.Config
if _, err := toml.Decode(`
[meta]
dir = "/tmp/meta"
[data]
dir = "/tmp/data"
[cluster]
[admin]
bind-address = ":8083"
[http]
bind-address = ":8087"
[[graphite]]
protocol = "udp"
[[graphite]]
protocol = "tcp"
[collectd]
bind-address = ":1000"
[opentsdb]
bind-address = ":2000"
[[udp]]
bind-address = ":4444"
[monitoring]
enabled = true
[continuous_queries]
enabled = true
`, &c); err != nil {
t.Fatal(err)
}
// Validate configuration.
if c.Meta.Dir != "/tmp/meta" {
t.Fatalf("unexpected meta dir: %s", c.Meta.Dir)
} else if c.Data.Dir != "/tmp/data" {
t.Fatalf("unexpected data dir: %s", c.Data.Dir)
} else if c.Admin.BindAddress != ":8083" {
t.Fatalf("unexpected admin bind address: %s", c.Admin.BindAddress)
} else if c.HTTPD.BindAddress != ":8087" {
t.Fatalf("unexpected api bind address: %s", c.HTTPD.BindAddress)
} else if len(c.Graphites) != 2 {
t.Fatalf("unexpected graphites count: %d", len(c.Graphites))
} else if c.Graphites[0].Protocol != "udp" {
t.Fatalf("unexpected graphite protocol(0): %s", c.Graphites[0].Protocol)
} else if c.Graphites[1].Protocol != "tcp" {
t.Fatalf("unexpected graphite protocol(1): %s", c.Graphites[1].Protocol)
} else if c.Collectd.BindAddress != ":1000" {
t.Fatalf("unexpected collectd bind address: %s", c.Collectd.BindAddress)
} else if c.OpenTSDB.BindAddress != ":2000" {
t.Fatalf("unexpected opentsdb bind address: %s", c.OpenTSDB.BindAddress)
} else if c.UDPs[0].BindAddress != ":4444" {
t.Fatalf("unexpected udp bind address: %s", c.UDPs[0].BindAddress)
} else if c.Monitoring.Enabled != true {
t.Fatalf("unexpected monitoring enabled: %v", c.Monitoring.Enabled)
} else if c.ContinuousQuery.Enabled != true {
t.Fatalf("unexpected continuous query enabled: %v", c.ContinuousQuery.Enabled)
}
}
// Ensure the configuration can be parsed.
func TestConfig_Parse_EnvOverride(t *testing.T) {
// Parse configuration.
var c run.Config
if _, err := toml.Decode(`
[meta]
dir = "/tmp/meta"
[data]
dir = "/tmp/data"
[cluster]
[admin]
bind-address = ":8083"
[http]
bind-address = ":8087"
[[graphite]]
protocol = "udp"
[[graphite]]
protocol = "tcp"
[collectd]
bind-address = ":1000"
[opentsdb]
bind-address = ":2000"
[[udp]]
bind-address = ":4444"
[monitoring]
enabled = true
[continuous_queries]
enabled = true
`, &c); err != nil {
t.Fatal(err)
}
if err := os.Setenv("INFLUXDB_UDP_BIND_ADDRESS", ":1234"); err != nil {
t.Fatalf("failed to set env var: %v", err)
}
if err := os.Setenv("INFLUXDB_GRAPHITE_1_PROTOCOL", "udp"); err != nil {
t.Fatalf("failed to set env var: %v", err)
}
if err := c.ApplyEnvOverrides(); err != nil {
t.Fatalf("failed to apply env overrides: %v", err)
}
if c.UDPs[0].BindAddress != ":4444" {
t.Fatalf("unexpected udp bind address: %s", c.UDPs[0].BindAddress)
}
if c.Graphites[1].Protocol != "udp" {
t.Fatalf("unexpected graphite protocol(0): %s", c.Graphites[0].Protocol)
}
}

536
Godeps/_workspace/src/github.com/influxdb/influxdb/cmd/influxd/run/server.go generated vendored
View File

@ -1,536 +0,0 @@
package run
import (
"bytes"
"fmt"
"log"
"net"
"net/http"
"os"
"runtime"
"runtime/pprof"
"strings"
"time"
"github.com/influxdb/influxdb/cluster"
"github.com/influxdb/influxdb/meta"
"github.com/influxdb/influxdb/services/admin"
"github.com/influxdb/influxdb/services/collectd"
"github.com/influxdb/influxdb/services/continuous_querier"
"github.com/influxdb/influxdb/services/graphite"
"github.com/influxdb/influxdb/services/hh"
"github.com/influxdb/influxdb/services/httpd"
"github.com/influxdb/influxdb/services/opentsdb"
"github.com/influxdb/influxdb/services/precreator"
"github.com/influxdb/influxdb/services/retention"
"github.com/influxdb/influxdb/services/snapshotter"
"github.com/influxdb/influxdb/services/udp"
"github.com/influxdb/influxdb/tcp"
"github.com/influxdb/influxdb/tsdb"
_ "github.com/influxdb/influxdb/tsdb/engine"
)
// Server represents a container for the metadata and storage data and services.
// It is built using a Config and it manages the startup and shutdown of all
// services in the proper order.
type Server struct {
version string // Build version
err chan error
closing chan struct{}
Hostname string
BindAddress string
Listener net.Listener
MetaStore *meta.Store
TSDBStore *tsdb.Store
QueryExecutor *tsdb.QueryExecutor
PointsWriter *cluster.PointsWriter
ShardWriter *cluster.ShardWriter
ShardMapper *cluster.ShardMapper
HintedHandoff *hh.Service
Services []Service
// These references are required for the tcp muxer.
ClusterService *cluster.Service
SnapshotterService *snapshotter.Service
// Server reporting
reportingDisabled bool
// Profiling
CPUProfile string
MemProfile string
}
// NewServer returns a new instance of Server built from a config.
func NewServer(c *Config, version string) (*Server, error) {
// Construct base meta store and data store.
tsdbStore := tsdb.NewStore(c.Data.Dir)
tsdbStore.EngineOptions.Config = c.Data
s := &Server{
version: version,
err: make(chan error),
closing: make(chan struct{}),
Hostname: c.Meta.Hostname,
BindAddress: c.Meta.BindAddress,
MetaStore: meta.NewStore(c.Meta),
TSDBStore: tsdbStore,
reportingDisabled: c.ReportingDisabled,
}
// Copy TSDB configuration.
s.TSDBStore.EngineOptions.MaxWALSize = c.Data.MaxWALSize
s.TSDBStore.EngineOptions.WALFlushInterval = time.Duration(c.Data.WALFlushInterval)
s.TSDBStore.EngineOptions.WALPartitionFlushDelay = time.Duration(c.Data.WALPartitionFlushDelay)
// Set the shard mapper
s.ShardMapper = cluster.NewShardMapper(time.Duration(c.Cluster.ShardMapperTimeout))
s.ShardMapper.ForceRemoteMapping = c.Cluster.ForceRemoteShardMapping
s.ShardMapper.MetaStore = s.MetaStore
s.ShardMapper.TSDBStore = s.TSDBStore
// Initialize query executor.
s.QueryExecutor = tsdb.NewQueryExecutor(s.TSDBStore)
s.QueryExecutor.MetaStore = s.MetaStore
s.QueryExecutor.MetaStatementExecutor = &meta.StatementExecutor{Store: s.MetaStore}
s.QueryExecutor.ShardMapper = s.ShardMapper
// Set the shard writer
s.ShardWriter = cluster.NewShardWriter(time.Duration(c.Cluster.ShardWriterTimeout))
s.ShardWriter.MetaStore = s.MetaStore
// Create the hinted handoff service
s.HintedHandoff = hh.NewService(c.HintedHandoff, s.ShardWriter)
// Initialize points writer.
s.PointsWriter = cluster.NewPointsWriter()
s.PointsWriter.WriteTimeout = time.Duration(c.Cluster.WriteTimeout)
s.PointsWriter.MetaStore = s.MetaStore
s.PointsWriter.TSDBStore = s.TSDBStore
s.PointsWriter.ShardWriter = s.ShardWriter
s.PointsWriter.HintedHandoff = s.HintedHandoff
// Append services.
s.appendClusterService(c.Cluster)
s.appendPrecreatorService(c.Precreator)
s.appendSnapshotterService()
s.appendAdminService(c.Admin)
s.appendContinuousQueryService(c.ContinuousQuery)
s.appendHTTPDService(c.HTTPD)
s.appendCollectdService(c.Collectd)
if err := s.appendOpenTSDBService(c.OpenTSDB); err != nil {
return nil, err
}
for _, g := range c.UDPs {
s.appendUDPService(g)
}
s.appendRetentionPolicyService(c.Retention)
for _, g := range c.Graphites {
if err := s.appendGraphiteService(g); err != nil {
return nil, err
}
}
return s, nil
}
func (s *Server) appendClusterService(c cluster.Config) {
srv := cluster.NewService(c)
srv.TSDBStore = s.TSDBStore
srv.MetaStore = s.MetaStore
s.Services = append(s.Services, srv)
s.ClusterService = srv
}
func (s *Server) appendSnapshotterService() {
srv := snapshotter.NewService()
srv.TSDBStore = s.TSDBStore
srv.MetaStore = s.MetaStore
s.Services = append(s.Services, srv)
s.SnapshotterService = srv
}
func (s *Server) appendRetentionPolicyService(c retention.Config) {
if !c.Enabled {
return
}
srv := retention.NewService(c)
srv.MetaStore = s.MetaStore
srv.TSDBStore = s.TSDBStore
s.Services = append(s.Services, srv)
}
func (s *Server) appendAdminService(c admin.Config) {
if !c.Enabled {
return
}
srv := admin.NewService(c)
s.Services = append(s.Services, srv)
}
func (s *Server) appendHTTPDService(c httpd.Config) {
if !c.Enabled {
return
}
srv := httpd.NewService(c)
srv.Handler.MetaStore = s.MetaStore
srv.Handler.QueryExecutor = s.QueryExecutor
srv.Handler.PointsWriter = s.PointsWriter
srv.Handler.Version = s.version
// If a ContinuousQuerier service has been started, attach it.
for _, srvc := range s.Services {
if cqsrvc, ok := srvc.(continuous_querier.ContinuousQuerier); ok {
srv.Handler.ContinuousQuerier = cqsrvc
}
}
s.Services = append(s.Services, srv)
}
func (s *Server) appendCollectdService(c collectd.Config) {
if !c.Enabled {
return
}
srv := collectd.NewService(c)
srv.MetaStore = s.MetaStore
srv.PointsWriter = s.PointsWriter
s.Services = append(s.Services, srv)
}
func (s *Server) appendOpenTSDBService(c opentsdb.Config) error {
if !c.Enabled {
return nil
}
srv, err := opentsdb.NewService(c)
if err != nil {
return err
}
srv.PointsWriter = s.PointsWriter
srv.MetaStore = s.MetaStore
s.Services = append(s.Services, srv)
return nil
}
func (s *Server) appendGraphiteService(c graphite.Config) error {
if !c.Enabled {
return nil
}
srv, err := graphite.NewService(c)
if err != nil {
return err
}
srv.PointsWriter = s.PointsWriter
srv.MetaStore = s.MetaStore
s.Services = append(s.Services, srv)
return nil
}
func (s *Server) appendPrecreatorService(c precreator.Config) error {
if !c.Enabled {
return nil
}
srv, err := precreator.NewService(c)
if err != nil {
return err
}
srv.MetaStore = s.MetaStore
s.Services = append(s.Services, srv)
return nil
}
func (s *Server) appendUDPService(c udp.Config) {
if !c.Enabled {
return
}
srv := udp.NewService(c)
srv.PointsWriter = s.PointsWriter
s.Services = append(s.Services, srv)
}
func (s *Server) appendContinuousQueryService(c continuous_querier.Config) {
if !c.Enabled {
return
}
srv := continuous_querier.NewService(c)
srv.MetaStore = s.MetaStore
srv.QueryExecutor = s.QueryExecutor
srv.PointsWriter = s.PointsWriter
s.Services = append(s.Services, srv)
}
// Err returns an error channel that multiplexes all out of band errors received from all services.
func (s *Server) Err() <-chan error { return s.err }
// Open opens the meta and data store and all services.
func (s *Server) Open() error {
if err := func() error {
// Start profiling, if set.
startProfile(s.CPUProfile, s.MemProfile)
host, port, err := s.hostAddr()
if err != nil {
return err
}
hostport := net.JoinHostPort(host, port)
addr, err := net.ResolveTCPAddr("tcp", hostport)
if err != nil {
return fmt.Errorf("resolve tcp: addr=%s, err=%s", hostport, err)
}
s.MetaStore.Addr = addr
s.MetaStore.RemoteAddr = &tcpaddr{hostport}
// Open shared TCP connection.
ln, err := net.Listen("tcp", s.BindAddress)
if err != nil {
return fmt.Errorf("listen: %s", err)
}
s.Listener = ln
// The port 0 is used, we need to retrieve the port assigned by the kernel
if strings.HasSuffix(s.BindAddress, ":0") {
s.MetaStore.Addr = ln.Addr()
}
// Multiplex listener.
mux := tcp.NewMux()
s.MetaStore.RaftListener = mux.Listen(meta.MuxRaftHeader)
s.MetaStore.ExecListener = mux.Listen(meta.MuxExecHeader)
s.MetaStore.RPCListener = mux.Listen(meta.MuxRPCHeader)
s.ClusterService.Listener = mux.Listen(cluster.MuxHeader)
s.SnapshotterService.Listener = mux.Listen(snapshotter.MuxHeader)
go mux.Serve(ln)
// Open meta store.
if err := s.MetaStore.Open(); err != nil {
return fmt.Errorf("open meta store: %s", err)
}
go s.monitorErrorChan(s.MetaStore.Err())
// Wait for the store to initialize.
<-s.MetaStore.Ready()
// Open TSDB store.
if err := s.TSDBStore.Open(); err != nil {
return fmt.Errorf("open tsdb store: %s", err)
}
// Open the hinted handoff service
if err := s.HintedHandoff.Open(); err != nil {
return fmt.Errorf("open hinted handoff: %s", err)
}
for _, service := range s.Services {
if err := service.Open(); err != nil {
return fmt.Errorf("open service: %s", err)
}
}
// Start the reporting service, if not disabled.
if !s.reportingDisabled {
go s.startServerReporting()
}
return nil
}(); err != nil {
s.Close()
return err
}
return nil
}
// Close shuts down the meta and data stores and all services.
func (s *Server) Close() error {
stopProfile()
if s.Listener != nil {
s.Listener.Close()
}
if s.MetaStore != nil {
s.MetaStore.Close()
}
if s.TSDBStore != nil {
s.TSDBStore.Close()
}
if s.HintedHandoff != nil {
s.HintedHandoff.Close()
}
for _, service := range s.Services {
service.Close()
}
close(s.closing)
return nil
}
// startServerReporting starts periodic server reporting.
func (s *Server) startServerReporting() {
for {
select {
case <-s.closing:
return
default:
}
if err := s.MetaStore.WaitForLeader(30 * time.Second); err != nil {
log.Printf("no leader available for reporting: %s", err.Error())
time.Sleep(time.Second)
continue
}
s.reportServer()
<-time.After(24 * time.Hour)
}
}
// reportServer reports anonymous statistics about the system.
func (s *Server) reportServer() {
dis, err := s.MetaStore.Databases()
if err != nil {
log.Printf("failed to retrieve databases for reporting: %s", err.Error())
return
}
numDatabases := len(dis)
numMeasurements := 0
numSeries := 0
for _, di := range dis {
d := s.TSDBStore.DatabaseIndex(di.Name)
if d == nil {
// No data in this store for this database.
continue
}
m, s := d.MeasurementSeriesCounts()
numMeasurements += m
numSeries += s
}
clusterID, err := s.MetaStore.ClusterID()
if err != nil {
log.Printf("failed to retrieve cluster ID for reporting: %s", err.Error())
return
}
json := fmt.Sprintf(`[{
"name":"reports",
"columns":["os", "arch", "version", "server_id", "cluster_id", "num_series", "num_measurements", "num_databases"],
"points":[["%s", "%s", "%s", "%x", "%x", "%d", "%d", "%d"]]
}]`, runtime.GOOS, runtime.GOARCH, s.version, s.MetaStore.NodeID(), clusterID, numSeries, numMeasurements, numDatabases)
data := bytes.NewBufferString(json)
log.Printf("Sending anonymous usage statistics to m.influxdb.com")
client := http.Client{Timeout: time.Duration(5 * time.Second)}
go client.Post("http://m.influxdb.com:8086/db/reporting/series?u=reporter&p=influxdb", "application/json", data)
}
// monitorErrorChan reads an error channel and resends it through the server.
func (s *Server) monitorErrorChan(ch <-chan error) {
for {
select {
case err, ok := <-ch:
if !ok {
return
}
s.err <- err
case <-s.closing:
return
}
}
}
// hostAddr returns the host and port that remote nodes will use to reach this
// node.
func (s *Server) hostAddr() (string, string, error) {
// Resolve host to address.
_, port, err := net.SplitHostPort(s.BindAddress)
if err != nil {
return "", "", fmt.Errorf("split bind address: %s", err)
}
host := s.Hostname
// See if we might have a port that will override the BindAddress port
if host != "" && host[len(host)-1] >= '0' && host[len(host)-1] <= '9' && strings.Contains(host, ":") {
hostArg, portArg, err := net.SplitHostPort(s.Hostname)
if err != nil {
return "", "", err
}
if hostArg != "" {
host = hostArg
}
if portArg != "" {
port = portArg
}
}
return host, port, nil
}
// Service represents a service attached to the server.
type Service interface {
Open() error
Close() error
}
// prof stores the file locations of active profiles.
var prof struct {
cpu *os.File
mem *os.File
}
// StartProfile initializes the cpu and memory profile, if specified.
func startProfile(cpuprofile, memprofile string) {
if cpuprofile != "" {
f, err := os.Create(cpuprofile)
if err != nil {
log.Fatalf("cpuprofile: %v", err)
}
log.Printf("writing CPU profile to: %s\n", cpuprofile)
prof.cpu = f
pprof.StartCPUProfile(prof.cpu)
}
if memprofile != "" {
f, err := os.Create(memprofile)
if err != nil {
log.Fatalf("memprofile: %v", err)
}
log.Printf("writing mem profile to: %s\n", memprofile)
prof.mem = f
runtime.MemProfileRate = 4096
}
}
// StopProfile closes the cpu and memory profiles if they are running.
func stopProfile() {
if prof.cpu != nil {
pprof.StopCPUProfile()
prof.cpu.Close()
log.Println("CPU profile stopped")
}
if prof.mem != nil {
pprof.Lookup("heap").WriteTo(prof.mem, 0)
prof.mem.Close()
log.Println("mem profile stopped")
}
}
type tcpaddr struct{ host string }
func (a *tcpaddr) Network() string { return "tcp" }
func (a *tcpaddr) String() string { return a.host }

312
Godeps/_workspace/src/github.com/influxdb/influxdb/cmd/influxd/run/server_helpers_test.go generated vendored
View File

@ -1,312 +0,0 @@
// This package is a set of convenience helpers and structs to make integration testing easier
package run_test
import (
"encoding/json"
"fmt"
"io"
"io/ioutil"
"log"
"math"
"net/http"
"net/url"
"os"
"regexp"
"strings"
"testing"
"time"
"github.com/influxdb/influxdb/cmd/influxd/run"
"github.com/influxdb/influxdb/meta"
"github.com/influxdb/influxdb/services/httpd"
"github.com/influxdb/influxdb/toml"
)
// Server represents a test wrapper for run.Server.
type Server struct {
*run.Server
Config *run.Config
}
// NewServer returns a new instance of Server.
func NewServer(c *run.Config) *Server {
srv, _ := run.NewServer(c, "testServer")
s := Server{
Server: srv,
Config: c,
}
s.TSDBStore.EngineOptions.Config = c.Data
configureLogging(&s)
return &s
}
// OpenServer opens a test server.
func OpenServer(c *run.Config, joinURLs string) *Server {
s := NewServer(c)
configureLogging(s)
if err := s.Open(); err != nil {
panic(err.Error())
}
return s
}
// OpenServerWithVersion opens a test server with a specific version.
func OpenServerWithVersion(c *run.Config, version string) *Server {
srv, _ := run.NewServer(c, version)
s := Server{
Server: srv,
Config: c,
}
configureLogging(&s)
if err := s.Open(); err != nil {
panic(err.Error())
}
return &s
}
// Close shuts down the server and removes all temporary paths.
func (s *Server) Close() {
os.RemoveAll(s.Config.Meta.Dir)
os.RemoveAll(s.Config.Data.Dir)
os.RemoveAll(s.Config.HintedHandoff.Dir)
s.Server.Close()
}
// URL returns the base URL for the httpd endpoint.
func (s *Server) URL() string {
for _, service := range s.Services {
if service, ok := service.(*httpd.Service); ok {
return "http://" + service.Addr().String()
}
}
panic("httpd server not found in services")
}
// CreateDatabaseAndRetentionPolicy will create the database and retention policy.
func (s *Server) CreateDatabaseAndRetentionPolicy(db string, rp *meta.RetentionPolicyInfo) error {
if _, err := s.MetaStore.CreateDatabase(db); err != nil {
return err
} else if _, err := s.MetaStore.CreateRetentionPolicy(db, rp); err != nil {
return err
}
return nil
}
// Query executes a query against the server and returns the results.
func (s *Server) Query(query string) (results string, err error) {
return s.QueryWithParams(query, nil)
}
// Query executes a query against the server and returns the results.
func (s *Server) QueryWithParams(query string, values url.Values) (results string, err error) {
if values == nil {
values = url.Values{}
}
values.Set("q", query)
resp, err := http.Get(s.URL() + "/query?" + values.Encode())
if err != nil {
return "", err
//} else if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusBadRequest {
}
body := string(MustReadAll(resp.Body))
switch resp.StatusCode {
case http.StatusBadRequest:
if !expectPattern(".*error parsing query*.", body) {
return "", fmt.Errorf("unexpected status code: code=%d, body=%s", resp.StatusCode, body)
}
return body, nil
case http.StatusOK:
return body, nil
default:
return "", fmt.Errorf("unexpected status code: code=%d, body=%s", resp.StatusCode, body)
}
}
// Write executes a write against the server and returns the results.
func (s *Server) Write(db, rp, body string, params url.Values) (results string, err error) {
if params == nil {
params = url.Values{}
}
if params.Get("db") == "" {
params.Set("db", db)
}
if params.Get("rp") == "" {
params.Set("rp", rp)
}
resp, err := http.Post(s.URL()+"/write?"+params.Encode(), "", strings.NewReader(body))
if err != nil {
return "", err
} else if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusNoContent {
return "", fmt.Errorf("invalid status code: code=%d, body=%s", resp.StatusCode, MustReadAll(resp.Body))
}
return string(MustReadAll(resp.Body)), nil
}
// NewConfig returns the default config with temporary paths.
func NewConfig() *run.Config {
c := run.NewConfig()
c.ReportingDisabled = true
c.Meta.Dir = MustTempFile()
c.Meta.BindAddress = "127.0.0.1:0"
c.Meta.HeartbeatTimeout = toml.Duration(50 * time.Millisecond)
c.Meta.ElectionTimeout = toml.Duration(50 * time.Millisecond)
c.Meta.LeaderLeaseTimeout = toml.Duration(50 * time.Millisecond)
c.Meta.CommitTimeout = toml.Duration(5 * time.Millisecond)
c.Data.Dir = MustTempFile()
c.Data.WALDir = MustTempFile()
c.HintedHandoff.Dir = MustTempFile()
c.HTTPD.Enabled = true
c.HTTPD.BindAddress = "127.0.0.1:0"
c.HTTPD.LogEnabled = testing.Verbose()
return c
}
func newRetentionPolicyInfo(name string, rf int, duration time.Duration) *meta.RetentionPolicyInfo {
return &meta.RetentionPolicyInfo{Name: name, ReplicaN: rf, Duration: duration}
}
func maxFloat64() string {
maxFloat64, _ := json.Marshal(math.MaxFloat64)
return string(maxFloat64)
}
func maxInt64() string {
maxInt64, _ := json.Marshal(^int64(0))
return string(maxInt64)
}
func now() time.Time {
return time.Now().UTC()
}
func yesterday() time.Time {
return now().Add(-1 * time.Hour * 24)
}
func mustParseTime(layout, value string) time.Time {
tm, err := time.Parse(layout, value)
if err != nil {
panic(err)
}
return tm
}
// MustReadAll reads r. Panic on error.
func MustReadAll(r io.Reader) []byte {
b, err := ioutil.ReadAll(r)
if err != nil {
panic(err)
}
return b
}
// MustTempFile returns a path to a temporary file.
func MustTempFile() string {
f, err := ioutil.TempFile("", "influxd-")
if err != nil {
panic(err)
}
f.Close()
os.Remove(f.Name())
return f.Name()
}
func expectPattern(exp, act string) bool {
re := regexp.MustCompile(exp)
if !re.MatchString(act) {
return false
}
return true
}
type Query struct {
name string
command string
params url.Values
exp, act string
pattern bool
skip bool
}
// Execute runs the command and returns an err if it fails
func (q *Query) Execute(s *Server) (err error) {
if q.params == nil {
q.act, err = s.Query(q.command)
return
}
q.act, err = s.QueryWithParams(q.command, q.params)
return
}
func (q *Query) success() bool {
if q.pattern {
return expectPattern(q.exp, q.act)
}
return q.exp == q.act
}
func (q *Query) Error(err error) string {
return fmt.Sprintf("%s: %v", q.name, err)
}
func (q *Query) failureMessage() string {
return fmt.Sprintf("%s: unexpected results\nquery: %s\nexp: %s\nactual: %s\n", q.name, q.command, q.exp, q.act)
}
type Test struct {
initialized bool
write string
params url.Values
db string
rp string
exp string
queries []*Query
}
func NewTest(db, rp string) Test {
return Test{
db: db,
rp: rp,
}
}
func (t *Test) addQueries(q ...*Query) {
t.queries = append(t.queries, q...)
}
func (t *Test) init(s *Server) error {
if t.write == "" || t.initialized {
return nil
}
t.initialized = true
if res, err := s.Write(t.db, t.rp, t.write, t.params); err != nil {
return err
} else if t.exp != res {
return fmt.Errorf("unexpected results\nexp: %s\ngot: %s\n", t.exp, res)
}
return nil
}
func configureLogging(s *Server) {
// Set the logger to discard unless verbose is on
if !testing.Verbose() {
type logSetter interface {
SetLogger(*log.Logger)
}
nullLogger := log.New(ioutil.Discard, "", 0)
s.MetaStore.Logger = nullLogger
s.TSDBStore.Logger = nullLogger
s.HintedHandoff.SetLogger(nullLogger)
for _, service := range s.Services {
if service, ok := service.(logSetter); ok {
service.SetLogger(nullLogger)
}
}
}
}

3719
Godeps/_workspace/src/github.com/influxdb/influxdb/cmd/influxd/run/server_test.go generated vendored
View File

File diff suppressed because it is too large Load Diff

150
Godeps/_workspace/src/github.com/influxdb/influxdb/cmd/influxd/run/server_test.md generated vendored
View File

@ -1,150 +0,0 @@
# Server Integration Tests
Currently, the file `server_test.go` has integration tests for single node scenarios.
At some point we'll need to add cluster tests, and may add them in a different file, or
rename `server_test.go` to `server_single_node_test.go` or something like that.
## What is in a test?
Each test is broken apart effectively into the following areas:
- Write sample data
- Use cases for table driven test, that include a command (typically a query) and an expected result.
When each test runs it does the following:
- init: determines if there are any writes and if so, writes them to the in-memory database
- queries: iterate through each query, executing the command, and comparing the results to the expected result.
## Idempotent - Allows for parallel tests
Each test should be `idempotent`, meaining that its data will not be affected by other tests, or use cases within the table tests themselves.
This allows for parallel testing, keeping the test suite total execution time very low.
### Basic sample test
```go
// Ensure the server can have a database with multiple measurements.
func TestServer_Query_Multiple_Measurements(t *testing.T) {
t.Parallel()
s := OpenServer(NewConfig(), "")
defer s.Close()
if err := s.CreateDatabaseAndRetentionPolicy("db0", newRetentionPolicyInfo("rp0", 1, 1*time.Hour)); err != nil {
t.Fatal(err)
}
// Make sure we do writes for measurements that will span across shards
writes := []string{
fmt.Sprintf("cpu,host=server01 value=100,core=4 %d", mustParseTime(time.RFC3339Nano, "2000-01-01T00:00:00Z").UnixNano()),
fmt.Sprintf("cpu1,host=server02 value=50,core=2 %d", mustParseTime(time.RFC3339Nano, "2015-01-01T00:00:00Z").UnixNano()),
}
test := NewTest("db0", "rp0")
test.write = strings.Join(writes, "\n")
test.addQueries([]*Query{
&Query{
name: "measurement in one shard but not another shouldn't panic server",
command: `SELECT host,value FROM db0.rp0.cpu`,
exp: `{"results":[{"series":[{"name":"cpu","tags":{"host":"server01"},"columns":["time","value"],"values":[["2000-01-01T00:00:00Z",100]]}]}]}`,
},
}...)
if err := test.init(s); err != nil {
t.Fatalf("test init failed: %s", err)
}
for _, query := range test.queries {
if query.skip {
t.Logf("SKIP:: %s", query.name)
continue
}
if err := query.Execute(s); err != nil {
t.Error(query.Error(err))
} else if !query.success() {
t.Error(query.failureMessage())
}
}
}
```
Let's break this down:
In this test, we first tell it to run in parallel with the `t.Parallel()` call.
We then open a new server with:
```go
s := OpenServer(NewConfig(), "")
defer s.Close()
```
If needed, we create a database and default retention policy. This is usually needed
when inserting and querying data. This is not needed if you are testing commands like `CREATE DATABASE`, `SHOW DIAGNOSTICS`, etc.
```go
if err := s.CreateDatabaseAndRetentionPolicy("db0", newRetentionPolicyInfo("rp0", 1, 1*time.Hour)); err != nil {
t.Fatal(err)
}
```
Next, set up the write data you need:
```go
writes := []string{
fmt.Sprintf("cpu,host=server01 value=100,core=4 %d", mustParseTime(time.RFC3339Nano, "2000-01-01T00:00:00Z").UnixNano()),
fmt.Sprintf("cpu1,host=server02 value=50,core=2 %d", mustParseTime(time.RFC3339Nano, "2015-01-01T00:00:00Z").UnixNano()),
}
```
Create a new test with the database and retention policy:
```go
test := NewTest("db0", "rp0")
```
Send in the writes:
```go
test.write = strings.Join(writes, "\n")
```
Add some queries (the second one is mocked out to show how to add more than one):
```go
test.addQueries([]*Query{
&Query{
name: "measurement in one shard but not another shouldn't panic server",
command: `SELECT host,value FROM db0.rp0.cpu`,
exp: `{"results":[{"series":[{"name":"cpu","tags":{"host":"server01"},"columns":["time","value"],"values":[["2000-01-01T00:00:00Z",100]]}]}]}`,
},
&Query{
name: "another test here...",
command: `Some query command`,
exp: `the expected results`,
},
}...)
```
The rest of the code is boilerplate execution code. It is purposefully not refactored out to a helper
to make sure the test failure reports the proper lines for debugging purposes.
#### Running the tests
To run the tests:
```sh
go test ./cmd/influxd/run -parallel 500 -timeout 10s
```
#### Running a specific test
```sh
go test ./cmd/influxd/run -parallel 500 -timeout 10s -run TestServer_Query_Fill
```
#### Verbose feedback
By default, all logs are silenced when testing. If you pass in the `-v` flag, the test suite becomes verbose, and enables all logging in the system
```sh
go test ./cmd/influxd/run -parallel 500 -timeout 10s -run TestServer_Query_Fill -v
```

143
Godeps/_workspace/src/github.com/influxdb/influxdb/diagnostics.go generated vendored
View File

@ -1,143 +0,0 @@
package influxdb
import (
"os"
"runtime"
"time"
"github.com/influxdb/influxdb/influxql"
)
// GoDiagnostics captures basic information about the runtime.
type GoDiagnostics struct {
GoMaxProcs int
NumGoroutine int
Version string
}
// NewGoDiagnostics returns a GoDiagnostics object.
func NewGoDiagnostics() *GoDiagnostics {
return &GoDiagnostics{
GoMaxProcs: runtime.GOMAXPROCS(0),
NumGoroutine: runtime.NumGoroutine(),
Version: runtime.Version(),
}
}
// AsRow returns the GoDiagnostic object as an InfluxQL row.
func (g *GoDiagnostics) AsRow(measurement string, tags map[string]string) *influxql.Row {
return &influxql.Row{
Name: measurement,
Columns: []string{"time", "goMaxProcs", "numGoRoutine", "version"},
Tags: tags,
Values: [][]interface{}{[]interface{}{time.Now().UTC(),
g.GoMaxProcs, g.NumGoroutine, g.Version}},
}
}
// SystemDiagnostics captures basic machine data.
type SystemDiagnostics struct {
Hostname string
PID int
OS string
Arch string
NumCPU int
}
// NewSystemDiagnostics returns a SystemDiagnostics object.
func NewSystemDiagnostics() *SystemDiagnostics {
hostname, err := os.Hostname()
if err != nil {
hostname = "unknown"
}
return &SystemDiagnostics{
Hostname: hostname,
PID: os.Getpid(),
OS: runtime.GOOS,
Arch: runtime.GOARCH,
NumCPU: runtime.NumCPU(),
}
}
// AsRow returns the GoDiagnostic object as an InfluxQL row.
func (s *SystemDiagnostics) AsRow(measurement string, tags map[string]string) *influxql.Row {
return &influxql.Row{
Name: measurement,
Columns: []string{"time", "hostname", "pid", "os", "arch", "numCPU"},
Tags: tags,
Values: [][]interface{}{[]interface{}{time.Now().UTC(),
s.Hostname, s.PID, s.OS, s.Arch, s.NumCPU}},
}
}
// MemoryDiagnostics captures Go memory stats.
type MemoryDiagnostics struct {
Alloc int64
TotalAlloc int64
Sys int64
Lookups int64
Mallocs int64
Frees int64
HeapAlloc int64
HeapSys int64
HeapIdle int64
HeapInUse int64
HeapReleased int64
HeapObjects int64
PauseTotalNs int64
NumGC int64
}
// NewMemoryDiagnostics returns a MemoryDiagnostics object.
func NewMemoryDiagnostics() *MemoryDiagnostics {
var m runtime.MemStats
runtime.ReadMemStats(&m)
return &MemoryDiagnostics{
Alloc: int64(m.Alloc),
TotalAlloc: int64(m.TotalAlloc),
Sys: int64(m.Sys),
Lookups: int64(m.Lookups),
Mallocs: int64(m.Mallocs),
Frees: int64(m.Frees),
HeapAlloc: int64(m.HeapAlloc),
HeapSys: int64(m.HeapSys),
HeapIdle: int64(m.HeapIdle),
HeapInUse: int64(m.HeapInuse),
HeapReleased: int64(m.HeapReleased),
HeapObjects: int64(m.HeapObjects),
PauseTotalNs: int64(m.PauseTotalNs),
NumGC: int64(m.NumGC),
}
}
// AsRow returns the MemoryDiagnostics object as an InfluxQL row.
func (m *MemoryDiagnostics) AsRow(measurement string, tags map[string]string) *influxql.Row {
return &influxql.Row{
Name: measurement,
Columns: []string{"time", "alloc", "totalAlloc", "sys", "lookups", "mallocs", "frees", "heapAlloc",
"heapSys", "heapIdle", "heapInUse", "heapReleased", "heapObjects", "pauseTotalNs", "numGG"},
Tags: tags,
Values: [][]interface{}{[]interface{}{time.Now().UTC(),
m.Alloc, m.TotalAlloc, m.Sys, m.Lookups, m.Mallocs, m.Frees, m.HeapAlloc,
m.HeapSys, m.HeapIdle, m.HeapInUse, m.HeapReleased, m.HeapObjects, m.PauseTotalNs, m.NumGC}},
}
}
// BuildDiagnostics capture basic build version information.
type BuildDiagnostics struct {
Version string
CommitHash string
}
// AsRow returns the BuildDiagnostics object as an InfluxQL row.
func (b *BuildDiagnostics) AsRow(measurement string, tags map[string]string) *influxql.Row {
return &influxql.Row{
Name: measurement,
Columns: []string{"time", "version", "commitHash"},
Tags: tags,
Values: [][]interface{}{[]interface{}{time.Now().UTC(),
b.Version, b.CommitHash}},
}
}

78
Godeps/_workspace/src/github.com/influxdb/influxdb/errors.go generated vendored
View File

@ -1,78 +0,0 @@
package influxdb
import (
"encoding/json"
"errors"
"fmt"
"runtime"
"strings"
)
var (
// ErrFieldsRequired is returned when a point does not any fields.
ErrFieldsRequired = errors.New("fields required")
// ErrFieldTypeConflict is returned when a new field already exists with a different type.
ErrFieldTypeConflict = errors.New("field type conflict")
)
func ErrDatabaseNotFound(name string) error { return fmt.Errorf("database not found: %s", name) }
func ErrMeasurementNotFound(name string) error { return fmt.Errorf("measurement not found: %s", name) }
func Errorf(format string, a ...interface{}) (err error) {
if _, file, line, ok := runtime.Caller(2); ok {
a = append(a, file, line)
err = fmt.Errorf(format+" (%s:%d)", a...)
} else {
err = fmt.Errorf(format, a...)
}
return
}
// IsClientError indicates whether an error is a known client error.
func IsClientError(err error) bool {
if err == nil {
return false
}
if err == ErrFieldsRequired {
return true
}
if err == ErrFieldTypeConflict {
return true
}
if strings.Contains(err.Error(), ErrFieldTypeConflict.Error()) {
return true
}
return false
}
// mustMarshal encodes a value to JSON.
// This will panic if an error occurs. This should only be used internally when
// an invalid marshal will cause corruption and a panic is appropriate.
func mustMarshalJSON(v interface{}) []byte {
b, err := json.Marshal(v)
if err != nil {
panic("marshal: " + err.Error())
}
return b
}
// mustUnmarshalJSON decodes a value from JSON.
// This will panic if an error occurs. This should only be used internally when
// an invalid unmarshal will cause corruption and a panic is appropriate.
func mustUnmarshalJSON(b []byte, v interface{}) {
if err := json.Unmarshal(b, v); err != nil {
panic("unmarshal: " + err.Error())
}
}
// assert will panic with a given formatted message if the given condition is false.
func assert(condition bool, msg string, v ...interface{}) {
if !condition {
panic(fmt.Sprintf("assert failed: "+msg, v...))
}
}

1
Godeps/_workspace/src/github.com/influxdb/influxdb/etc/burn-in/.rvmrc generated vendored
View File

@ -1 +0,0 @@
rvm use ruby-2.1.0@burn-in --create

4
Godeps/_workspace/src/github.com/influxdb/influxdb/etc/burn-in/Gemfile generated vendored
View File

@ -1,4 +0,0 @@
source 'https://rubygems.org'
gem "colorize"
gem "influxdb"

14
Godeps/_workspace/src/github.com/influxdb/influxdb/etc/burn-in/Gemfile.lock generated vendored
View File

@ -1,14 +0,0 @@
GEM
remote: https://rubygems.org/
specs:
colorize (0.6.0)
influxdb (0.0.16)
json
json (1.8.1)
PLATFORMS
ruby
DEPENDENCIES
colorize
influxdb

79
Godeps/_workspace/src/github.com/influxdb/influxdb/etc/burn-in/burn-in.rb generated vendored
View File

@ -1,79 +0,0 @@
require "influxdb"
require "colorize"
require "benchmark"
require_relative "log"
require_relative "random_gaussian"
BATCH_SIZE = 10_000
Log.info "Starting burn-in suite"
master = InfluxDB::Client.new
master.delete_database("burn-in") rescue nil
master.create_database("burn-in")
master.create_database_user("burn-in", "user", "pass")
master.database = "burn-in"
# master.query "select * from test1 into test2;"
# master.query "select count(value) from test1 group by time(1m) into test2;"
influxdb = InfluxDB::Client.new "burn-in", username: "user", password: "pass"
Log.success "Connected to server #{influxdb.host}:#{influxdb.port}"
Log.log "Creating RandomGaussian(500, 25)"
gaussian = RandomGaussian.new(500, 25)
point_count = 0
while true
Log.log "Generating 10,000 points.."
points = []
BATCH_SIZE.times do |n|
points << {value: gaussian.rand.to_i.abs}
end
point_count += points.length
Log.info "Sending points to server.."
begin
st = Time.now
foo = influxdb.write_point("test1", points)
et = Time.now
Log.log foo.inspect
Log.log "#{et-st} seconds elapsed"
Log.success "Write successful."
rescue => e
Log.failure "Write failed:"
Log.log e
end
sleep 0.5
Log.info "Checking regular points"
st = Time.now
response = influxdb.query("select count(value) from test1;")
et = Time.now
Log.log "#{et-st} seconds elapsed"
response_count = response["test1"].first["count"]
if point_count == response_count
Log.success "Point counts match: #{point_count} == #{response_count}"
else
Log.failure "Point counts don't match: #{point_count} != #{response_count}"
end
# Log.info "Checking continuous query points for test2"
# st = Time.now
# response = influxdb.query("select count(value) from test2;")
# et = Time.now
# Log.log "#{et-st} seconds elapsed"
# response_count = response["test2"].first["count"]
# if point_count == response_count
# Log.success "Point counts match: #{point_count} == #{response_count}"
# else
# Log.failure "Point counts don't match: #{point_count} != #{response_count}"
# end
end

23
Godeps/_workspace/src/github.com/influxdb/influxdb/etc/burn-in/log.rb generated vendored
View File

@ -1,23 +0,0 @@
module Log
def self.info(msg)
print Time.now.strftime("%r") + " | "
puts msg.to_s.colorize(:yellow)
end
def self.success(msg)
print Time.now.strftime("%r") + " | "
puts msg.to_s.colorize(:green)
end
def self.failure(msg)
print Time.now.strftime("%r") + " | "
puts msg.to_s.colorize(:red)
end
def self.log(msg)
print Time.now.strftime("%r") + " | "
puts msg.to_s
end
end

31
Godeps/_workspace/src/github.com/influxdb/influxdb/etc/burn-in/random_gaussian.rb generated vendored
View File

@ -1,31 +0,0 @@
class RandomGaussian
def initialize(mean, stddev, rand_helper = lambda { Kernel.rand })
@rand_helper = rand_helper
@mean = mean
@stddev = stddev
@valid = false
@next = 0
end
def rand
if @valid then
@valid = false
return @next
else
@valid = true
x, y = self.class.gaussian(@mean, @stddev, @rand_helper)
@next = y
return x
end
end
private
def self.gaussian(mean, stddev, rand)
theta = 2 * Math::PI * rand.call
rho = Math.sqrt(-2 * Math.log(1 - rand.call))
scale = stddev * rho
x = mean + scale * Math.cos(theta)
y = mean + scale * Math.sin(theta)
return x, y
end
end

29
Godeps/_workspace/src/github.com/influxdb/influxdb/etc/burn-in/random_points.rb generated vendored
View File

@ -1,29 +0,0 @@
require "influxdb"
ONE_WEEK_IN_SECONDS = 7*24*60*60
NUM_POINTS = 10_000
BATCHES = 100
master = InfluxDB::Client.new
master.delete_database("ctx") rescue nil
master.create_database("ctx")
influxdb = InfluxDB::Client.new "ctx"
influxdb.time_precision = "s"
names = ["foo", "bar", "baz", "quu", "qux"]
st = Time.now
BATCHES.times do |m|
points = []
puts "Writing #{NUM_POINTS} points, time ##{m}.."
NUM_POINTS.times do |n|
timestamp = Time.now.to_i - rand(ONE_WEEK_IN_SECONDS)
points << {value: names.sample, time: timestamp}
end
influxdb.write_point("ct1", points)
end
puts st
puts Time.now

246
Godeps/_workspace/src/github.com/influxdb/influxdb/etc/config.sample.toml generated vendored
View File

@ -1,246 +0,0 @@
### Welcome to the InfluxDB configuration file.
# Once every 24 hours InfluxDB will report anonymous data to m.influxdb.com
# The data includes raft id (random 8 bytes), os, arch, version, and metadata.
# We don't track ip addresses of servers reporting. This is only used
# to track the number of instances running and the versions, which
# is very helpful for us.
# Change this option to true to disable reporting.
reporting-disabled = false
###
### [meta]
###
### Controls the parameters for the Raft consensus group that stores metadata
### about the InfluxDB cluster.
###
[meta]
dir = "/var/opt/influxdb/meta"
hostname = "localhost"
bind-address = ":8088"
retention-autocreate = true
election-timeout = "1s"
heartbeat-timeout = "1s"
leader-lease-timeout = "500ms"
commit-timeout = "50ms"
###
### [data]
###
### Controls where the actual shard data for InfluxDB lives and how it is
### flushed from the WAL. "dir" may need to be changed to a suitable place
### for your system, but the WAL settings are an advanced configuration. The
### defaults should work for most systems.
###
[data]
dir = "/var/opt/influxdb/data"
# The following WAL settings are for the b1 storage engine used in 0.9.2. They won't
# apply to any new shards created after upgrading to a version > 0.9.3.
max-wal-size = 104857600 # Maximum size the WAL can reach before a flush. Defaults to 100MB.
wal-flush-interval = "10m" # Maximum time data can sit in WAL before a flush.
wal-partition-flush-delay = "2s" # The delay time between each WAL partition being flushed.
# These are the WAL settings for the storage engine >= 0.9.3
wal-dir = "/var/opt/influxdb/wal"
wal-enable-logging = true
# When a series in the WAL in-memory cache reaches this size in bytes it is marked as ready to
# flush to the index
# wal-ready-series-size = 25600
# Flush and compact a partition once this ratio of series are over the ready size
# wal-compaction-threshold = 0.6
# Force a flush and compaction if any series in a partition gets above this size in bytes
# wal-max-series-size = 2097152
# Force a flush of all series and full compaction if there have been no writes in this
# amount of time. This is useful for ensuring that shards that are cold for writes don't
# keep a bunch of data cached in memory and in the WAL.
# wal-flush-cold-interval = "10m"
# Force a partition to flush its largest series if it reaches this approximate size in
# bytes. Remember there are 5 partitions so you'll need at least 5x this amount of memory.
# The more memory you have, the bigger this can be.
# wal-partition-size-threshold = 20971520
###
### [cluster]
###
### Controls non-Raft cluster behavior, which generally includes how data is
### shared across shards.
###
[cluster]
shard-writer-timeout = "5s" # The time within which a shard must respond to write.
write-timeout = "5s" # The time within which a write operation must complete on the cluster.
###
### [retention]
###
### Controls the enforcement of retention policies for evicting old data.
###
[retention]
enabled = true
check-interval = "10m"
###
### [admin]
###
### Controls the availability of the built-in, web-based admin interface. If HTTPS is
### enabled for the admin interface, HTTPS must also be enabled on the [http] service.
###
[admin]
enabled = true
bind-address = ":8083"
https-enabled = false
https-certificate = "/etc/ssl/influxdb.pem"
###
### [http]
###
### Controls how the HTTP endpoints are configured. These are the primary
### mechanism for getting data into and out of InfluxDB.
###
[http]
enabled = true
bind-address = ":8086"
auth-enabled = false
log-enabled = true
write-tracing = false
pprof-enabled = false
https-enabled = false
https-certificate = "/etc/ssl/influxdb.pem"
###
### [[graphite]]
###
### Controls one or many listeners for Graphite data.
###
[[graphite]]
enabled = false
# bind-address = ":2003"
# protocol = "tcp"
# consistency-level = "one"
# name-separator = "."
# These next lines control how batching works. You should have this enabled
# otherwise you could get dropped metrics or poor performance. Batching
# will buffer points in memory if you have many coming in.
# batch-size = 1000 # will flush if this many points get buffered
# batch-timeout = "1s" # will flush at least this often even if we haven't hit buffer limit
## "name-schema" configures tag names for parsing the metric name from graphite protocol;
## separated by `name-separator`.
## The "measurement" tag is special and the corresponding field will become
## the name of the metric.
## e.g. "type.host.measurement.device" will parse "server.localhost.cpu.cpu0" as
## {
## measurement: "cpu",
## tags: {
## "type": "server",
## "host": "localhost,
## "device": "cpu0"
## }
## }
# name-schema = "type.host.measurement.device"
## If set to true, when the input metric name has more fields than `name-schema` specified,
## the extra fields will be ignored.
## Otherwise an error will be logged and the metric rejected.
# ignore-unnamed = true
###
### [collectd]
###
### Controls the listener for collectd data.
###
[collectd]
enabled = false
# bind-address = ""
# database = ""
# typesdb = ""
# These next lines control how batching works. You should have this enabled
# otherwise you could get dropped metrics or poor performance. Batching
# will buffer points in memory if you have many coming in.
# batch-size = 1000 # will flush if this many points get buffered
# batch-timeout = "1s" # will flush at least this often even if we haven't hit buffer limit
###
### [opentsdb]
###
### Controls the listener for OpenTSDB data.
###
[opentsdb]
enabled = false
# bind-address = ""
# database = ""
# retention-policy = ""
###
### [[udp]]
###
### Controls the listeners for InfluxDB line protocol data via UDP.
###
[[udp]]
enabled = false
# bind-address = ""
# database = ""
# These next lines control how batching works. You should have this enabled
# otherwise you could get dropped metrics or poor performance. Batching
# will buffer points in memory if you have many coming in.
# batch-size = 1000 # will flush if this many points get buffered
# batch-timeout = "1s" # will flush at least this often even if we haven't hit buffer limit
###
### [monitoring]
###
[monitoring]
enabled = true
write-interval = "24h"
###
### [continuous_queries]
###
### Controls how continuous queries are run within InfluxDB.
###
[continuous_queries]
log-enabled = true
enabled = true
recompute-previous-n = 2
recompute-no-older-than = "10m"
compute-runs-per-interval = 10
compute-no-more-than = "2m"
###
### [hinted-handoff]
###
### Controls the hinted handoff feature, which allows nodes to temporarily
### store queued data when one node of a cluster is down for a short period
### of time.
###
[hinted-handoff]
enabled = true
dir = "/var/opt/influxdb/hh"
max-size = 1073741824
max-age = "168h"
retry-rate-limit = 0
retry-interval = "1s"

186
Godeps/_workspace/src/github.com/influxdb/influxdb/importer/README.md generated vendored
View File

@ -1,186 +0,0 @@
# Import/Export
## Exporting from 0.8.9
Version `0.8.9` of InfluxDB adds support to export your data to a format that can be imported into `0.9.3` and later.
### Design
`0.8.9` exports raw data to a flat file that includes two sections, `DDL` and `DML`. You can choose to export them independently (see below).
The `DDL` section contains the sql commands to create databases and retention policies. the `DML` section is [line protocol](https://github.com/influxdb/influxdb/blob/master/tsdb/README.md) and can be directly posted to the [http endpoint](https://influxdb.com/docs/v0.9/guides/writing_data.html) in `0.9`. Remember that batching is important and we don't recommend batch sizes over 5k.
You need to specify a database and shard group when you export.
To list out your shards, use the following http endpoint:
`/cluster/shard_spaces`
example:
```sh
http://username:password@localhost:8086/cluster/shard_spaces
```
Then, to export a database with then name "metrics" and a shard space with the name "default", issue the following curl command:
```sh
curl -o export http://username:password@http://localhost:8086/export/metrics/default
```
Compression is supported, and will result in a significantly smaller file size.
Use the following command for compression:
```sh
curl -o export.gz --compressed http://username:password@http://localhost:8086/export/metrics/default
```
You can also export just the `DDL` with this option:
```sh
curl -o export.ddl http://username:password@http://localhost:8086/export/metrics/default?l=ddl
```
Or just the `DML` with this option:
```sh
curl -o export.dml.gz --compressed http://username:password@http://localhost:8086/export/metrics/default?l=dml
```
### Assumptions
- Series name mapping follows these [guidelines](https://influxdb.com/docs/v0.8/advanced_topics/schema_design.html)
- Database name will map directly from `0.8` to `0.9`
- Shard Spaces map to Retention Policies
- Shard Space Duration is ignored, as in `0.9` we determine shard size automatically
- Regex is used to match the correct series names and only exports that data for the database
- Duration becomes the new Retention Policy duration
- Users are not migrated due to inability to get passwords. Anyone using users will need to manually set these back up in `0.9`
### Upgrade Recommendations
It's recommended that you upgrade to `0.9.3` first and have all your writes going there. Then, on the `0.8.X` instances, upgrade to `0.8.9`.
It is important that when exporting you change your config to allow for the http endpoints not timing out. To do so, make this change in your config:
```toml
# Configure the http api
[api]
read-timeout = "0s"
```
### Exceptions
If a series can't be exported to tags based on the guidelines mentioned above,
we will insert the entire series name as the measurement name. You can either
allow that to import into the new InfluxDB instance, or you can do your own
data massage on it prior to importing it.
For example, if you have the following series name:
```
metric.disk.c.host.server01.single
```
It will export as exactly thta as the measurement name and no tags:
```
metric.disk.c.host.server01.single
```
### Export Metrics
When you export, you will now get comments inline in the `DML`:
`# Found 999 Series for export`
As well as count totals for each series exported:
`# Series FOO - Points Exported: 999`
With a total at the bottom:
`# Points Exported: 999`
You can grep the file that was exported at the end to get all the export metrics:
`cat myexport | grep Exported`
## Importing
Version `0.9.3` of InfluxDB adds support to import your data from version `0.8.9`.
## Caveats
For the export/import to work, all requisites have to be met. For export, all series names in `0.8` should be in the following format:
```
<tagName>.<tagValue>.<tagName>.<tagValue>.<measurement>
```
for example:
```
az.us-west-1.host.serverA.cpu
```
or any number of tags
```
building.2.temperature
```
Additionally, the fields need to have a consistent type (all float64, int64, etc) for every write in `0.8`. Otherwise they have the potential to fail writes in the import.
See below for more information.
## Running the import command
To import via the cli, you can specify the following command:
```sh
influx -import -path=metrics-default.gz -compressed
```
If the file is not compressed you can issue it without the `-compressed` flag:
```sh
influx -import -path=metrics-default
```
To redirect failed import lines to another file, run this command:
```sh
influx -import -path=metrics-default.gz -compressed > failures
```
The import will use the line protocol in batches of 5,000 lines per batch when sending data to the server.
### Throttiling the import
If you need to throttle the import so the database has time to ingest, you can use the `-pps` flag. This will limit the points per second that will be sent to the server.
```sh
influx -import -path=metrics-default.gz -compressed -pps 50000 > failures
```
Which is stating that you don't want MORE than 50,000 points per second to write to the database. Due to the processing that is taking place however, you will likely never get exactly 50,000 pps, more like 35,000 pps, etc.
## Understanding the results of the import
During the import, a status message will write out for every 100,000 points imported and report stats on the progress of the import:
```
2015/08/21 14:48:01 Processed 3100000 lines. Time elapsed: 56.740578415s. Points per second (PPS): 54634
```
The batch will give some basic stats when finished:
```sh
2015/07/29 23:15:20 Processed 2 commands
2015/07/29 23:15:20 Processed 70207923 inserts
2015/07/29 23:15:20 Failed 29785000 inserts
```
Most inserts fail due to the following types of error:
```sh
2015/07/29 22:18:28 error writing batch: write failed: field type conflict: input field "value" on measurement "metric" is type float64, already exists as type integer
```
This is due to the fact that in `0.8` a field could get created and saved as int or float types for independent writes. In `0.9` the field has to have a consistent type.

236
Godeps/_workspace/src/github.com/influxdb/influxdb/importer/v8/importer.go generated vendored
View File

@ -1,236 +0,0 @@
package v8
import (
"bufio"
"compress/gzip"
"fmt"
"io"
"log"
"net/url"
"os"
"strings"
"time"
"github.com/influxdb/influxdb/client"
)
const batchSize = 5000
// Config is the config used to initialize a Importer importer
type Config struct {
Username string
Password string
URL url.URL
Precision string
WriteConsistency string
Path string
Version string
Compressed bool
PPS int
}
// NewConfig returns an initialized *Config
func NewConfig() *Config {
return &Config{}
}
// Importer is the importer used for importing 0.8 data
type Importer struct {
client *client.Client
database string
retentionPolicy string
config *Config
batch []string
totalInserts int
failedInserts int
totalCommands int
throttlePointsWritten int
lastWrite time.Time
throttle *time.Ticker
}
// NewImporter will return an intialized Importer struct
func NewImporter(config *Config) *Importer {
return &Importer{
config: config,
batch: make([]string, 0, batchSize),
}
}
// Import processes the specified file in the Config and writes the data to the databases in chunks specified by batchSize
func (i *Importer) Import() error {
// Create a client and try to connect
config := client.NewConfig()
config.URL = i.config.URL
config.Username = i.config.Username
config.Password = i.config.Password
config.UserAgent = fmt.Sprintf("influxDB importer/%s", i.config.Version)
cl, err := client.NewClient(config)
if err != nil {
return fmt.Errorf("could not create client %s", err)
}
i.client = cl
if _, _, e := i.client.Ping(); e != nil {
return fmt.Errorf("failed to connect to %s\n", i.client.Addr())
}
// Validate args
if i.config.Path == "" {
return fmt.Errorf("file argument required")
}
defer func() {
if i.totalInserts > 0 {
log.Printf("Processed %d commands\n", i.totalCommands)
log.Printf("Processed %d inserts\n", i.totalInserts)
log.Printf("Failed %d inserts\n", i.failedInserts)
}
}()
// Open the file
f, err := os.Open(i.config.Path)
if err != nil {
return err
}
defer f.Close()
var r io.Reader
// If gzipped, wrap in a gzip reader
if i.config.Compressed {
gr, err := gzip.NewReader(f)
if err != nil {
return err
}
defer gr.Close()
// Set the reader to the gzip reader
r = gr
} else {
// Standard text file so our reader can just be the file
r = f
}
// Get our reader
scanner := bufio.NewScanner(r)
// Process the DDL
i.processDDL(scanner)
// Set up our throttle channel. Since there is effectively no other activity at this point
// the smaller resolution gets us much closer to the requested PPS
i.throttle = time.NewTicker(time.Microsecond)
defer i.throttle.Stop()
// Prime the last write
i.lastWrite = time.Now()
// Process the DML
i.processDML(scanner)
// Check if we had any errors scanning the file
if err := scanner.Err(); err != nil {
return fmt.Errorf("reading standard input: %s", err)
}
return nil
}
func (i *Importer) processDDL(scanner *bufio.Scanner) {
for scanner.Scan() {
line := scanner.Text()
// If we find the DML token, we are done with DDL
if strings.HasPrefix(line, "# DML") {
return
}
if strings.HasPrefix(line, "#") {
continue
}
i.queryExecutor(line)
}
}
func (i *Importer) processDML(scanner *bufio.Scanner) {
start := time.Now()
for scanner.Scan() {
line := scanner.Text()
if strings.HasPrefix(line, "# CONTEXT-DATABASE:") {
i.database = strings.TrimSpace(strings.Split(line, ":")[1])
}
if strings.HasPrefix(line, "# CONTEXT-RETENTION-POLICY:") {
i.retentionPolicy = strings.TrimSpace(strings.Split(line, ":")[1])
}
if strings.HasPrefix(line, "#") {
continue
}
i.batchAccumulator(line, start)
}
}
func (i *Importer) execute(command string) {
response, err := i.client.Query(client.Query{Command: command, Database: i.database})
if err != nil {
log.Printf("error: %s\n", err)
return
}
if err := response.Error(); err != nil {
log.Printf("error: %s\n", response.Error())
}
}
func (i *Importer) queryExecutor(command string) {
i.totalCommands++
i.execute(command)
}
func (i *Importer) batchAccumulator(line string, start time.Time) {
i.batch = append(i.batch, line)
if len(i.batch) == batchSize {
if e := i.batchWrite(); e != nil {
log.Println("error writing batch: ", e)
// Output failed lines to STDOUT so users can capture lines that failed to import
fmt.Println(strings.Join(i.batch, "\n"))
i.failedInserts += len(i.batch)
} else {
i.totalInserts += len(i.batch)
}
i.batch = i.batch[:0]
// Give some status feedback every 100000 lines processed
processed := i.totalInserts + i.failedInserts
if processed%100000 == 0 {
since := time.Since(start)
pps := float64(processed) / since.Seconds()
log.Printf("Processed %d lines. Time elapsed: %s. Points per second (PPS): %d", processed, since.String(), int64(pps))
}
}
}
func (i *Importer) batchWrite() error {
// Accumulate the batch size to see how many points we have written this second
i.throttlePointsWritten += len(i.batch)
// Find out when we last wrote data
since := time.Since(i.lastWrite)
// Check to see if we've exceeded our points per second for the current timeframe
var currentPPS int
if since.Seconds() > 0 {
currentPPS = int(float64(i.throttlePointsWritten) / since.Seconds())
} else {
currentPPS = i.throttlePointsWritten
}
// If our currentPPS is greater than the PPS specified, then we wait and retry
if int(currentPPS) > i.config.PPS && i.config.PPS != 0 {
// Wait for the next tick
<-i.throttle.C
// Decrement the batch size back out as it is going to get called again
i.throttlePointsWritten -= len(i.batch)
return i.batchWrite()
}
_, e := i.client.WriteLineProtocol(strings.Join(i.batch, "\n"), i.database, i.retentionPolicy, i.config.Precision, i.config.WriteConsistency)
i.throttlePointsWritten = 0
i.lastWrite = time.Now()
return e
}

14
Godeps/_workspace/src/github.com/influxdb/influxdb/nightly.sh generated vendored
View File

@ -1,14 +0,0 @@
#!/bin/bash
REPO_DIR=`mktemp -d`
echo "Using $REPO_DIR for all work..."
cd $REPO_DIR
export GOPATH=`pwd`
mkdir -p $GOPATH/src/github.com/influxdb
cd $GOPATH/src/github.com/influxdb
git clone https://github.com/influxdb/influxdb.git
cd $GOPATH/src/github.com/influxdb/influxdb
NIGHTLY_BUILD=true ./package.sh 0.9.3-nightly-`git log --pretty=format:'%h' -n 1`
rm -rf $REPO_DIR

409
Godeps/_workspace/src/github.com/influxdb/influxdb/package.sh generated vendored
View File

@ -1,409 +0,0 @@
#!/usr/bin/env bash
###########################################################################
# Packaging script which creates debian and RPM packages. It optionally
# tags the repo with the given version.
#
# Requirements: GOPATH must be set. 'fpm' must be on the path, and the AWS
# CLI tools must also be installed.
#
# https://github.com/jordansissel/fpm
# http://aws.amazon.com/cli/
#
# Packaging process: to package a build, simple execute:
#
# package.sh <version>
#
# where <version> is the desired version. If generation of a debian and RPM
# package is successful, the script will offer to tag the repo using the
# supplied version string.
#
# AWS upload: the script will also offer to upload the packages to S3. If
# this option is selected, the credentials should be present in the file
# ~/aws.conf. The contents should be of the form:
#
# [default]
# aws_access_key_id=<access ID>
# aws_secret_access_key=<secret key>
# region = us-east-1
#
# Trim the leading spaces when creating the file. The script will exit if
# S3 upload is requested, but this file does not exist.
[ -z $DEBUG ] || set -x
AWS_FILE=~/aws.conf
INSTALL_ROOT_DIR=/opt/influxdb
INFLUXDB_LOG_DIR=/var/log/influxdb
INFLUXDB_DATA_DIR=/var/opt/influxdb
CONFIG_ROOT_DIR=/etc/opt/influxdb
SAMPLE_CONFIGURATION=etc/config.sample.toml
INITD_SCRIPT=scripts/init.sh
TMP_WORK_DIR=`mktemp -d`
POST_INSTALL_PATH=`mktemp`
ARCH=`uname -i`
LICENSE=MIT
URL=influxdb.com
MAINTAINER=support@influxdb.com
VENDOR=Influxdb
DESCRIPTION="Distributed time-series database"
# Allow path to FPM to be set by environment variables. Some execution contexts
# like cron don't have PATH set correctly to pick it up.
if [ -z "$FPM" ]; then
FPM=`which fpm`
fi
GO_VERSION="go1.4.2"
GOPATH_INSTALL=
BINS=(
influxd
influx
)
###########################################################################
# Helper functions.
# usage prints simple usage information.
usage() {
echo -e "$0 [<version>] [-h]\n"
cleanup_exit $1
}
# cleanup_exit removes all resources created during the process and exits with
# the supplied returned code.
cleanup_exit() {
rm -r $TMP_WORK_DIR
rm $POST_INSTALL_PATH
exit $1
}
# current_branch echos the current git branch.
current_branch() {
echo `git rev-parse --abbrev-ref HEAD`
}
# check_gopath sanity checks the value of the GOPATH env variable, and determines
# the path where build artifacts are installed. GOPATH may be a colon-delimited
# list of directories.
check_gopath() {
[ -z "$GOPATH" ] && echo "GOPATH is not set." && cleanup_exit 1
GOPATH_INSTALL=`echo $GOPATH | cut -d ':' -f 1`
[ ! -d "$GOPATH_INSTALL" ] && echo "GOPATH_INSTALL is not a directory." && cleanup_exit 1
echo "GOPATH ($GOPATH) looks sane, using $GOPATH_INSTALL for installation."
}
check_gvm() {
if [ -n "$GOPATH" ]; then
existing_gopath=$GOPATH
fi
source $HOME/.gvm/scripts/gvm
which gvm
if [ $? -ne 0 ]; then
echo "gvm not found -- aborting."
cleanup_exit $1
fi
gvm use $GO_VERSION
if [ $? -ne 0 ]; then
echo "gvm cannot find Go version $GO_VERSION -- aborting."
cleanup_exit $1
fi
# Keep any existing GOPATH set.
if [ -n "$existing_gopath" ]; then
GOPATH=$existing_gopath
fi
}
# check_clean_tree ensures that no source file is locally modified.
check_clean_tree() {
modified=$(git ls-files --modified | wc -l)
if [ $modified -ne 0 ]; then
echo "The source tree is not clean -- aborting."
cleanup_exit 1
fi
echo "Git tree is clean."
}
# update_tree ensures the tree is in-sync with the repo.
update_tree() {
git pull origin $TARGET_BRANCH
if [ $? -ne 0 ]; then
echo "Failed to pull latest code -- aborting."
cleanup_exit 1
fi
git fetch --tags
if [ $? -ne 0 ]; then
echo "Failed to fetch tags -- aborting."
cleanup_exit 1
fi
echo "Git tree updated successfully."
}
# check_tag_exists checks if the existing release already exists in the tags.
check_tag_exists () {
version=$1
git tag | grep -q "^v$version$"
if [ $? -eq 0 ]; then
echo "Proposed version $version already exists as a tag -- aborting."
cleanup_exit 1
fi
}
# make_dir_tree creates the directory structure within the packages.
make_dir_tree() {
work_dir=$1
version=$2
mkdir -p $work_dir/$INSTALL_ROOT_DIR/versions/$version/scripts
if [ $? -ne 0 ]; then
echo "Failed to create installation directory -- aborting."
cleanup_exit 1
fi
mkdir -p $work_dir/$CONFIG_ROOT_DIR
if [ $? -ne 0 ]; then
echo "Failed to create configuration directory -- aborting."
cleanup_exit 1
fi
}
# do_build builds the code. The version and commit must be passed in.
do_build() {
for b in ${BINS[*]}; do
rm -f $GOPATH_INSTALL/bin/$b
done
go get -u -f -d ./...
if [ $? -ne 0 ]; then
echo "WARNING: failed to 'go get' packages."
fi
git checkout $TARGET_BRANCH # go get switches to master, so ensure we're back.
version=$1
commit=`git rev-parse HEAD`
branch=`current_branch`
if [ $? -ne 0 ]; then
echo "Unable to retrieve current commit -- aborting"
cleanup_exit 1
fi
go install -a -ldflags="-X main.version $version -X main.branch $branch -X main.commit $commit" ./...
if [ $? -ne 0 ]; then
echo "Build failed, unable to create package -- aborting"
cleanup_exit 1
fi
echo "Build completed successfully."
}
# generate_postinstall_script creates the post-install script for the
# package. It must be passed the version.
generate_postinstall_script() {
version=$1
cat <<EOF >$POST_INSTALL_PATH
rm -f $INSTALL_ROOT_DIR/influxd
rm -f $INSTALL_ROOT_DIR/influx
rm -f $INSTALL_ROOT_DIR/init.sh
ln -s $INSTALL_ROOT_DIR/versions/$version/influxd $INSTALL_ROOT_DIR/influxd
ln -s $INSTALL_ROOT_DIR/versions/$version/influx $INSTALL_ROOT_DIR/influx
ln -s $INSTALL_ROOT_DIR/versions/$version/scripts/init.sh $INSTALL_ROOT_DIR/init.sh
rm -f /etc/init.d/influxdb
ln -sfn $INSTALL_ROOT_DIR/init.sh /etc/init.d/influxdb
chmod +x /etc/init.d/influxdb
if which update-rc.d > /dev/null 2>&1 ; then
update-rc.d -f influxdb remove
update-rc.d influxdb defaults
else
chkconfig --add influxdb
fi
if ! id influxdb >/dev/null 2>&1; then
useradd --system -U -M influxdb
fi
chown -R -L influxdb:influxdb $INSTALL_ROOT_DIR
chmod -R a+rX $INSTALL_ROOT_DIR
mkdir -p $INFLUXDB_LOG_DIR
chown -R -L influxdb:influxdb $INFLUXDB_LOG_DIR
mkdir -p $INFLUXDB_DATA_DIR
chown -R -L influxdb:influxdb $INFLUXDB_DATA_DIR
EOF
echo "Post-install script created successfully at $POST_INSTALL_PATH"
}
###########################################################################
# Start the packaging process.
if [ $# -ne 1 ]; then
usage 1
elif [ $1 == "-h" ]; then
usage 0
else
VERSION=$1
VERSION_UNDERSCORED=`echo "$VERSION" | tr - _`
fi
echo -e "\nStarting package process...\n"
# Ensure the current is correct.
TARGET_BRANCH=`current_branch`
if [ -z "$NIGHTLY_BUILD" ]; then
echo -n "Current branch is $TARGET_BRANCH. Start packaging this branch? [Y/n] "
read response
response=`echo $response | tr 'A-Z' 'a-z'`
if [ "x$response" == "xn" ]; then
echo "Packaging aborted."
cleanup_exit 1
fi
fi
check_gvm
check_gopath
if [ -z "$NIGHTLY_BUILD" ]; then
check_clean_tree
update_tree
check_tag_exists $VERSION
fi
do_build $VERSION
make_dir_tree $TMP_WORK_DIR $VERSION
###########################################################################
# Copy the assets to the installation directories.
for b in ${BINS[*]}; do
cp $GOPATH_INSTALL/bin/$b $TMP_WORK_DIR/$INSTALL_ROOT_DIR/versions/$VERSION
if [ $? -ne 0 ]; then
echo "Failed to copy binaries to packaging directory -- aborting."
cleanup_exit 1
fi
done
echo "${BINS[*]} copied to $TMP_WORK_DIR/$INSTALL_ROOT_DIR/versions/$VERSION"
cp $INITD_SCRIPT $TMP_WORK_DIR/$INSTALL_ROOT_DIR/versions/$VERSION/scripts
if [ $? -ne 0 ]; then
echo "Failed to copy init.d script to packaging directory -- aborting."
cleanup_exit 1
fi
echo "$INITD_SCRIPT copied to $TMP_WORK_DIR/$INSTALL_ROOT_DIR/versions/$VERSION/scripts"
cp $SAMPLE_CONFIGURATION $TMP_WORK_DIR/$CONFIG_ROOT_DIR/influxdb.conf
if [ $? -ne 0 ]; then
echo "Failed to copy $SAMPLE_CONFIGURATION to packaging directory -- aborting."
cleanup_exit 1
fi
generate_postinstall_script $VERSION
###########################################################################
# Create the actual packages.
if [ -z "$NIGHTLY_BUILD" ]; then
echo -n "Commence creation of $ARCH packages, version $VERSION? [Y/n] "
read response
response=`echo $response | tr 'A-Z' 'a-z'`
if [ "x$response" == "xn" ]; then
echo "Packaging aborted."
cleanup_exit 1
fi
fi
if [ $ARCH == "i386" ]; then
rpm_package=influxdb-${VERSION}-1.i686.rpm # RPM packages use 1 for default package release.
debian_package=influxdb_${VERSION}_i686.deb
deb_args="-a i686"
rpm_args="setarch i686"
elif [ $ARCH == "arm" ]; then
rpm_package=influxdb-${VERSION}-1.armel.rpm
debian_package=influxdb_${VERSION}_armel.deb
else
rpm_package=influxdb-${VERSION}-1.x86_64.rpm
debian_package=influxdb_${VERSION}_amd64.deb
fi
COMMON_FPM_ARGS="-C $TMP_WORK_DIR --vendor $VENDOR --url $URL --license $LICENSE --maintainer $MAINTAINER --after-install $POST_INSTALL_PATH --name influxdb --version $VERSION --config-files $CONFIG_ROOT_DIR ."
$rpm_args $FPM -s dir -t rpm --description "$DESCRIPTION" $COMMON_FPM_ARGS
if [ $? -ne 0 ]; then
echo "Failed to create RPM package -- aborting."
cleanup_exit 1
fi
echo "RPM package created successfully."
$FPM -s dir -t deb $deb_args --description "$DESCRIPTION" $COMMON_FPM_ARGS
if [ $? -ne 0 ]; then
echo "Failed to create Debian package -- aborting."
cleanup_exit 1
fi
echo "Debian package created successfully."
$FPM -s dir -t tar --prefix influxdb_${VERSION}_${ARCH} -p influxdb_${VERSION}_${ARCH}.tar.gz --description "$DESCRIPTION" $COMMON_FPM_ARGS
if [ $? -ne 0 ]; then
echo "Failed to create Tar package -- aborting."
cleanup_exit 1
fi
echo "Tar package created successfully."
###########################################################################
# Offer to tag the repo.
if [ -z "$NIGHTLY_BUILD" ]; then
echo -n "Tag source tree with v$VERSION and push to repo? [y/N] "
read response
response=`echo $response | tr 'A-Z' 'a-z'`
if [ "x$response" == "xy" ]; then
echo "Creating tag v$VERSION and pushing to repo"
git tag v$VERSION
if [ $? -ne 0 ]; then
echo "Failed to create tag v$VERSION -- aborting"
cleanup_exit 1
fi
git push origin v$VERSION
if [ $? -ne 0 ]; then
echo "Failed to push tag v$VERSION to repo -- aborting"
cleanup_exit 1
fi
else
echo "Not creating tag v$VERSION."
fi
fi
###########################################################################
# Offer to publish the packages.
if [ -z "$NIGHTLY_BUILD" ]; then
echo -n "Publish packages to S3? [y/N] "
read response
response=`echo $response | tr 'A-Z' 'a-z'`
fi
if [ "x$response" == "xy" -o -n "$NIGHTLY_BUILD" ]; then
echo "Publishing packages to S3."
if [ ! -e "$AWS_FILE" ]; then
echo "$AWS_FILE does not exist -- aborting."
cleanup_exit 1
fi
for filepath in `ls *.{deb,rpm,gz}`; do
filename=`basename $filepath`
if [ -n "$NIGHTLY_BUILD" ]; then
filename=`echo $filename | sed s/$VERSION/nightly/`
filename=`echo $filename | sed s/$VERSION_UNDERSCORED/nightly/`
fi
AWS_CONFIG_FILE=$AWS_FILE aws s3 cp $filepath s3://influxdb/$filename --acl public-read --region us-east-1
if [ $? -ne 0 ]; then
echo "Upload failed -- aborting".
cleanup_exit 1
fi
done
else
echo "Not publishing packages to S3."
fi
###########################################################################
# All done.
echo -e "\nPackaging process complete."
cleanup_exit 0

16
Godeps/_workspace/src/github.com/influxdb/influxdb/scripts/influxdb.service generated vendored
View File

@ -1,16 +0,0 @@
# If you modify this, please also make sure to edit init.sh
[Unit]
Description=InfluxDB is an open-source, distributed, time series database
After=network.target
[Service]
User=influxdb
Group=influxdb
LimitNOFILE=65536
EnvironmentFile=-/etc/default/influxdb
ExecStart=/opt/influxdb/influxd -config /etc/opt/influxdb/influxdb.conf $INFLUXD_OPTS
Restart=on-failure
[Install]
WantedBy=multi-user.target

205
Godeps/_workspace/src/github.com/influxdb/influxdb/scripts/init.sh generated vendored
View File

@ -1,205 +0,0 @@
#! /usr/bin/env bash
### BEGIN INIT INFO
# Provides: influxd
# Required-Start: $all
# Required-Stop: $remote_fs $syslog
# Default-Start: 2 3 4 5
# Default-Stop: 0 1 6
# Short-Description: Start influxd at boot time
### END INIT INFO
# If you modify this, please make sure to also edit influxdb.service
# this init script supports three different variations:
# 1. New lsb that define start-stop-daemon
# 2. Old lsb that don't have start-stop-daemon but define, log, pidofproc and killproc
# 3. Centos installations without lsb-core installed
#
# In the third case we have to define our own functions which are very dumb
# and expect the args to be positioned correctly.
# Command-line options that can be set in /etc/default/influxdb. These will override
# any config file values. Example: "-join http://1.2.3.4:8086"
DEFAULT=/etc/default/influxdb
# Daemon options
INFLUXD_OPTS=
# Process name ( For display )
NAME=influxdb
# User and group
USER=influxdb
GROUP=influxdb
# Daemon name, where is the actual executable
# If the daemon is not there, then exit.
DAEMON=/opt/influxdb/influxd
[ -x $DAEMON ] || exit 5
# Configuration file
CONFIG=/etc/opt/influxdb/influxdb.conf
# PID file for the daemon
PIDFILE=/var/run/influxdb/influxd.pid
PIDDIR=`dirname $PIDFILE`
if [ ! -d "$PIDDIR" ]; then
mkdir -p $PIDDIR
chown $GROUP:$USER $PIDDIR
fi
# Max open files
OPEN_FILE_LIMIT=65536
if [ -r /lib/lsb/init-functions ]; then
source /lib/lsb/init-functions
fi
# Logging
if [ -z "$STDOUT" ]; then
STDOUT=/dev/null
fi
if [ ! -f "$STDOUT" ]; then
mkdir -p $(dirname $STDOUT)
fi
if [ -z "$STDERR" ]; then
STDERR=/var/log/influxdb/influxd.log
fi
if [ ! -f "$STDERR" ]; then
mkdir -p $(dirname $STDERR)
fi
# Overwrite init script variables with /etc/default/influxdb values
if [ -r $DEFAULT ]; then
source $DEFAULT
fi
function pidofproc() {
if [ $# -ne 3 ]; then
echo "Expected three arguments, e.g. $0 -p pidfile daemon-name"
fi
PID=`pgrep -f $3`
local PIDFILE=`cat $2`
if [ "x$PIDFILE" == "x" ]; then
return 1
fi
if [ "x$PID" != "x" -a "$PIDFILE" == "$PID" ]; then
return 0
fi
return 1
}
function killproc() {
if [ $# -ne 3 ]; then
echo "Expected three arguments, e.g. $0 -p pidfile signal"
fi
PID=`cat $2`
kill -s $3 $PID
}
function log_failure_msg() {
echo "$@" "[ FAILED ]"
}
function log_success_msg() {
echo "$@" "[ OK ]"
}
case $1 in
start)
# Check if config file exist
if [ ! -r $CONFIG ]; then
log_failure_msg "config file doesn't exists"
exit 4
fi
# Checked the PID file exists and check the actual status of process
if [ -e $PIDFILE ]; then
pidofproc -p $PIDFILE $DAEMON > /dev/null 2>&1 && STATUS="0" || STATUS="$?"
# If the status is SUCCESS then don't need to start again.
if [ "x$STATUS" = "x0" ]; then
log_failure_msg "$NAME process is running"
exit 0 # Exit
fi
# if PID file does not exist, check if writable
else
su -c "touch $PIDFILE" $USER > /dev/null 2>&1
if [ $? -ne 0 ]; then
log_failure_msg "$PIDFILE not writable, check permissions"
exit 5
fi
fi
# Bump the file limits, before launching the daemon. These will carry over to
# launched processes.
ulimit -n $OPEN_FILE_LIMIT
if [ $? -ne 0 ]; then
log_failure_msg "set open file limit to $OPEN_FILE_LIMIT"
exit 1
fi
log_success_msg "Starting the process" "$NAME"
if which start-stop-daemon > /dev/null 2>&1; then
start-stop-daemon --chuid $GROUP:$USER --start --quiet --pidfile $PIDFILE --exec $DAEMON -- -pidfile $PIDFILE -config $CONFIG $INFLUXD_OPTS >>$STDOUT 2>>$STDERR &
else
nohup $DAEMON -pidfile $PIDFILE -config $CONFIG $INFLUXD_OPTS >>$STDOUT 2>>$STDERR &
fi
log_success_msg "$NAME process was started"
;;
stop)
# Stop the daemon.
if [ -e $PIDFILE ]; then
pidofproc -p $PIDFILE $DAEMON > /dev/null 2>&1 && STATUS="0" || STATUS="$?"
if [ "$STATUS" = 0 ]; then
if killproc -p $PIDFILE SIGTERM && /bin/rm -rf $PIDFILE; then
log_success_msg "$NAME process was stopped"
else
log_failure_msg "$NAME failed to stop service"
fi
fi
else
log_failure_msg "$NAME process is not running"
fi
;;
restart)
# Restart the daemon.
$0 stop && sleep 2 && $0 start
;;
status)
# Check the status of the process.
if [ -e $PIDFILE ]; then
if pidofproc -p $PIDFILE $DAEMON > /dev/null; then
log_success_msg "$NAME Process is running"
exit 0
else
log_failure_msg "$NAME Process is not running"
exit 1
fi
else
log_failure_msg "$NAME Process is not running"
exit 3
fi
;;
version)
$DAEMON version
;;
*)
# For invalid arguments, print the usage message.
echo "Usage: $0 {start|stop|restart|status|version}"
exit 2
;;
esac

21
Godeps/_workspace/src/github.com/influxdb/influxdb/services/admin/config.go generated vendored
View File

@ -1,21 +0,0 @@
package admin
const (
// DefaultBindAddress is the default bind address for the HTTP server.
DefaultBindAddress = ":8083"
)
type Config struct {
Enabled bool `toml:"enabled"`
BindAddress string `toml:"bind-address"`
HttpsEnabled bool `toml:"https-enabled"`
HttpsCertificate string `toml:"https-certificate"`
}
func NewConfig() Config {
return Config{
BindAddress: DefaultBindAddress,
HttpsEnabled: false,
HttpsCertificate: "/etc/ssl/influxdb.pem",
}
}

32
Godeps/_workspace/src/github.com/influxdb/influxdb/services/admin/config_test.go generated vendored
View File

@ -1,32 +0,0 @@
package admin_test
import (
"testing"
"github.com/BurntSushi/toml"
"github.com/influxdb/influxdb/services/admin"
)
func TestConfig_Parse(t *testing.T) {
// Parse configuration.
var c admin.Config
if _, err := toml.Decode(`
enabled = true
bind-address = ":8083"
https-enabled = true
https-certificate = "/dev/null"
`, &c); err != nil {
t.Fatal(err)
}
// Validate configuration.
if c.Enabled != true {
t.Fatalf("unexpected enabled: %v", c.Enabled)
} else if c.BindAddress != ":8083" {
t.Fatalf("unexpected bind address: %s", c.BindAddress)
} else if c.HttpsEnabled != true {
t.Fatalf("unexpected https enabled: %v", c.HttpsEnabled)
} else if c.HttpsCertificate != "/dev/null" {
t.Fatalf("unexpected https certificate: %v", c.HttpsCertificate)
}
}

111
Godeps/_workspace/src/github.com/influxdb/influxdb/services/admin/service.go generated vendored
View File

@ -1,111 +0,0 @@
package admin
import (
"crypto/tls"
"fmt"
"log"
"net"
"net/http"
"os"
"strings"
// Register static assets via statik.
_ "github.com/influxdb/influxdb/statik"
"github.com/rakyll/statik/fs"
)
// Service manages the listener for an admin endpoint.
type Service struct {
listener net.Listener
addr string
https bool
cert string
err chan error
logger *log.Logger
}
// NewService returns a new instance of Service.
func NewService(c Config) *Service {
return &Service{
addr: c.BindAddress,
https: c.HttpsEnabled,
cert: c.HttpsCertificate,
err: make(chan error),
logger: log.New(os.Stderr, "[admin] ", log.LstdFlags),
}
}
// Open starts the service
func (s *Service) Open() error {
s.logger.Printf("Starting admin service")
// Open listener.
if s.https {
cert, err := tls.LoadX509KeyPair(s.cert, s.cert)
if err != nil {
return err
}
listener, err := tls.Listen("tcp", s.addr, &tls.Config{
Certificates: []tls.Certificate{cert},
})
if err != nil {
return err
}
s.logger.Println("Listening on HTTPS:", listener.Addr().String())
s.listener = listener
} else {
listener, err := net.Listen("tcp", s.addr)
if err != nil {
return err
}
s.logger.Println("Listening on HTTP:", listener.Addr().String())
s.listener = listener
}
// Begin listening for requests in a separate goroutine.
go s.serve()
return nil
}
// Close closes the underlying listener.
func (s *Service) Close() error {
if s.listener != nil {
return s.listener.Close()
}
return nil
}
// SetLogger sets the internal logger to the logger passed in.
func (s *Service) SetLogger(l *log.Logger) {
s.logger = l
}
// Err returns a channel for fatal errors that occur on the listener.
func (s *Service) Err() <-chan error { return s.err }
// Addr returns the listener's address. Returns nil if listener is closed.
func (s *Service) Addr() net.Addr {
if s.listener != nil {
return s.listener.Addr()
}
return nil
}
// serve serves the handler from the listener.
func (s *Service) serve() {
// Instantiate file system from embedded admin.
statikFS, err := fs.New()
if err != nil {
panic(err)
}
// Run file system handler on listener.
err = http.Serve(s.listener, http.FileServer(statikFS))
if err != nil && !strings.Contains(err.Error(), "closed") {
s.err <- fmt.Errorf("listener error: addr=%s, err=%s", s.Addr(), err)
}
}

33
Godeps/_workspace/src/github.com/influxdb/influxdb/services/admin/service_test.go generated vendored
View File

@ -1,33 +0,0 @@
package admin_test
import (
"io/ioutil"
"net/http"
"testing"
"github.com/influxdb/influxdb/services/admin"
)
// Ensure service can serve the root index page of the admin.
func TestService_Index(t *testing.T) {
// Start service on random port.
s := admin.NewService(admin.Config{BindAddress: "127.0.0.1:0"})
if err := s.Open(); err != nil {
t.Fatal(err)
}
defer s.Close()
// Request root index page.
resp, err := http.Get("http://" + s.Addr().String())
if err != nil {
t.Fatal(err)
}
defer resp.Body.Close()
// Validate status code and body.
if resp.StatusCode != http.StatusOK {
t.Fatalf("unexpected status: %d", resp.StatusCode)
} else if _, err := ioutil.ReadAll(resp.Body); err != nil {
t.Fatalf("unable to read body: %s", err)
}
}

209
Godeps/_workspace/src/github.com/influxdb/influxdb/services/collectd/collectd_test.conf generated vendored
View File

@ -1,209 +0,0 @@
absolute value:ABSOLUTE:0:U
apache_bytes value:DERIVE:0:U
apache_connections value:GAUGE:0:65535
apache_idle_workers value:GAUGE:0:65535
apache_requests value:DERIVE:0:U
apache_scoreboard value:GAUGE:0:65535
ath_nodes value:GAUGE:0:65535
ath_stat value:DERIVE:0:U
backends value:GAUGE:0:65535
bitrate value:GAUGE:0:4294967295
bytes value:GAUGE:0:U
cache_eviction value:DERIVE:0:U
cache_operation value:DERIVE:0:U
cache_ratio value:GAUGE:0:100
cache_result value:DERIVE:0:U
cache_size value:GAUGE:0:U
charge value:GAUGE:0:U
compression_ratio value:GAUGE:0:2
compression uncompressed:DERIVE:0:U, compressed:DERIVE:0:U
connections value:DERIVE:0:U
conntrack value:GAUGE:0:4294967295
contextswitch value:DERIVE:0:U
counter value:COUNTER:U:U
cpufreq value:GAUGE:0:U
cpu value:DERIVE:0:U
current_connections value:GAUGE:0:U
current_sessions value:GAUGE:0:U
current value:GAUGE:U:U
delay value:GAUGE:-1000000:1000000
derive value:DERIVE:0:U
df_complex value:GAUGE:0:U
df_inodes value:GAUGE:0:U
df used:GAUGE:0:1125899906842623, free:GAUGE:0:1125899906842623
disk_latency read:GAUGE:0:U, write:GAUGE:0:U
disk_merged read:DERIVE:0:U, write:DERIVE:0:U
disk_octets read:DERIVE:0:U, write:DERIVE:0:U
disk_ops_complex value:DERIVE:0:U
disk_ops read:DERIVE:0:U, write:DERIVE:0:U
disk_time read:DERIVE:0:U, write:DERIVE:0:U
dns_answer value:DERIVE:0:U
dns_notify value:DERIVE:0:U
dns_octets queries:DERIVE:0:U, responses:DERIVE:0:U
dns_opcode value:DERIVE:0:U
dns_qtype_cached value:GAUGE:0:4294967295
dns_qtype value:DERIVE:0:U
dns_query value:DERIVE:0:U
dns_question value:DERIVE:0:U
dns_rcode value:DERIVE:0:U
dns_reject value:DERIVE:0:U
dns_request value:DERIVE:0:U
dns_resolver value:DERIVE:0:U
dns_response value:DERIVE:0:U
dns_transfer value:DERIVE:0:U
dns_update value:DERIVE:0:U
dns_zops value:DERIVE:0:U
duration seconds:GAUGE:0:U
email_check value:GAUGE:0:U
email_count value:GAUGE:0:U
email_size value:GAUGE:0:U
entropy value:GAUGE:0:4294967295
fanspeed value:GAUGE:0:U
file_size value:GAUGE:0:U
files value:GAUGE:0:U
flow value:GAUGE:0:U
fork_rate value:DERIVE:0:U
frequency_offset value:GAUGE:-1000000:1000000
frequency value:GAUGE:0:U
fscache_stat value:DERIVE:0:U
gauge value:GAUGE:U:U
hash_collisions value:DERIVE:0:U
http_request_methods value:DERIVE:0:U
http_requests value:DERIVE:0:U
http_response_codes value:DERIVE:0:U
humidity value:GAUGE:0:100
if_collisions value:DERIVE:0:U
if_dropped rx:DERIVE:0:U, tx:DERIVE:0:U
if_errors rx:DERIVE:0:U, tx:DERIVE:0:U
if_multicast value:DERIVE:0:U
if_octets rx:DERIVE:0:U, tx:DERIVE:0:U
if_packets rx:DERIVE:0:U, tx:DERIVE:0:U
if_rx_errors value:DERIVE:0:U
if_rx_octets value:DERIVE:0:U
if_tx_errors value:DERIVE:0:U
if_tx_octets value:DERIVE:0:U
invocations value:DERIVE:0:U
io_octets rx:DERIVE:0:U, tx:DERIVE:0:U
io_packets rx:DERIVE:0:U, tx:DERIVE:0:U
ipt_bytes value:DERIVE:0:U
ipt_packets value:DERIVE:0:U
irq value:DERIVE:0:U
latency value:GAUGE:0:U
links value:GAUGE:0:U
load shortterm:GAUGE:0:5000, midterm:GAUGE:0:5000, longterm:GAUGE:0:5000
md_disks value:GAUGE:0:U
memcached_command value:DERIVE:0:U
memcached_connections value:GAUGE:0:U
memcached_items value:GAUGE:0:U
memcached_octets rx:DERIVE:0:U, tx:DERIVE:0:U
memcached_ops value:DERIVE:0:U
memory value:GAUGE:0:281474976710656
multimeter value:GAUGE:U:U
mutex_operations value:DERIVE:0:U
mysql_commands value:DERIVE:0:U
mysql_handler value:DERIVE:0:U
mysql_locks value:DERIVE:0:U
mysql_log_position value:DERIVE:0:U
mysql_octets rx:DERIVE:0:U, tx:DERIVE:0:U
nfs_procedure value:DERIVE:0:U
nginx_connections value:GAUGE:0:U
nginx_requests value:DERIVE:0:U
node_octets rx:DERIVE:0:U, tx:DERIVE:0:U
node_rssi value:GAUGE:0:255
node_stat value:DERIVE:0:U
node_tx_rate value:GAUGE:0:127
objects value:GAUGE:0:U
operations value:DERIVE:0:U
percent value:GAUGE:0:100.1
percent_bytes value:GAUGE:0:100.1
percent_inodes value:GAUGE:0:100.1
pf_counters value:DERIVE:0:U
pf_limits value:DERIVE:0:U
pf_source value:DERIVE:0:U
pf_states value:GAUGE:0:U
pf_state value:DERIVE:0:U
pg_blks value:DERIVE:0:U
pg_db_size value:GAUGE:0:U
pg_n_tup_c value:DERIVE:0:U
pg_n_tup_g value:GAUGE:0:U
pg_numbackends value:GAUGE:0:U
pg_scan value:DERIVE:0:U
pg_xact value:DERIVE:0:U
ping_droprate value:GAUGE:0:100
ping_stddev value:GAUGE:0:65535
ping value:GAUGE:0:65535
players value:GAUGE:0:1000000
power value:GAUGE:0:U
protocol_counter value:DERIVE:0:U
ps_code value:GAUGE:0:9223372036854775807
ps_count processes:GAUGE:0:1000000, threads:GAUGE:0:1000000
ps_cputime user:DERIVE:0:U, syst:DERIVE:0:U
ps_data value:GAUGE:0:9223372036854775807
ps_disk_octets read:DERIVE:0:U, write:DERIVE:0:U
ps_disk_ops read:DERIVE:0:U, write:DERIVE:0:U
ps_pagefaults minflt:DERIVE:0:U, majflt:DERIVE:0:U
ps_rss value:GAUGE:0:9223372036854775807
ps_stacksize value:GAUGE:0:9223372036854775807
ps_state value:GAUGE:0:65535
ps_vm value:GAUGE:0:9223372036854775807
queue_length value:GAUGE:0:U
records value:GAUGE:0:U
requests value:GAUGE:0:U
response_time value:GAUGE:0:U
response_code value:GAUGE:0:U
route_etx value:GAUGE:0:U
route_metric value:GAUGE:0:U
routes value:GAUGE:0:U
serial_octets rx:DERIVE:0:U, tx:DERIVE:0:U
signal_noise value:GAUGE:U:0
signal_power value:GAUGE:U:0
signal_quality value:GAUGE:0:U
snr value:GAUGE:0:U
spam_check value:GAUGE:0:U
spam_score value:GAUGE:U:U
spl value:GAUGE:U:U
swap_io value:DERIVE:0:U
swap value:GAUGE:0:1099511627776
tcp_connections value:GAUGE:0:4294967295
temperature value:GAUGE:U:U
threads value:GAUGE:0:U
time_dispersion value:GAUGE:-1000000:1000000
timeleft value:GAUGE:0:U
time_offset value:GAUGE:-1000000:1000000
total_bytes value:DERIVE:0:U
total_connections value:DERIVE:0:U
total_objects value:DERIVE:0:U
total_operations value:DERIVE:0:U
total_requests value:DERIVE:0:U
total_sessions value:DERIVE:0:U
total_threads value:DERIVE:0:U
total_time_in_ms value:DERIVE:0:U
total_values value:DERIVE:0:U
uptime value:GAUGE:0:4294967295
users value:GAUGE:0:65535
vcl value:GAUGE:0:65535
vcpu value:GAUGE:0:U
virt_cpu_total value:DERIVE:0:U
virt_vcpu value:DERIVE:0:U
vmpage_action value:DERIVE:0:U
vmpage_faults minflt:DERIVE:0:U, majflt:DERIVE:0:U
vmpage_io in:DERIVE:0:U, out:DERIVE:0:U
vmpage_number value:GAUGE:0:4294967295
volatile_changes value:GAUGE:0:U
voltage_threshold value:GAUGE:U:U, threshold:GAUGE:U:U
voltage value:GAUGE:U:U
vs_memory value:GAUGE:0:9223372036854775807
vs_processes value:GAUGE:0:65535
vs_threads value:GAUGE:0:65535
#
# Legacy types
# (required for the v5 upgrade target)
#
arc_counts demand_data:COUNTER:0:U, demand_metadata:COUNTER:0:U, prefetch_data:COUNTER:0:U, prefetch_metadata:COUNTER:0:U
arc_l2_bytes read:COUNTER:0:U, write:COUNTER:0:U
arc_l2_size value:GAUGE:0:U
arc_ratio value:GAUGE:0:U
arc_size current:GAUGE:0:U, target:GAUGE:0:U, minlimit:GAUGE:0:U, maxlimit:GAUGE:0:U
mysql_qcache hits:COUNTER:0:U, inserts:COUNTER:0:U, not_cached:COUNTER:0:U, lowmem_prunes:COUNTER:0:U, queries_in_cache:GAUGE:0:U
mysql_threads running:GAUGE:0:U, connected:GAUGE:0:U, cached:GAUGE:0:U, created:COUNTER:0:U

44
Godeps/_workspace/src/github.com/influxdb/influxdb/services/collectd/config.go generated vendored
View File

@ -1,44 +0,0 @@
package collectd
import (
"time"
"github.com/influxdb/influxdb/toml"
)
const (
DefaultBindAddress = ":25826"
DefaultDatabase = "collectd"
DefaultRetentionPolicy = ""
DefaultBatchSize = 5000
DefaultBatchDuration = toml.Duration(10 * time.Second)
DefaultTypesDB = "/usr/share/collectd/types.db"
)
// Config represents a configuration for the collectd service.
type Config struct {
Enabled bool `toml:"enabled"`
BindAddress string `toml:"bind-address"`
Database string `toml:"database"`
RetentionPolicy string `toml:"retention-policy"`
BatchSize int `toml:"batch-size"`
BatchDuration toml.Duration `toml:"batch-timeout"`
TypesDB string `toml:"typesdb"`
}
// NewConfig returns a new instance of Config with defaults.
func NewConfig() Config {
return Config{
BindAddress: DefaultBindAddress,
Database: DefaultDatabase,
RetentionPolicy: DefaultRetentionPolicy,
BatchSize: DefaultBatchSize,
BatchDuration: DefaultBatchDuration,
TypesDB: DefaultTypesDB,
}
}

32
Godeps/_workspace/src/github.com/influxdb/influxdb/services/collectd/config_test.go generated vendored
View File

@ -1,32 +0,0 @@
package collectd_test
import (
"testing"
"github.com/BurntSushi/toml"
"github.com/influxdb/influxdb/services/collectd"
)
func TestConfig_Parse(t *testing.T) {
// Parse configuration.
var c collectd.Config
if _, err := toml.Decode(`
enabled = true
bind-address = ":9000"
database = "xxx"
typesdb = "yyy"
`, &c); err != nil {
t.Fatal(err)
}
// Validate configuration.
if c.Enabled != true {
t.Fatalf("unexpected enabled: %v", c.Enabled)
} else if c.BindAddress != ":9000" {
t.Fatalf("unexpected bind address: %s", c.BindAddress)
} else if c.Database != "xxx" {
t.Fatalf("unexpected database: %s", c.Database)
} else if c.TypesDB != "yyy" {
t.Fatalf("unexpected types db: %s", c.TypesDB)
}
}

278
Godeps/_workspace/src/github.com/influxdb/influxdb/services/collectd/service.go generated vendored
View File

@ -1,278 +0,0 @@
package collectd
import (
"fmt"
"log"
"net"
"os"
"sync"
"time"
"github.com/influxdb/influxdb/cluster"
"github.com/influxdb/influxdb/meta"
"github.com/influxdb/influxdb/tsdb"
"github.com/kimor79/gollectd"
)
const leaderWaitTimeout = 30 * time.Second
// pointsWriter is an internal interface to make testing easier.
type pointsWriter interface {
WritePoints(p *cluster.WritePointsRequest) error
}
// metaStore is an internal interface to make testing easier.
type metaStore interface {
WaitForLeader(d time.Duration) error
CreateDatabaseIfNotExists(name string) (*meta.DatabaseInfo, error)
}
// Service represents a UDP server which receives metrics in collectd's binary
// protocol and stores them in InfluxDB.
type Service struct {
Config *Config
MetaStore metaStore
PointsWriter pointsWriter
Logger *log.Logger
wg sync.WaitGroup
err chan error
stop chan struct{}
ln *net.UDPConn
batcher *tsdb.PointBatcher
typesdb gollectd.Types
addr net.Addr
}
// NewService returns a new instance of the collectd service.
func NewService(c Config) *Service {
s := &Service{
Config: &c,
Logger: log.New(os.Stderr, "[collectd] ", log.LstdFlags),
err: make(chan error),
}
return s
}
// Open starts the service.
func (s *Service) Open() error {
s.Logger.Printf("Starting collectd service")
if s.Config.BindAddress == "" {
return fmt.Errorf("bind address is blank")
} else if s.Config.Database == "" {
return fmt.Errorf("database name is blank")
} else if s.PointsWriter == nil {
return fmt.Errorf("PointsWriter is nil")
}
if err := s.MetaStore.WaitForLeader(leaderWaitTimeout); err != nil {
s.Logger.Printf("Failed to detect a cluster leader: %s", err.Error())
return err
}
if _, err := s.MetaStore.CreateDatabaseIfNotExists(s.Config.Database); err != nil {
s.Logger.Printf("Failed to ensure target database %s exists: %s", s.Config.Database, err.Error())
return err
}
if s.typesdb == nil {
// Open collectd types.
typesdb, err := gollectd.TypesDBFile(s.Config.TypesDB)
if err != nil {
return fmt.Errorf("Open(): %s", err)
}
s.typesdb = typesdb
}
// Resolve our address.
addr, err := net.ResolveUDPAddr("udp", s.Config.BindAddress)
if err != nil {
return fmt.Errorf("unable to resolve UDP address: %s", err)
}
s.addr = addr
// Start listening
ln, err := net.ListenUDP("udp", addr)
if err != nil {
return fmt.Errorf("unable to listen on UDP: %s", err)
}
s.ln = ln
s.Logger.Println("Listening on UDP: ", ln.LocalAddr().String())
// Start the points batcher.
s.batcher = tsdb.NewPointBatcher(s.Config.BatchSize, time.Duration(s.Config.BatchDuration))
s.batcher.Start()
// Create channel and wait group for signalling goroutines to stop.
s.stop = make(chan struct{})
s.wg.Add(2)
// Start goroutines that process collectd packets.
go s.serve()
go s.writePoints()
return nil
}
// Close stops the service.
func (s *Service) Close() error {
// Close the connection, and wait for the goroutine to exit.
if s.stop != nil {
close(s.stop)
}
if s.ln != nil {
s.ln.Close()
}
if s.batcher != nil {
s.batcher.Stop()
}
s.wg.Wait()
// Release all remaining resources.
s.stop = nil
s.ln = nil
s.batcher = nil
s.Logger.Println("collectd UDP closed")
return nil
}
// SetLogger sets the internal logger to the logger passed in.
func (s *Service) SetLogger(l *log.Logger) {
s.Logger = l
}
// SetTypes sets collectd types db.
func (s *Service) SetTypes(types string) (err error) {
s.typesdb, err = gollectd.TypesDB([]byte(types))
return
}
// Err returns a channel for fatal errors that occur on go routines.
func (s *Service) Err() chan error { return s.err }
// Addr returns the listener's address. Returns nil if listener is closed.
func (s *Service) Addr() net.Addr {
return s.ln.LocalAddr()
}
func (s *Service) serve() {
defer s.wg.Done()
// From https://collectd.org/wiki/index.php/Binary_protocol
// 1024 bytes (payload only, not including UDP / IP headers)
// In versions 4.0 through 4.7, the receive buffer has a fixed size
// of 1024 bytes. When longer packets are received, the trailing data
// is simply ignored. Since version 4.8, the buffer size can be
// configured. Version 5.0 will increase the default buffer size to
// 1452 bytes (the maximum payload size when using UDP/IPv6 over
// Ethernet).
buffer := make([]byte, 1452)
for {
select {
case <-s.stop:
// We closed the connection, time to go.
return
default:
// Keep processing.
}
n, _, err := s.ln.ReadFromUDP(buffer)
if err != nil {
s.Logger.Printf("collectd ReadFromUDP error: %s", err)
continue
}
if n > 0 {
s.handleMessage(buffer[:n])
}
}
}
func (s *Service) handleMessage(buffer []byte) {
packets, err := gollectd.Packets(buffer, s.typesdb)
if err != nil {
s.Logger.Printf("Collectd parse error: %s", err)
return
}
for _, packet := range *packets {
points := Unmarshal(&packet)
for _, p := range points {
s.batcher.In() <- p
}
}
}
func (s *Service) writePoints() {
defer s.wg.Done()
for {
select {
case <-s.stop:
return
case batch := <-s.batcher.Out():
req := &cluster.WritePointsRequest{
Database: s.Config.Database,
RetentionPolicy: s.Config.RetentionPolicy,
ConsistencyLevel: cluster.ConsistencyLevelAny,
Points: batch,
}
if err := s.PointsWriter.WritePoints(req); err != nil {
s.Logger.Printf("failed to write batch: %s", err)
continue
}
}
}
}
// Unmarshal translates a collectd packet into InfluxDB data points.
func Unmarshal(packet *gollectd.Packet) []tsdb.Point {
// Prefer high resolution timestamp.
var timestamp time.Time
if packet.TimeHR > 0 {
// TimeHR is "near" nanosecond measurement, but not exactly nanasecond time
// Since we store time in microseconds, we round here (mostly so tests will work easier)
sec := packet.TimeHR >> 30
// Shifting, masking, and dividing by 1 billion to get nanoseconds.
nsec := ((packet.TimeHR & 0x3FFFFFFF) << 30) / 1000 / 1000 / 1000
timestamp = time.Unix(int64(sec), int64(nsec)).UTC().Round(time.Microsecond)
} else {
// If we don't have high resolution time, fall back to basic unix time
timestamp = time.Unix(int64(packet.Time), 0).UTC()
}
var points []tsdb.Point
for i := range packet.Values {
name := fmt.Sprintf("%s_%s", packet.Plugin, packet.Values[i].Name)
tags := make(map[string]string)
fields := make(map[string]interface{})
fields["value"] = packet.Values[i].Value
if packet.Hostname != "" {
tags["host"] = packet.Hostname
}
if packet.PluginInstance != "" {
tags["instance"] = packet.PluginInstance
}
if packet.Type != "" {
tags["type"] = packet.Type
}
if packet.TypeInstance != "" {
tags["type_instance"] = packet.TypeInstance
}
p := tsdb.NewPoint(name, tags, fields, timestamp)
points = append(points, p)
}
return points
}
// assert will panic with a given formatted message if the given condition is false.
func assert(condition bool, msg string, v ...interface{}) {
if !condition {
panic(fmt.Sprintf("assert failed: "+msg, v...))
}
}

501
Godeps/_workspace/src/github.com/influxdb/influxdb/services/collectd/service_test.go generated vendored
View File

@ -1,501 +0,0 @@
package collectd
import (
"encoding/hex"
"errors"
"io/ioutil"
"log"
"net"
"testing"
"time"
"github.com/influxdb/influxdb/cluster"
"github.com/influxdb/influxdb/meta"
"github.com/influxdb/influxdb/toml"
"github.com/influxdb/influxdb/tsdb"
)
// Test that the service checks / creates the target database on startup.
func TestService_CreatesDatabase(t *testing.T) {
t.Parallel()
s := newTestService(1, time.Second)
createDatabaseCalled := false
ms := &testMetaStore{}
ms.CreateDatabaseIfNotExistsFn = func(name string) (*meta.DatabaseInfo, error) {
if name != s.Config.Database {
t.Errorf("\n\texp = %s\n\tgot = %s\n", s.Config.Database, name)
}
createDatabaseCalled = true
return nil, nil
}
s.Service.MetaStore = ms
s.Open()
s.Close()
if !createDatabaseCalled {
t.Errorf("CreateDatabaseIfNotExists should have been called when the service opened.")
}
}
// Test that the collectd service correctly batches points by BatchSize.
func TestService_BatchSize(t *testing.T) {
t.Parallel()
totalPoints := len(expPoints)
// Batch sizes that totalTestPoints divide evenly by.
batchSizes := []int{1, 2, 13}
for _, batchSize := range batchSizes {
func() {
s := newTestService(batchSize, time.Second)
pointCh := make(chan tsdb.Point)
s.MetaStore.CreateDatabaseIfNotExistsFn = func(name string) (*meta.DatabaseInfo, error) { return nil, nil }
s.PointsWriter.WritePointsFn = func(req *cluster.WritePointsRequest) error {
if len(req.Points) != batchSize {
t.Errorf("\n\texp = %d\n\tgot = %d\n", batchSize, len(req.Points))
}
for _, p := range req.Points {
pointCh <- p
}
return nil
}
if err := s.Open(); err != nil {
t.Fatal(err)
}
defer func() { t.Log("closing service"); s.Close() }()
// Get the address & port the service is listening on for collectd data.
addr := s.Addr()
conn, err := net.Dial("udp", addr.String())
if err != nil {
t.Fatal(err)
}
// Send the test data to the service.
if n, err := conn.Write(testData); err != nil {
t.Fatal(err)
} else if n != len(testData) {
t.Fatalf("only sent %d of %d bytes", n, len(testData))
}
points := []tsdb.Point{}
Loop:
for {
select {
case p := <-pointCh:
points = append(points, p)
if len(points) == totalPoints {
break Loop
}
case <-time.After(time.Second):
t.Logf("exp %d points, got %d", totalPoints, len(points))
t.Fatal("timed out waiting for points from collectd service")
}
}
if len(points) != totalPoints {
t.Fatalf("exp %d points, got %d", totalPoints, len(points))
}
for i, exp := range expPoints {
got := points[i].String()
if got != exp {
t.Fatalf("\n\texp = %s\n\tgot = %s\n", exp, got)
}
}
}()
}
}
// Test that the collectd service correctly batches points using BatchDuration.
func TestService_BatchDuration(t *testing.T) {
t.Parallel()
totalPoints := len(expPoints)
s := newTestService(5000, 250*time.Millisecond)
pointCh := make(chan tsdb.Point, 1000)
s.MetaStore.CreateDatabaseIfNotExistsFn = func(name string) (*meta.DatabaseInfo, error) { return nil, nil }
s.PointsWriter.WritePointsFn = func(req *cluster.WritePointsRequest) error {
for _, p := range req.Points {
pointCh <- p
}
return nil
}
if err := s.Open(); err != nil {
t.Fatal(err)
}
defer func() { t.Log("closing service"); s.Close() }()
// Get the address & port the service is listening on for collectd data.
addr := s.Addr()
conn, err := net.Dial("udp", addr.String())
if err != nil {
t.Fatal(err)
}
// Send the test data to the service.
if n, err := conn.Write(testData); err != nil {
t.Fatal(err)
} else if n != len(testData) {
t.Fatalf("only sent %d of %d bytes", n, len(testData))
}
points := []tsdb.Point{}
Loop:
for {
select {
case p := <-pointCh:
points = append(points, p)
if len(points) == totalPoints {
break Loop
}
case <-time.After(time.Second):
t.Logf("exp %d points, got %d", totalPoints, len(points))
t.Fatal("timed out waiting for points from collectd service")
}
}
if len(points) != totalPoints {
t.Fatalf("exp %d points, got %d", totalPoints, len(points))
}
for i, exp := range expPoints {
got := points[i].String()
if got != exp {
t.Fatalf("\n\texp = %s\n\tgot = %s\n", exp, got)
}
}
}
type testService struct {
*Service
MetaStore testMetaStore
PointsWriter testPointsWriter
}
func newTestService(batchSize int, batchDuration time.Duration) *testService {
s := &testService{
Service: NewService(Config{
BindAddress: "127.0.0.1:0",
Database: "collectd_test",
BatchSize: batchSize,
BatchDuration: toml.Duration(batchDuration),
}),
}
s.Service.PointsWriter = &s.PointsWriter
s.Service.MetaStore = &s.MetaStore
// Set the collectd types using test string.
if err := s.SetTypes(typesDBText); err != nil {
panic(err)
}
if !testing.Verbose() {
s.Logger = log.New(ioutil.Discard, "", log.LstdFlags)
}
return s
}
type testPointsWriter struct {
WritePointsFn func(*cluster.WritePointsRequest) error
}
func (w *testPointsWriter) WritePoints(p *cluster.WritePointsRequest) error {
return w.WritePointsFn(p)
}
type testMetaStore struct {
CreateDatabaseIfNotExistsFn func(name string) (*meta.DatabaseInfo, error)
//DatabaseFn func(name string) (*meta.DatabaseInfo, error)
}
func (ms *testMetaStore) CreateDatabaseIfNotExists(name string) (*meta.DatabaseInfo, error) {
return ms.CreateDatabaseIfNotExistsFn(name)
}
func (ms *testMetaStore) WaitForLeader(d time.Duration) error {
return nil
}
func wait(c chan struct{}, d time.Duration) (err error) {
select {
case <-c:
case <-time.After(d):
err = errors.New("timed out")
}
return
}
func waitInt(c chan int, d time.Duration) (i int, err error) {
select {
case i = <-c:
case <-time.After(d):
err = errors.New("timed out")
}
return
}
func check(err error) {
if err != nil {
panic(err)
}
}
// Raw data sent by collectd, captured using Wireshark.
var testData = func() []byte {
b, err := hex.DecodeString("000000167066312d36322d3231302d39342d313733000001000c00000000544928ff0007000c00000000000000050002000c656e74726f7079000004000c656e74726f7079000006000f0001010000000000007240000200086370750000030006310000040008637075000005000969646c65000006000f0001000000000000a674620005000977616974000006000f0001000000000000000000000200076466000003000500000400076466000005000d6c6976652d636f7700000600180002010100000000a090b641000000a0cb6a2742000200086370750000030006310000040008637075000005000e696e74657272757074000006000f00010000000000000000fe0005000c736f6674697271000006000f000100000000000000000000020007646600000300050000040007646600000500096c6976650000060018000201010000000000000000000000e0ec972742000200086370750000030006310000040008637075000005000a737465616c000006000f00010000000000000000000003000632000005000975736572000006000f0001000000000000005f36000500096e696365000006000f0001000000000000000ad80002000e696e746572666163650000030005000004000e69665f6f6374657473000005000b64756d6d79300000060018000200000000000000000000000000000000041a000200076466000004000764660000050008746d70000006001800020101000000000000f240000000a0ea972742000200086370750000030006320000040008637075000005000b73797374656d000006000f00010000000000000045d30002000e696e746572666163650000030005000004000f69665f7061636b657473000005000b64756d6d79300000060018000200000000000000000000000000000000000f000200086370750000030006320000040008637075000005000969646c65000006000f0001000000000000a66480000200076466000003000500000400076466000005000d72756e2d6c6f636b000006001800020101000000000000000000000000000054410002000e696e74657266616365000004000e69665f6572726f7273000005000b64756d6d793000000600180002000000000000000000000000000000000000000200086370750000030006320000040008637075000005000977616974000006000f00010000000000000000000005000e696e74657272757074000006000f0001000000000000000132")
check(err)
return b
}()
var expPoints = []string{
"entropy_value,host=pf1-62-210-94-173,type=entropy value=288.0 1414080767000000000",
"cpu_value,host=pf1-62-210-94-173,instance=1,type=cpu,type_instance=idle value=10908770.0 1414080767000000000",
"cpu_value,host=pf1-62-210-94-173,instance=1,type=cpu,type_instance=wait value=0.0 1414080767000000000",
"df_used,host=pf1-62-210-94-173,type=df,type_instance=live-cow value=378576896.0 1414080767000000000",
"df_free,host=pf1-62-210-94-173,type=df,type_instance=live-cow value=50287988736.0 1414080767000000000",
"cpu_value,host=pf1-62-210-94-173,instance=1,type=cpu,type_instance=interrupt value=254.0 1414080767000000000",
"cpu_value,host=pf1-62-210-94-173,instance=1,type=cpu,type_instance=softirq value=0.0 1414080767000000000",
"df_used,host=pf1-62-210-94-173,type=df,type_instance=live value=0.0 1414080767000000000",
"df_free,host=pf1-62-210-94-173,type=df,type_instance=live value=50666565632.0 1414080767000000000",
"cpu_value,host=pf1-62-210-94-173,instance=1,type=cpu,type_instance=steal value=0.0 1414080767000000000",
"cpu_value,host=pf1-62-210-94-173,instance=2,type=cpu,type_instance=user value=24374.0 1414080767000000000",
"cpu_value,host=pf1-62-210-94-173,instance=2,type=cpu,type_instance=nice value=2776.0 1414080767000000000",
"interface_rx,host=pf1-62-210-94-173,type=if_octets,type_instance=dummy0 value=0.0 1414080767000000000",
"interface_tx,host=pf1-62-210-94-173,type=if_octets,type_instance=dummy0 value=1050.0 1414080767000000000",
"df_used,host=pf1-62-210-94-173,type=df,type_instance=tmp value=73728.0 1414080767000000000",
"df_free,host=pf1-62-210-94-173,type=df,type_instance=tmp value=50666491904.0 1414080767000000000",
"cpu_value,host=pf1-62-210-94-173,instance=2,type=cpu,type_instance=system value=17875.0 1414080767000000000",
"interface_rx,host=pf1-62-210-94-173,type=if_packets,type_instance=dummy0 value=0.0 1414080767000000000",
"interface_tx,host=pf1-62-210-94-173,type=if_packets,type_instance=dummy0 value=15.0 1414080767000000000",
"cpu_value,host=pf1-62-210-94-173,instance=2,type=cpu,type_instance=idle value=10904704.0 1414080767000000000",
"df_used,host=pf1-62-210-94-173,type=df,type_instance=run-lock value=0.0 1414080767000000000",
"df_free,host=pf1-62-210-94-173,type=df,type_instance=run-lock value=5242880.0 1414080767000000000",
"interface_rx,host=pf1-62-210-94-173,type=if_errors,type_instance=dummy0 value=0.0 1414080767000000000",
"interface_tx,host=pf1-62-210-94-173,type=if_errors,type_instance=dummy0 value=0.0 1414080767000000000",
"cpu_value,host=pf1-62-210-94-173,instance=2,type=cpu,type_instance=wait value=0.0 1414080767000000000",
"cpu_value,host=pf1-62-210-94-173,instance=2,type=cpu,type_instance=interrupt value=306.0 1414080767000000000",
}
// Taken from /usr/share/collectd/types.db on a Ubuntu system
var typesDBText = `
absolute value:ABSOLUTE:0:U
apache_bytes value:DERIVE:0:U
apache_connections value:GAUGE:0:65535
apache_idle_workers value:GAUGE:0:65535
apache_requests value:DERIVE:0:U
apache_scoreboard value:GAUGE:0:65535
ath_nodes value:GAUGE:0:65535
ath_stat value:DERIVE:0:U
backends value:GAUGE:0:65535
bitrate value:GAUGE:0:4294967295
bytes value:GAUGE:0:U
cache_eviction value:DERIVE:0:U
cache_operation value:DERIVE:0:U
cache_ratio value:GAUGE:0:100
cache_result value:DERIVE:0:U
cache_size value:GAUGE:0:4294967295
charge value:GAUGE:0:U
compression_ratio value:GAUGE:0:2
compression uncompressed:DERIVE:0:U, compressed:DERIVE:0:U
connections value:DERIVE:0:U
conntrack value:GAUGE:0:4294967295
contextswitch value:DERIVE:0:U
counter value:COUNTER:U:U
cpufreq value:GAUGE:0:U
cpu value:DERIVE:0:U
current_connections value:GAUGE:0:U
current_sessions value:GAUGE:0:U
current value:GAUGE:U:U
delay value:GAUGE:-1000000:1000000
derive value:DERIVE:0:U
df_complex value:GAUGE:0:U
df_inodes value:GAUGE:0:U
df used:GAUGE:0:1125899906842623, free:GAUGE:0:1125899906842623
disk_latency read:GAUGE:0:U, write:GAUGE:0:U
disk_merged read:DERIVE:0:U, write:DERIVE:0:U
disk_octets read:DERIVE:0:U, write:DERIVE:0:U
disk_ops_complex value:DERIVE:0:U
disk_ops read:DERIVE:0:U, write:DERIVE:0:U
disk_time read:DERIVE:0:U, write:DERIVE:0:U
dns_answer value:DERIVE:0:U
dns_notify value:DERIVE:0:U
dns_octets queries:DERIVE:0:U, responses:DERIVE:0:U
dns_opcode value:DERIVE:0:U
dns_qtype_cached value:GAUGE:0:4294967295
dns_qtype value:DERIVE:0:U
dns_query value:DERIVE:0:U
dns_question value:DERIVE:0:U
dns_rcode value:DERIVE:0:U
dns_reject value:DERIVE:0:U
dns_request value:DERIVE:0:U
dns_resolver value:DERIVE:0:U
dns_response value:DERIVE:0:U
dns_transfer value:DERIVE:0:U
dns_update value:DERIVE:0:U
dns_zops value:DERIVE:0:U
duration seconds:GAUGE:0:U
email_check value:GAUGE:0:U
email_count value:GAUGE:0:U
email_size value:GAUGE:0:U
entropy value:GAUGE:0:4294967295
fanspeed value:GAUGE:0:U
file_size value:GAUGE:0:U
files value:GAUGE:0:U
fork_rate value:DERIVE:0:U
frequency_offset value:GAUGE:-1000000:1000000
frequency value:GAUGE:0:U
fscache_stat value:DERIVE:0:U
gauge value:GAUGE:U:U
hash_collisions value:DERIVE:0:U
http_request_methods value:DERIVE:0:U
http_requests value:DERIVE:0:U
http_response_codes value:DERIVE:0:U
humidity value:GAUGE:0:100
if_collisions value:DERIVE:0:U
if_dropped rx:DERIVE:0:U, tx:DERIVE:0:U
if_errors rx:DERIVE:0:U, tx:DERIVE:0:U
if_multicast value:DERIVE:0:U
if_octets rx:DERIVE:0:U, tx:DERIVE:0:U
if_packets rx:DERIVE:0:U, tx:DERIVE:0:U
if_rx_errors value:DERIVE:0:U
if_rx_octets value:DERIVE:0:U
if_tx_errors value:DERIVE:0:U
if_tx_octets value:DERIVE:0:U
invocations value:DERIVE:0:U
io_octets rx:DERIVE:0:U, tx:DERIVE:0:U
io_packets rx:DERIVE:0:U, tx:DERIVE:0:U
ipt_bytes value:DERIVE:0:U
ipt_packets value:DERIVE:0:U
irq value:DERIVE:0:U
latency value:GAUGE:0:U
links value:GAUGE:0:U
load shortterm:GAUGE:0:5000, midterm:GAUGE:0:5000, longterm:GAUGE:0:5000
md_disks value:GAUGE:0:U
memcached_command value:DERIVE:0:U
memcached_connections value:GAUGE:0:U
memcached_items value:GAUGE:0:U
memcached_octets rx:DERIVE:0:U, tx:DERIVE:0:U
memcached_ops value:DERIVE:0:U
memory value:GAUGE:0:281474976710656
multimeter value:GAUGE:U:U
mutex_operations value:DERIVE:0:U
mysql_commands value:DERIVE:0:U
mysql_handler value:DERIVE:0:U
mysql_locks value:DERIVE:0:U
mysql_log_position value:DERIVE:0:U
mysql_octets rx:DERIVE:0:U, tx:DERIVE:0:U
nfs_procedure value:DERIVE:0:U
nginx_connections value:GAUGE:0:U
nginx_requests value:DERIVE:0:U
node_octets rx:DERIVE:0:U, tx:DERIVE:0:U
node_rssi value:GAUGE:0:255
node_stat value:DERIVE:0:U
node_tx_rate value:GAUGE:0:127
objects value:GAUGE:0:U
operations value:DERIVE:0:U
percent value:GAUGE:0:100.1
percent_bytes value:GAUGE:0:100.1
percent_inodes value:GAUGE:0:100.1
pf_counters value:DERIVE:0:U
pf_limits value:DERIVE:0:U
pf_source value:DERIVE:0:U
pf_states value:GAUGE:0:U
pf_state value:DERIVE:0:U
pg_blks value:DERIVE:0:U
pg_db_size value:GAUGE:0:U
pg_n_tup_c value:DERIVE:0:U
pg_n_tup_g value:GAUGE:0:U
pg_numbackends value:GAUGE:0:U
pg_scan value:DERIVE:0:U
pg_xact value:DERIVE:0:U
ping_droprate value:GAUGE:0:100
ping_stddev value:GAUGE:0:65535
ping value:GAUGE:0:65535
players value:GAUGE:0:1000000
power value:GAUGE:0:U
protocol_counter value:DERIVE:0:U
ps_code value:GAUGE:0:9223372036854775807
ps_count processes:GAUGE:0:1000000, threads:GAUGE:0:1000000
ps_cputime user:DERIVE:0:U, syst:DERIVE:0:U
ps_data value:GAUGE:0:9223372036854775807
ps_disk_octets read:DERIVE:0:U, write:DERIVE:0:U
ps_disk_ops read:DERIVE:0:U, write:DERIVE:0:U
ps_pagefaults minflt:DERIVE:0:U, majflt:DERIVE:0:U
ps_rss value:GAUGE:0:9223372036854775807
ps_stacksize value:GAUGE:0:9223372036854775807
ps_state value:GAUGE:0:65535
ps_vm value:GAUGE:0:9223372036854775807
queue_length value:GAUGE:0:U
records value:GAUGE:0:U
requests value:GAUGE:0:U
response_time value:GAUGE:0:U
response_code value:GAUGE:0:U
route_etx value:GAUGE:0:U
route_metric value:GAUGE:0:U
routes value:GAUGE:0:U
serial_octets rx:DERIVE:0:U, tx:DERIVE:0:U
signal_noise value:GAUGE:U:0
signal_power value:GAUGE:U:0
signal_quality value:GAUGE:0:U
snr value:GAUGE:0:U
spam_check value:GAUGE:0:U
spam_score value:GAUGE:U:U
spl value:GAUGE:U:U
swap_io value:DERIVE:0:U
swap value:GAUGE:0:1099511627776
tcp_connections value:GAUGE:0:4294967295
temperature value:GAUGE:U:U
threads value:GAUGE:0:U
time_dispersion value:GAUGE:-1000000:1000000
timeleft value:GAUGE:0:U
time_offset value:GAUGE:-1000000:1000000
total_bytes value:DERIVE:0:U
total_connections value:DERIVE:0:U
total_objects value:DERIVE:0:U
total_operations value:DERIVE:0:U
total_requests value:DERIVE:0:U
total_sessions value:DERIVE:0:U
total_threads value:DERIVE:0:U
total_time_in_ms value:DERIVE:0:U
total_values value:DERIVE:0:U
uptime value:GAUGE:0:4294967295
users value:GAUGE:0:65535
vcl value:GAUGE:0:65535
vcpu value:GAUGE:0:U
virt_cpu_total value:DERIVE:0:U
virt_vcpu value:DERIVE:0:U
vmpage_action value:DERIVE:0:U
vmpage_faults minflt:DERIVE:0:U, majflt:DERIVE:0:U
vmpage_io in:DERIVE:0:U, out:DERIVE:0:U
vmpage_number value:GAUGE:0:4294967295
volatile_changes value:GAUGE:0:U
voltage_threshold value:GAUGE:U:U, threshold:GAUGE:U:U
voltage value:GAUGE:U:U
vs_memory value:GAUGE:0:9223372036854775807
vs_processes value:GAUGE:0:65535
vs_threads value:GAUGE:0:65535
#
# Legacy types
# (required for the v5 upgrade target)
#
arc_counts demand_data:COUNTER:0:U, demand_metadata:COUNTER:0:U, prefetch_data:COUNTER:0:U, prefetch_metadata:COUNTER:0:U
arc_l2_bytes read:COUNTER:0:U, write:COUNTER:0:U
arc_l2_size value:GAUGE:0:U
arc_ratio value:GAUGE:0:U
arc_size current:GAUGE:0:U, target:GAUGE:0:U, minlimit:GAUGE:0:U, maxlimit:GAUGE:0:U
mysql_qcache hits:COUNTER:0:U, inserts:COUNTER:0:U, not_cached:COUNTER:0:U, lowmem_prunes:COUNTER:0:U, queries_in_cache:GAUGE:0:U
mysql_threads running:GAUGE:0:U, connected:GAUGE:0:U, cached:GAUGE:0:U, created:COUNTER:0:U
`

3
Godeps/_workspace/src/github.com/influxdb/influxdb/services/collectd/test_client/README.md generated vendored
View File

@ -1,3 +0,0 @@
collectD Client
============
This directory contains code for generating collectd load.

71
Godeps/_workspace/src/github.com/influxdb/influxdb/services/collectd/test_client/client.go generated vendored
View File

@ -1,71 +0,0 @@
package main
import (
"collectd.org/api"
"collectd.org/network"
"flag"
"fmt"
"math/rand"
"os"
"strconv"
"time"
)
var nMeasurments = flag.Int("m", 1, "Number of measurements")
var tagVariance = flag.Int("v", 1, "Number of values per tag. Client is fixed at one tag")
var rate = flag.Int("r", 1, "Number of points per second")
var total = flag.Int("t", -1, "Total number of points to send (default is no limit)")
var host = flag.String("u", "127.0.0.1:25826", "Destination host in the form host:port")
func main() {
flag.Parse()
conn, err := network.Dial(*host, network.ClientOptions{})
if err != nil {
fmt.Println(err)
os.Exit(1)
}
defer conn.Close()
rateLimiter := make(chan int, *rate)
go func() {
ticker := time.NewTicker(time.Second)
for {
select {
case <-ticker.C:
for i := 0; i < *rate; i++ {
rateLimiter <- i
}
}
}
}()
nSent := 0
for {
if nSent >= *total && *total > 0 {
break
}
<-rateLimiter
vl := api.ValueList{
Identifier: api.Identifier{
Host: "tagvalue" + strconv.Itoa(int(rand.Int31n(int32(*tagVariance)))),
Plugin: "golang" + strconv.Itoa(int(rand.Int31n(int32(*nMeasurments)))),
Type: "gauge",
},
Time: time.Now(),
Interval: 10 * time.Second,
Values: []api.Value{api.Gauge(42.0)},
}
if err := conn.Write(vl); err != nil {
fmt.Println(err)
os.Exit(1)
}
conn.Flush()
nSent = nSent + 1
}
fmt.Println("Number of points sent:", nSent)
}

65
Godeps/_workspace/src/github.com/influxdb/influxdb/services/continuous_querier/config.go generated vendored
View File

@ -1,65 +0,0 @@
package continuous_querier
import (
"time"
"github.com/influxdb/influxdb/toml"
)
const (
DefaultRecomputePreviousN = 2
DefaultRecomputeNoOlderThan = 10 * time.Minute
DefaultComputeRunsPerInterval = 10
DefaultComputeNoMoreThan = 2 * time.Minute
)
// Config represents a configuration for the continuous query service.
type Config struct {
// Enables logging in CQ service to display when CQ's are processed and how many points are wrote.
LogEnabled bool `toml:"log-enabled"`
// If this flag is set to false, both the brokers and data nodes should ignore any CQ processing.
Enabled bool `toml:"enabled"`
// when continuous queries are run we'll automatically recompute previous intervals
// in case lagged data came in. Set to zero if you never have lagged data. We do
// it this way because invalidating previously computed intervals would be insanely hard
// and expensive.
RecomputePreviousN int `toml:"recompute-previous-n"`
// The RecomputePreviousN setting provides guidance for how far back to recompute, the RecomputeNoOlderThan
// setting sets a ceiling on how far back in time it will go. For example, if you have 2 PreviousN
// and have this set to 10m, then we'd only compute the previous two intervals for any
// CQs that have a group by time <= 5m. For all others, we'd only recompute the previous window
RecomputeNoOlderThan toml.Duration `toml:"recompute-no-older-than"`
// ComputeRunsPerInterval will determine how many times the current and previous N intervals
// will be computed. The group by time will be divided by this and it will get computed this many times:
// group by time seconds / runs per interval
// This will give partial results for current group by intervals and will determine how long it will
// be until lagged data is recomputed. For example, if this number is 10 and the group by time is 10m, it
// will be a minute past the previous 10m bucket of time before lagged data is picked up
ComputeRunsPerInterval int `toml:"compute-runs-per-interval"`
// ComputeNoMoreThan paired with the RunsPerInterval will determine the ceiling of how many times smaller
// group by times will be computed. For example, if you have RunsPerInterval set to 10 and this setting
// to 1m. Then for a group by time(1m) will actually only get computed once per interval (and once per PreviousN).
// If you have a group by time(5m) then you'll get five computes per interval. Any group by time window larger
// than 10m will get computed 10 times for each interval.
ComputeNoMoreThan toml.Duration `toml:"compute-no-more-than"`
}
// NewConfig returns a new instance of Config with defaults.
func NewConfig() Config {
return Config{
LogEnabled: true,
Enabled: true,
RecomputePreviousN: DefaultRecomputePreviousN,
RecomputeNoOlderThan: toml.Duration(DefaultRecomputeNoOlderThan),
ComputeRunsPerInterval: DefaultComputeRunsPerInterval,
ComputeNoMoreThan: toml.Duration(DefaultComputeNoMoreThan),
}
}

36
Godeps/_workspace/src/github.com/influxdb/influxdb/services/continuous_querier/config_test.go generated vendored
View File

@ -1,36 +0,0 @@
package continuous_querier_test
import (
"testing"
"time"
"github.com/BurntSushi/toml"
"github.com/influxdb/influxdb/services/continuous_querier"
)
func TestConfig_Parse(t *testing.T) {
// Parse configuration.
var c continuous_querier.Config
if _, err := toml.Decode(`
recompute-previous-n = 1
recompute-no-older-than = "10s"
compute-runs-per-interval = 2
compute-no-more-than = "20s"
enabled = true
`, &c); err != nil {
t.Fatal(err)
}
// Validate configuration.
if c.RecomputePreviousN != 1 {
t.Fatalf("unexpected recompute previous n: %d", c.RecomputePreviousN)
} else if time.Duration(c.RecomputeNoOlderThan) != 10*time.Second {
t.Fatalf("unexpected recompute no older than: %v", c.RecomputeNoOlderThan)
} else if c.ComputeRunsPerInterval != 2 {
t.Fatalf("unexpected compute runs per interval: %d", c.ComputeRunsPerInterval)
} else if time.Duration(c.ComputeNoMoreThan) != 20*time.Second {
t.Fatalf("unexpected compute no more than: %v", c.ComputeNoMoreThan)
} else if c.Enabled != true {
t.Fatalf("unexpected enabled: %v", c.Enabled)
}
}

236
Godeps/_workspace/src/github.com/influxdb/influxdb/services/continuous_querier/continuous_queries.md generated vendored
View File

@ -1,236 +0,0 @@
# Continuous Queries
This document lays out continuous queries and a proposed architecture for how they'll work within an InfluxDB cluster.
## Definition of Continuous Queries
Continuous queries serve two purposes in InfluxDB:
1. Combining many series into a single series (i.e. removing 1 or more tag dimensions to make queries more efficient)
2. Aggregating and downsampling series
The purpose of both types of continuous queries is to duplicate or downsample data automatically in the background to make querying thier results fast and efficient. Think of them as another way to create indexes on data.
Generally, there are continuous queries that create copyies of data into another measurement or tagset and queries that downsample and aggregate data. The only difference between the two types is if the query has a `GROUP BY time` clause.
Before we get to the continuous query examples, we need to define the `INTO` syntax of queries.
### INTO
`INTO` is a method for running a query and having it output into either another measurement name, retention policy, or database. The syntax looks like this:
```sql
SELECT *
INTO [<retention policy>.]<measurement> [ON <database>]
FROM <measurement>
[WHERE ...]
[GROUP BY ...]
```
The syntax states that the retention policy, database, where clause, and group by clause are all optional. If a retention policy isn't specified, the database's default retention policy will be written into. If the database isn't specified, the database the query is running from will be written into.
By selecting specific fields, `INTO` can merge many series into one that will go into a new either a new measurement, retention policy, or database. For example:
```sql
SELECT mean(value) as value, region
INTO "1h.cpu_load"
FROM cpu_load
GROUP BY time(1h), region
```
That will give 1h summaries of the mean value of the `cpu_load` for each `region`. Specifying `region` in the `GROUP BY` clause is unnecessary since having it in the `SELECT` clause forces it to be grouped by that tag, we've just included it in the example for clarity.
With `SELECT ... INTO`, fields will be written as fields and tags will be written as tags.
### Continuous Query Syntax
The `INTO` queries run once. Continuous queries will turn `INTO` queries into something that run in the background in the cluster. They're kind of like triggers in SQL.
```sql
CREATE CONTINUOUS QUERY "1h_cpu_load"
ON database_name
BEGIN
SELECT mean(value) as value, region
INTO "1h.cpu_load"
FROM cpu_load
GROUP BY time(1h), region
END
```
Or chain them together:
```sql
CREATE CONTINUOUS QUERY "10m_event_count"
ON database_name
BEGIN
SELECT count(value)
INTO "10m.events"
FROM events
GROUP BY time(10m)
END
-- this selects from the output of one continuous query and outputs to another series
CREATE CONTINUOUS QUERY "1h_event_count"
ON database_name
BEGIN
SELECT sum(count) as count
INTO "1h.events"
FROM events
GROUP BY time(1h)
END
```
Or multiple aggregations from all series in a measurement. This example assumes you have a retention policy named `1h`.
```sql
CREATE CONTINUOUS QUERY "1h_cpu_load"
ON database_name
BEGIN
SELECT mean(value), percentile(80, value) as percentile_80, percentile(95, value) as percentile_95
INTO "1h.cpu_load"
FROM cpu_load
GROUP BY time(1h), *
END
```
The `GROUP BY *` indicates that we want to group by the tagset of the points written in. The same tags will be written to the output series. The multiple aggregates in the `SELECT` clause (percentile, mean) will be written in as fields to the resulting series.
Showing what continuous queries we have:
```sql
LIST CONTINUOUS QUERIES
```
Dropping continuous queries:
```sql
DROP CONTINUOUS QUERY <name>
ON <database>
```
### Security
To create or drop a continuous query, the user must be an admin.
### Limitations
In order to prevent cycles and endless copying of data, the following limitation is enforced on continuous queries at create time:
*The output of a continuous query must go to either a different measurement or to a different retention policy.*
In theory they'd still be able to create a cycle with multiple continuous queries. We should check for these and disallow.
## Proposed Architecture
Continuous queries should be stored in the metastore cluster wide. That is, they amount to database schema that should be stored in every server in a cluster.
Continuous queries will have to be handled in a different way for two different use cases: those that simply copy data (CQs without a group by time) and those that aggregate and downsample data (those with a group by time).
### No group by time
For CQs that have no `GROUP BY time` clause, they should be evaluated at the data node as part of the write. The single write should create any other writes for the CQ and submit those in the same request to the brokers to ensure that all writes succeed (both the original and the new CQ writes) or none do.
I imagine the process going something like this:
1. Convert the data point into its compact form `<series id><time><values>`
2. For each CQ on the measurement and retention policy without a group by time:
3. Run the data point through a special query engine that will output 0 or 1 data point
4. Goto #1 for each newly generated data point
5. Write all the data points in a single call to the brokers
6. Return success to the user
Note that for the generated data points, we need to go through and run this process against them since they can feed into different retention policies, measurements, and new tagsets. On #3 I mention that the output will either be a data point or not. That's because of `WHERE` clauses on the query. However, it will never be more than a single data point.
I mention that we'll need a special query engine for these types of queries. In this case, they never have an aggregate function. Any query with an aggregate function also has a group by time, and these queries by definition don't have that.
The only thing we have to worry about is which fields are being selected, and what the where clause looks like. We should be able to put the raw data point through a simple transform function that either outputs another raw points or doesn't.
I think this transform function be something separate from the regular query planner and engine. It can be in `influxQL` but it should be something fairly simply since the only purpose of these types of queries is to either filter some data out and output to a new series or transform into a new series by dropping tags.
### Has group by time
CQs that have a `GROUP BY time` (or aggregate CQs) will need to be handled differently.
One key point on continuous queries with a group by time is that all their writes should always be `overwrite = true`. That is, they should only have a single data point for each timestamp. This distinction means that continuous queries for previous blocks of time can be safely run multiple times without duplicating data (i.e. they're idempotent).
There are two different ideas I have for how CQs with group by time could be handled. The first is through periodic updates handled by the Raft Leader. The second would be to expand out writes for each CQ and handle them on the data node.
#### Periodic Updates
In this approach the management of how CQs run in a cluster will be centrally located on the Raft Leader. It will be responsible for orchestrating which data nodes run CQs and when.
The naive approach would be to have the leader hand out each CQ for a block of time periodically. The leader could also rerun CQ for periods of time that have recently passed. This would be an easy way to handle the "lagging data" problem, but it's not precise.
Unfortunately, there's no easy way to tell cluster wide if there were data points written in an already passed window of time for a CQ. We might be able to add this at the data nodes and have them track it, but it would be quite a bit more work.
The easy way would just be to have CQs re-execute for periods that recently passed and have some user-configurable window of time that they stop checking after. Then we could give the user the ability to recalculate CQs ranges of time if they need to correct for some problem that occurred or the loading of a bunch of historical data.
With this approach, we'd have the metadata in the database store the last time each CQ was run. Whenever the Raft leader sent out a command to a data node to handle a CQ, the data node would use this metadata to determine which windows of time it should compute.
This approach is like what exists in 0.8, with the exception that it will automatically catch data that is lagged behind in a small window of time and give the user the ability to force recalculation.
#### Expanding writes
When a write comes into a data node, we could have it evaluated against group by CQs in addition to the non-group by ones. It would then create writes that would then go through the brokers. When the CQ writes arrive at the data nodes, they would have to handle each write differently depending on if it was a write to a raw series or if it was a CQ write.
Let's lay out a concrete example.
```sql
CREATE CONTINUOUS QUERY "10m_cpu_by_region"
ON foo
BEGIN
SELECT mean(value)
INTO cpu_by_region
FROM cpu
GROUP BY time(10m), region
END
```
In this example we write values into `cpu` with the tags `region` and `host`.
Here's another example CQ:
```sql
CREATE CONTINUOUS QUERY "1h_cpu"
ON foo
BEGIN
SELECT mean(value)
INTO "1h.cpu"
FROM raw.cpu
GROUP BY time(10m), *
END
```
That would output one series into the `1h` retention policy for the `cpu` measurement for every series from the `raw` retention policy and the `cpu` measurement.
Both of these examples would be handled the same way despite one being a big merge of a bunch of series into one and the other being an aggregation of series in a 1-to-1 mapping.
Say we're collecting data for two hosts in a single region. Then we'd have two distinct series like this:
```
1 - cpu host=serverA region=uswest
2 - cpu host=serverB region=uswest
```
Whenever a write came into a server, we'd look at the continuous queries and see if we needed to create new writes. If we had the two CQ examples above, we'd have to expand a single write into two more writes (one for each CQ).
The first CQ would have to create a new series:
```
3 - cpu_by_region region=uswest
```
The second CQ would use the same series id as the write, but would send it to another retention policy (and thus shard).
We'd need to keep track of which series + retention policy combinations were the result of a CQ. When the data nodes get writes replicated downward, they would have to handle them like this:
1. If write is normal, write through
2. If write is CQ write, compute based on existing values, write to DB
#### Approach tradeoffs
The first approach of periodically running queries would almost certainly be the easiest to implement quickly. It also has the added advantage of not putting additional load on the brokers by ballooning up the number of writes that go through the system.
The second approach is appealing because it would be accurate regardless of when writes come in. However, it would take more work and cause the number of writes going through the brokers to be multiplied by the number of continuous queries, which might not scale to where we need.
Also, if the data nodes write for every single update, the load on the underlying storage engine would go up significantly as well.

458
Godeps/_workspace/src/github.com/influxdb/influxdb/services/continuous_querier/service.go generated vendored
View File

@ -1,458 +0,0 @@
package continuous_querier
import (
"errors"
"fmt"
"log"
"os"
"strings"
"sync"
"time"
"github.com/influxdb/influxdb/cluster"
"github.com/influxdb/influxdb/influxql"
"github.com/influxdb/influxdb/meta"
"github.com/influxdb/influxdb/tsdb"
)
const (
// When planning a select statement, passing zero tells it not to chunk results. Only applies to raw queries
NoChunkingSize = 0
)
// ContinuousQuerier represents a service that executes continuous queries.
type ContinuousQuerier interface {
// Run executes the named query in the named database. Blank database or name matches all.
Run(database, name string) error
}
// queryExecutor is an internal interface to make testing easier.
type queryExecutor interface {
ExecuteQuery(query *influxql.Query, database string, chunkSize int) (<-chan *influxql.Result, error)
}
// metaStore is an internal interface to make testing easier.
type metaStore interface {
IsLeader() bool
Databases() ([]meta.DatabaseInfo, error)
Database(name string) (*meta.DatabaseInfo, error)
}
// pointsWriter is an internal interface to make testing easier.
type pointsWriter interface {
WritePoints(p *cluster.WritePointsRequest) error
}
// Service manages continuous query execution.
type Service struct {
MetaStore metaStore
QueryExecutor queryExecutor
PointsWriter pointsWriter
Config *Config
RunInterval time.Duration
// RunCh can be used by clients to signal service to run CQs.
RunCh chan struct{}
Logger *log.Logger
loggingEnabled bool
// lastRuns maps CQ name to last time it was run.
lastRuns map[string]time.Time
stop chan struct{}
wg *sync.WaitGroup
}
// NewService returns a new instance of Service.
func NewService(c Config) *Service {
s := &Service{
Config: &c,
RunInterval: time.Second,
RunCh: make(chan struct{}),
loggingEnabled: c.LogEnabled,
Logger: log.New(os.Stderr, "[continuous_querier] ", log.LstdFlags),
lastRuns: map[string]time.Time{},
}
return s
}
// Open starts the service.
func (s *Service) Open() error {
s.Logger.Println("Starting continuous query service")
if s.stop != nil {
return nil
}
assert(s.MetaStore != nil, "MetaStore is nil")
assert(s.QueryExecutor != nil, "QueryExecutor is nil")
assert(s.PointsWriter != nil, "PointsWriter is nil")
s.stop = make(chan struct{})
s.wg = &sync.WaitGroup{}
s.wg.Add(1)
go s.backgroundLoop()
return nil
}
// Close stops the service.
func (s *Service) Close() error {
if s.stop == nil {
return nil
}
close(s.stop)
s.wg.Wait()
s.wg = nil
s.stop = nil
return nil
}
// SetLogger sets the internal logger to the logger passed in.
func (s *Service) SetLogger(l *log.Logger) {
s.Logger = l
}
// Run runs the specified continuous query, or all CQs if none is specified.
func (s *Service) Run(database, name string) error {
var dbs []meta.DatabaseInfo
if database != "" {
// Find the requested database.
db, err := s.MetaStore.Database(database)
if err != nil {
return err
} else if db == nil {
return tsdb.ErrDatabaseNotFound(database)
}
dbs = append(dbs, *db)
} else {
// Get all databases.
var err error
dbs, err = s.MetaStore.Databases()
if err != nil {
return err
}
}
// Loop through databases.
for _, db := range dbs {
// Loop through CQs in each DB executing the ones that match name.
for _, cq := range db.ContinuousQueries {
if name == "" || cq.Name == name {
// Reset the last run time for the CQ.
s.lastRuns[cq.Name] = time.Time{}
}
}
}
// Signal the background routine to run CQs.
s.RunCh <- struct{}{}
return nil
}
// backgroundLoop runs on a go routine and periodically executes CQs.
func (s *Service) backgroundLoop() {
defer s.wg.Done()
for {
select {
case <-s.stop:
s.Logger.Println("continuous query service terminating")
return
case <-s.RunCh:
if s.MetaStore.IsLeader() {
s.Logger.Print("running continuous queries by request")
s.runContinuousQueries()
}
case <-time.After(s.RunInterval):
if s.MetaStore.IsLeader() {
s.runContinuousQueries()
}
}
}
}
// runContinuousQueries gets CQs from the meta store and runs them.
func (s *Service) runContinuousQueries() {
// Get list of all databases.
dbs, err := s.MetaStore.Databases()
if err != nil {
s.Logger.Println("error getting databases")
return
}
// Loop through all databases executing CQs.
for _, db := range dbs {
// TODO: distribute across nodes
for _, cq := range db.ContinuousQueries {
if err := s.ExecuteContinuousQuery(&db, &cq); err != nil {
s.Logger.Printf("error executing query: %s: err = %s", cq.Query, err)
}
}
}
}
// ExecuteContinuousQuery executes a single CQ.
func (s *Service) ExecuteContinuousQuery(dbi *meta.DatabaseInfo, cqi *meta.ContinuousQueryInfo) error {
// TODO: re-enable stats
//s.stats.Inc("continuousQueryExecuted")
// Local wrapper / helper.
cq, err := NewContinuousQuery(dbi.Name, cqi)
if err != nil {
return err
}
// Get the last time this CQ was run from the service's cache.
cq.LastRun = s.lastRuns[cqi.Name]
// Set the retention policy to default if it wasn't specified in the query.
if cq.intoRP() == "" {
cq.setIntoRP(dbi.DefaultRetentionPolicy)
}
// See if this query needs to be run.
computeNoMoreThan := time.Duration(s.Config.ComputeNoMoreThan)
run, err := cq.shouldRunContinuousQuery(s.Config.ComputeRunsPerInterval, computeNoMoreThan)
if err != nil {
return err
} else if !run {
return nil
}
// We're about to run the query so store the time.
now := time.Now()
cq.LastRun = now
s.lastRuns[cqi.Name] = now
// Get the group by interval.
interval, err := cq.q.GroupByInterval()
if err != nil {
return err
} else if interval == 0 {
return nil
}
// Calculate and set the time range for the query.
startTime := now.Round(interval)
if startTime.UnixNano() > now.UnixNano() {
startTime = startTime.Add(-interval)
}
if err := cq.q.SetTimeRange(startTime, startTime.Add(interval)); err != nil {
s.Logger.Printf("error setting time range: %s\n", err)
}
if s.loggingEnabled {
s.Logger.Printf("executing continuous query %s", cq.Info.Name)
}
// Do the actual processing of the query & writing of results.
if err := s.runContinuousQueryAndWriteResult(cq); err != nil {
s.Logger.Printf("error: %s. running: %s\n", err, cq.q.String())
return err
}
recomputeNoOlderThan := time.Duration(s.Config.RecomputeNoOlderThan)
for i := 0; i < s.Config.RecomputePreviousN; i++ {
// if we're already more time past the previous window than we're going to look back, stop
if now.Sub(startTime) > recomputeNoOlderThan {
return nil
}
newStartTime := startTime.Add(-interval)
if err := cq.q.SetTimeRange(newStartTime, startTime); err != nil {
s.Logger.Printf("error setting time range: %s\n", err)
return err
}
if err := s.runContinuousQueryAndWriteResult(cq); err != nil {
s.Logger.Printf("error during recompute previous: %s. running: %s\n", err, cq.q.String())
return err
}
startTime = newStartTime
}
return nil
}
// runContinuousQueryAndWriteResult will run the query against the cluster and write the results back in
func (s *Service) runContinuousQueryAndWriteResult(cq *ContinuousQuery) error {
// Wrap the CQ's inner SELECT statement in a Query for the QueryExecutor.
q := &influxql.Query{
Statements: influxql.Statements{cq.q},
}
// Execute the SELECT.
ch, err := s.QueryExecutor.ExecuteQuery(q, cq.Database, NoChunkingSize)
if err != nil {
return err
}
// Drain results
defer func() {
for _ = range ch {
}
}()
// Read all rows from the result channel.
points := make([]tsdb.Point, 0, 100)
for result := range ch {
if result.Err != nil {
return result.Err
}
for _, row := range result.Series {
// Convert the result row to points.
part, err := s.convertRowToPoints(cq.intoMeasurement(), row)
if err != nil {
log.Println(err)
continue
}
if len(part) == 0 {
continue
}
// If the points have any nil values, can't write.
// This happens if the CQ is created and running before data is written to the measurement.
for _, p := range part {
fields := p.Fields()
for _, v := range fields {
if v == nil {
return nil
}
}
}
points = append(points, part...)
}
}
if len(points) == 0 {
return nil
}
// Create a write request for the points.
req := &cluster.WritePointsRequest{
Database: cq.intoDB(),
RetentionPolicy: cq.intoRP(),
ConsistencyLevel: cluster.ConsistencyLevelAny,
Points: points,
}
// Write the request.
if err := s.PointsWriter.WritePoints(req); err != nil {
s.Logger.Println(err)
return err
}
if s.loggingEnabled {
s.Logger.Printf("wrote %d point(s) to %s.%s.%s", len(points), cq.intoDB(), cq.intoRP(), cq.Info.Name)
}
return nil
}
// convertRowToPoints will convert a query result Row into Points that can be written back in.
// Used for continuous and INTO queries
func (s *Service) convertRowToPoints(measurementName string, row *influxql.Row) ([]tsdb.Point, error) {
// figure out which parts of the result are the time and which are the fields
timeIndex := -1
fieldIndexes := make(map[string]int)
for i, c := range row.Columns {
if c == "time" {
timeIndex = i
} else {
fieldIndexes[c] = i
}
}
if timeIndex == -1 {
return nil, errors.New("error finding time index in result")
}
points := make([]tsdb.Point, 0, len(row.Values))
for _, v := range row.Values {
vals := make(map[string]interface{})
for fieldName, fieldIndex := range fieldIndexes {
vals[fieldName] = v[fieldIndex]
}
p := tsdb.NewPoint(measurementName, row.Tags, vals, v[timeIndex].(time.Time))
points = append(points, p)
}
return points, nil
}
// ContinuousQuery is a local wrapper / helper around continuous queries.
type ContinuousQuery struct {
Database string
Info *meta.ContinuousQueryInfo
LastRun time.Time
q *influxql.SelectStatement
}
func (cq *ContinuousQuery) intoDB() string { return cq.q.Target.Measurement.Database }
func (cq *ContinuousQuery) intoRP() string { return cq.q.Target.Measurement.RetentionPolicy }
func (cq *ContinuousQuery) setIntoRP(rp string) { cq.q.Target.Measurement.RetentionPolicy = rp }
func (cq *ContinuousQuery) intoMeasurement() string { return cq.q.Target.Measurement.Name }
// NewContinuousQuery returns a ContinuousQuery object with a parsed influxql.CreateContinuousQueryStatement
func NewContinuousQuery(database string, cqi *meta.ContinuousQueryInfo) (*ContinuousQuery, error) {
stmt, err := influxql.NewParser(strings.NewReader(cqi.Query)).ParseStatement()
if err != nil {
return nil, err
}
q, ok := stmt.(*influxql.CreateContinuousQueryStatement)
if !ok || q.Source.Target == nil || q.Source.Target.Measurement == nil {
return nil, errors.New("query isn't a valid continuous query")
}
cquery := &ContinuousQuery{
Database: database,
Info: cqi,
q: q.Source,
}
return cquery, nil
}
// shouldRunContinuousQuery returns true if the CQ should be schedule to run. It will use the
// lastRunTime of the CQ and the rules for when to run set through the config to determine
// if this CQ should be run
func (cq *ContinuousQuery) shouldRunContinuousQuery(runsPerInterval int, noMoreThan time.Duration) (bool, error) {
// if it's not aggregated we don't run it
if cq.q.IsRawQuery {
return false, errors.New("continuous queries must be aggregate queries")
}
// since it's aggregated we need to figure how often it should be run
interval, err := cq.q.GroupByInterval()
if err != nil {
return false, err
}
// determine how often we should run this continuous query.
// group by time / the number of times to compute
computeEvery := time.Duration(interval.Nanoseconds()/int64(runsPerInterval)) * time.Nanosecond
// make sure we're running no more frequently than the setting in the config
if computeEvery < noMoreThan {
computeEvery = noMoreThan
}
// if we've passed the amount of time since the last run, do it up
if cq.LastRun.Add(computeEvery).UnixNano() <= time.Now().UnixNano() {
return true, nil
}
return false, nil
}
// assert will panic with a given formatted message if the given condition is false.
func assert(condition bool, msg string, v ...interface{}) {
if !condition {
panic(fmt.Sprintf("assert failed: "+msg, v...))
}
}

503
Godeps/_workspace/src/github.com/influxdb/influxdb/services/continuous_querier/service_test.go generated vendored
View File

@ -1,503 +0,0 @@
package continuous_querier
import (
"errors"
"fmt"
"io/ioutil"
"log"
"sync"
"testing"
"time"
"github.com/influxdb/influxdb/cluster"
"github.com/influxdb/influxdb/influxql"
"github.com/influxdb/influxdb/meta"
)
var (
expectedErr = errors.New("expected error")
unexpectedErr = errors.New("unexpected error")
)
// Test closing never opened, open, open already open, close, and close already closed.
func TestOpenAndClose(t *testing.T) {
s := NewTestService(t)
if err := s.Close(); err != nil {
t.Error(err)
} else if err = s.Open(); err != nil {
t.Error(err)
} else if err = s.Open(); err != nil {
t.Error(err)
} else if err = s.Close(); err != nil {
t.Error(err)
} else if err = s.Close(); err != nil {
t.Error(err)
}
}
// Test ExecuteContinuousQuery happy path.
func TestExecuteContinuousQuery_HappyPath(t *testing.T) {
s := NewTestService(t)
dbis, _ := s.MetaStore.Databases()
dbi := dbis[0]
cqi := dbi.ContinuousQueries[0]
pointCnt := 100
qe := s.QueryExecutor.(*QueryExecutor)
qe.Results = []*influxql.Result{genResult(1, pointCnt)}
pw := s.PointsWriter.(*PointsWriter)
pw.WritePointsFn = func(p *cluster.WritePointsRequest) error {
if len(p.Points) != pointCnt {
return fmt.Errorf("exp = %d, got = %d", pointCnt, len(p.Points))
}
return nil
}
err := s.ExecuteContinuousQuery(&dbi, &cqi)
if err != nil {
t.Error(err)
}
}
// Test the service happy path.
func TestService_HappyPath(t *testing.T) {
s := NewTestService(t)
pointCnt := 100
qe := s.QueryExecutor.(*QueryExecutor)
qe.Results = []*influxql.Result{genResult(1, pointCnt)}
pw := s.PointsWriter.(*PointsWriter)
ch := make(chan int, 5)
defer close(ch)
pw.WritePointsFn = func(p *cluster.WritePointsRequest) error {
ch <- len(p.Points)
return nil
}
s.Open()
if cnt, err := waitInt(ch, time.Second); err != nil {
t.Error(err)
} else if cnt != pointCnt {
t.Errorf("exp = %d, got = %d", pointCnt, cnt)
}
s.Close()
}
// Test Run method.
func TestService_Run(t *testing.T) {
s := NewTestService(t)
// Set RunInterval high so we can trigger using Run method.
s.RunInterval = 10 * time.Minute
// Only want one call to ExecuteQueryFn per CQ.
s.Config.RecomputePreviousN = 0
done := make(chan struct{})
expectCallCnt := 2
callCnt := 0
// Set a callback for ExecuteQuery.
qe := s.QueryExecutor.(*QueryExecutor)
qe.ExecuteQueryFn = func(query *influxql.Query, database string, chunkSize int) (<-chan *influxql.Result, error) {
callCnt++
if callCnt >= expectCallCnt {
done <- struct{}{}
}
return nil, nil
}
s.Open()
// Trigger service to run all CQs.
s.Run("", "")
// Shouldn't time out.
if err := wait(done, 100*time.Millisecond); err != nil {
t.Error(err)
}
// This time it should timeout because ExecuteQuery should not get called again.
if err := wait(done, 100*time.Millisecond); err == nil {
t.Error("too many queries executed")
}
s.Close()
// Now test just one query.
expectCallCnt = 1
callCnt = 0
s.Open()
s.Run("db", "cq")
// Shouldn't time out.
if err := wait(done, 100*time.Millisecond); err != nil {
t.Error(err)
}
// This time it should timeout because ExecuteQuery should not get called again.
if err := wait(done, 100*time.Millisecond); err == nil {
t.Error("too many queries executed")
}
s.Close()
}
// Test service when not the cluster leader (CQs shouldn't run).
func TestService_NotLeader(t *testing.T) {
s := NewTestService(t)
// Set RunInterval high so we can test triggering with the RunCh below.
s.RunInterval = 10 * time.Second
s.MetaStore.(*MetaStore).Leader = false
done := make(chan struct{})
qe := s.QueryExecutor.(*QueryExecutor)
// Set a callback for ExecuteQuery. Shouldn't get called because we're not the leader.
qe.ExecuteQueryFn = func(query *influxql.Query, database string, chunkSize int) (<-chan *influxql.Result, error) {
done <- struct{}{}
return nil, unexpectedErr
}
s.Open()
// Trigger service to run CQs.
s.RunCh <- struct{}{}
// Expect timeout error because ExecuteQuery callback wasn't called.
if err := wait(done, 100*time.Millisecond); err == nil {
t.Error(err)
}
s.Close()
}
// Test service behavior when meta store fails to get databases.
func TestService_MetaStoreFailsToGetDatabases(t *testing.T) {
s := NewTestService(t)
// Set RunInterval high so we can test triggering with the RunCh below.
s.RunInterval = 10 * time.Second
s.MetaStore.(*MetaStore).Err = expectedErr
done := make(chan struct{})
qe := s.QueryExecutor.(*QueryExecutor)
// Set ExecuteQuery callback, which shouldn't get called because of meta store failure.
qe.ExecuteQueryFn = func(query *influxql.Query, database string, chunkSize int) (<-chan *influxql.Result, error) {
done <- struct{}{}
return nil, unexpectedErr
}
s.Open()
// Trigger service to run CQs.
s.RunCh <- struct{}{}
// Expect timeout error because ExecuteQuery callback wasn't called.
if err := wait(done, 100*time.Millisecond); err == nil {
t.Error(err)
}
s.Close()
}
// Test ExecuteContinuousQuery with invalid queries.
func TestExecuteContinuousQuery_InvalidQueries(t *testing.T) {
s := NewTestService(t)
dbis, _ := s.MetaStore.Databases()
dbi := dbis[0]
cqi := dbi.ContinuousQueries[0]
cqi.Query = `this is not a query`
err := s.ExecuteContinuousQuery(&dbi, &cqi)
if err == nil {
t.Error("expected error but got nil")
}
// Valid query but invalid continuous query.
cqi.Query = `SELECT * FROM cpu`
err = s.ExecuteContinuousQuery(&dbi, &cqi)
if err == nil {
t.Error("expected error but got nil")
}
// Group by requires aggregate.
cqi.Query = `SELECT value INTO other_value FROM cpu WHERE time > now() - 1h GROUP BY time(1s)`
err = s.ExecuteContinuousQuery(&dbi, &cqi)
if err == nil {
t.Error("expected error but got nil")
}
}
// Test ExecuteContinuousQuery when QueryExecutor returns an error.
func TestExecuteContinuousQuery_QueryExecutor_Error(t *testing.T) {
s := NewTestService(t)
qe := s.QueryExecutor.(*QueryExecutor)
qe.Err = expectedErr
dbis, _ := s.MetaStore.Databases()
dbi := dbis[0]
cqi := dbi.ContinuousQueries[0]
err := s.ExecuteContinuousQuery(&dbi, &cqi)
if err != expectedErr {
t.Errorf("exp = %s, got = %v", expectedErr, err)
}
}
// NewTestService returns a new *Service with default mock object members.
func NewTestService(t *testing.T) *Service {
s := NewService(NewConfig())
ms := NewMetaStore(t)
s.MetaStore = ms
s.QueryExecutor = NewQueryExecutor(t)
s.PointsWriter = NewPointsWriter(t)
s.RunInterval = time.Millisecond
// Set Logger to write to dev/null so stdout isn't polluted.
if !testing.Verbose() {
s.Logger = log.New(ioutil.Discard, "", log.LstdFlags)
}
// Add a couple test databases and CQs.
ms.CreateDatabase("db", "rp")
ms.CreateContinuousQuery("db", "cq", `CREATE CONTINUOUS QUERY cq ON db BEGIN SELECT count(cpu) INTO cpu_count FROM cpu WHERE time > now() - 1h GROUP BY time(1s) END`)
ms.CreateDatabase("db2", "default")
ms.CreateContinuousQuery("db2", "cq2", `CREATE CONTINUOUS QUERY cq2 ON db2 BEGIN SELECT mean(value) INTO cpu_mean FROM cpu WHERE time > now() - 10m GROUP BY time(1m) END`)
return s
}
// MetaStore is a mock meta store.
type MetaStore struct {
mu sync.RWMutex
Leader bool
DatabaseInfos []meta.DatabaseInfo
Err error
t *testing.T
}
// NewMetaStore returns a *MetaStore.
func NewMetaStore(t *testing.T) *MetaStore {
return &MetaStore{
Leader: true,
t: t,
}
}
// IsLeader returns true if the node is the cluster leader.
func (ms *MetaStore) IsLeader() bool {
ms.mu.RLock()
defer ms.mu.RUnlock()
return ms.Leader
}
// Databases returns a list of database info about each database in the cluster.
func (ms *MetaStore) Databases() ([]meta.DatabaseInfo, error) {
ms.mu.RLock()
defer ms.mu.RUnlock()
return ms.DatabaseInfos, ms.Err
}
// Database returns a single database by name.
func (ms *MetaStore) Database(name string) (*meta.DatabaseInfo, error) {
ms.mu.RLock()
defer ms.mu.RUnlock()
return ms.database(name)
}
func (ms *MetaStore) database(name string) (*meta.DatabaseInfo, error) {
if ms.Err != nil {
return nil, ms.Err
}
for i := range ms.DatabaseInfos {
if ms.DatabaseInfos[i].Name == name {
return &ms.DatabaseInfos[i], nil
}
}
return nil, fmt.Errorf("database not found: %s", name)
}
// CreateDatabase adds a new database to the meta store.
func (ms *MetaStore) CreateDatabase(name, defaultRetentionPolicy string) error {
ms.mu.Lock()
defer ms.mu.Unlock()
if ms.Err != nil {
return ms.Err
}
// See if the database already exists.
for _, dbi := range ms.DatabaseInfos {
if dbi.Name == name {
return fmt.Errorf("database already exists: %s", name)
}
}
// Create database.
ms.DatabaseInfos = append(ms.DatabaseInfos, meta.DatabaseInfo{
Name: name,
DefaultRetentionPolicy: defaultRetentionPolicy,
})
return nil
}
// CreateContinuousQuery adds a CQ to the meta store.
func (ms *MetaStore) CreateContinuousQuery(database, name, query string) error {
ms.mu.Lock()
defer ms.mu.Unlock()
if ms.Err != nil {
return ms.Err
}
dbi, err := ms.database(database)
if err != nil {
return err
} else if dbi == nil {
return fmt.Errorf("database not found: %s", database)
}
// See if CQ already exists.
for _, cqi := range dbi.ContinuousQueries {
if cqi.Name == name {
return fmt.Errorf("continuous query already exists: %s", name)
}
}
// Create a new CQ and store it.
dbi.ContinuousQueries = append(dbi.ContinuousQueries, meta.ContinuousQueryInfo{
Name: name,
Query: query,
})
return nil
}
// QueryExecutor is a mock query executor.
type QueryExecutor struct {
ExecuteQueryFn func(query *influxql.Query, database string, chunkSize int) (<-chan *influxql.Result, error)
Results []*influxql.Result
ResultInterval time.Duration
Err error
ErrAfterResult int
StopRespondingAfter int
t *testing.T
}
// NewQueryExecutor returns a *QueryExecutor.
func NewQueryExecutor(t *testing.T) *QueryExecutor {
return &QueryExecutor{
ErrAfterResult: -1,
StopRespondingAfter: -1,
t: t,
}
}
// ExecuteQuery returns a channel that the caller can read query results from.
func (qe *QueryExecutor) ExecuteQuery(query *influxql.Query, database string, chunkSize int) (<-chan *influxql.Result, error) {
// If the test set a callback, call it.
if qe.ExecuteQueryFn != nil {
if _, err := qe.ExecuteQueryFn(query, database, chunkSize); err != nil {
return nil, err
}
}
// Are we supposed to error immediately?
if qe.ErrAfterResult == -1 && qe.Err != nil {
return nil, qe.Err
}
ch := make(chan *influxql.Result)
// Start a go routine to send results and / or error.
go func() {
n := 0
for i, r := range qe.Results {
if i == qe.ErrAfterResult-1 {
qe.t.Logf("ExecuteQuery(): ErrAfterResult %d", qe.ErrAfterResult-1)
ch <- &influxql.Result{Err: qe.Err}
close(ch)
return
} else if i == qe.StopRespondingAfter {
qe.t.Log("ExecuteQuery(): StopRespondingAfter")
return
}
ch <- r
n++
time.Sleep(qe.ResultInterval)
}
qe.t.Logf("ExecuteQuery(): all (%d) results sent", n)
close(ch)
}()
return ch, nil
}
// PointsWriter is a mock points writer.
type PointsWriter struct {
WritePointsFn func(p *cluster.WritePointsRequest) error
Err error
PointsPerSecond int
t *testing.T
}
// NewPointsWriter returns a new *PointsWriter.
func NewPointsWriter(t *testing.T) *PointsWriter {
return &PointsWriter{
PointsPerSecond: 25000,
t: t,
}
}
// WritePoints mocks writing points.
func (pw *PointsWriter) WritePoints(p *cluster.WritePointsRequest) error {
// If the test set a callback, call it.
if pw.WritePointsFn != nil {
if err := pw.WritePointsFn(p); err != nil {
return err
}
}
if pw.Err != nil {
return pw.Err
}
ns := time.Duration((1 / pw.PointsPerSecond) * 1000000000)
time.Sleep(ns)
return nil
}
// genResult generates a dummy query result.
func genResult(rowCnt, valCnt int) *influxql.Result {
rows := make(influxql.Rows, 0, rowCnt)
now := time.Now()
for n := 0; n < rowCnt; n++ {
vals := make([][]interface{}, 0, valCnt)
for m := 0; m < valCnt; m++ {
vals = append(vals, []interface{}{now, float64(m)})
now.Add(time.Second)
}
row := &influxql.Row{
Name: "cpu",
Tags: map[string]string{"host": "server01"},
Columns: []string{"time", "value"},
Values: vals,
}
rows = append(rows, row)
}
return &influxql.Result{
Series: rows,
}
}
func wait(c chan struct{}, d time.Duration) (err error) {
select {
case <-c:
case <-time.After(d):
err = errors.New("timed out")
}
return
}
func waitInt(c chan int, d time.Duration) (i int, err error) {
select {
case i = <-c:
case <-time.After(d):
err = errors.New("timed out")
}
return
}
func check(err error) {
if err != nil {
panic(err)
}
}

125
Godeps/_workspace/src/github.com/influxdb/influxdb/services/graphite/README.md generated vendored
View File

@ -1,125 +0,0 @@
## Introduction
The graphite plugin allows measurements to be saved using the graphite line protocol. By default, enabling the graphite plugin will allow you to collect metrics and store them using the metric name as the measurement. If you send a metric named `servers.localhost.cpu.loadavg.10`, it will store the full metric name as the measurement with no extracted tags.
While this default setup works, it is not the ideal way to store measurements in InfluxDB since it does not take advantage of tags. It also will not perform optimally with a large dataset sizes since queries will be forced to use regexes which is known to not scale well.
To extract tags from metrics, one or more templates must be configured to parse metrics into tags and measurements.
## Templates
Templates allow matching parts of a metric name to be used as tag names in the stored metric. They have a similar format to graphite metric names. The values in between the separators are used as the tag name. The location of the tag name that matches the same position as the graphite metric section is used as the value. If there is no value, the graphite portion is skipped.
The special value _measurement_ is used to define the measurement name. It can have a trailing `*` to indicate that the remainder of the metric should be used. If a _measurement_ is not specified, the full metric name is used.
### Basic Matching
`servers.localhost.cpu.loadavg.10`
* Template: `.host.resource.measurement*`
* Output: _measurement_ =`loadavg.10` _tags_ =`host=localhost resource=cpu`
### Multiple Measurement Matching
The _measurement_ can be specified multiple times in a template to provide more control over the measurement name. Multiple values
will be joined together using the _Separator_ config variable. By default, this value is `.`.
`servers.localhost.cpu.cpu0.user`
* Template: `.host.measurement.cpu.measurement`
* Output: _measurement_ = `cpu.user` _tags_ = `host=localhost cpu=cpu0`
Since '.' requires queries on measurements to be double-quoted, you may want to set this to `_` to simplify querying parsed metrics.
`servers.localhost.cpu.cpu0.user`
* Separator: `_`
* Template: `.host.measurement.cpu.measurement`
* Output: _measurement_ = `cpu_user` _tags_ = `host=localhost cpu=cpu0`
### Adding Tags
Additional tags can be added to a metric that don't exist on the received metric. You can add additional tags by specifying them after the pattern. Tags have the same format as the line protocol. Multiple tags are separated by commas.
`servers.localhost.cpu.loadavg.10`
* Template: `.host.resource.measurement* region=us-west,zone=1a`
* Output: _measurement_ = `loading.10` _tags_ = `host=localhost resource=cpu region=us-west zone=1a`
## Multiple Templates
One template may not match all metrics. For example, using multiple plugins with diamond will produce metrics in different formats. If you need to use multiple templates, you'll need to define a prefix filter that must match before the template can be applied.
### Filters
Filters have a similar format to templates but work more like wildcard expressions. When multiple filters would match a metric, the more specific one is chosen. Filters are configured by adding them before the template.
For example,
```
servers.localhost.cpu.loadavg.10
servers.host123.elasticsearch.cache_hits 100
servers.host456.mysql.tx_count 10
```
* `servers.*` would match all values
* `servers.*.mysql` would match `servers.host456.mysql.tx_count 10`
* `servers.localhost.*` would match `servers.localhost.cpu.loadavg`
## Default Templates
If no template filters are defined or you want to just have one basic template, you can define a default template. This template will apply to any metric that has not already matched a filter.
```
dev.http.requests.200
prod.myapp.errors.count
dev.db.queries.count
```
* `env.app.measurement*` would create
* _measurement_=`requests.200` _tags_=`env=dev,app=http`
* _measurement_= `errors.count` _tags_=`env=prod,app=myapp`
* _measurement_=`queries.count` _tags_=`env=dev,app=db`
## Global Tags
If you need to add the same set of tags to all metrics, you can define them globally at the plugin level and not within each template description.
## Minimal Config
```
[[graphite]]
enabled = true
# bind-address = ":2003"
# protocol = "tcp"
# consistency-level = "one"
### If matching multiple measurement files, this string will be used to join the matched values.
# separator = "."
### Default tags that will be added to all metrics. These can be overridden at the template level
### or by tags extracted from metric
# tags = ["region=us-east", "zone=1c"]
### Each template line requires a template pattern. It can have an optional
### filter before the template and separated by spaces. It can also have optional extra
### tags following the template. Multiple tags should be separated by commas and no spaces
### similar to the line protocol format. The can be only one default template.
# templates = [
# "*.app env.service.resource.measurement",
# # Default template
# "server.*",
#]
```
## Customized Config
```
[[graphite]]
enabled = true
separator = "_"
tags = ["region=us-east", "zone=1c"]
templates = [
# filter + template
"*.app env.service.resource.measurement",
# filter + template + extra tag
"stats.* .host.measurement* region=us-west,agent=sensu",
# default template. Ignore the first graphite component "servers"
".measurement*",
]
```

221
Godeps/_workspace/src/github.com/influxdb/influxdb/services/graphite/config.go generated vendored
View File

@ -1,221 +0,0 @@
package graphite
import (
"fmt"
"strings"
"time"
"github.com/influxdb/influxdb/toml"
"github.com/influxdb/influxdb/tsdb"
)
const (
// DefaultBindAddress is the default binding interface if none is specified.
DefaultBindAddress = ":2003"
// DefaultDatabase is the default database if none is specified.
DefaultDatabase = "graphite"
// DefaultProtocol is the default IP protocol used by the Graphite input.
DefaultProtocol = "tcp"
// DefaultConsistencyLevel is the default write consistency for the Graphite input.
DefaultConsistencyLevel = "one"
// DefaultSeparator is the default join character to use when joining multiple
// measurment parts in a template.
DefaultSeparator = "."
// DefaultBatchSize is the default Graphite batch size.
DefaultBatchSize = 1000
// DefaultBatchTimeout is the default Graphite batch timeout.
DefaultBatchTimeout = time.Second
)
// Config represents the configuration for Graphite endpoints.
type Config struct {
BindAddress string `toml:"bind-address"`
Database string `toml:"database"`
Enabled bool `toml:"enabled"`
Protocol string `toml:"protocol"`
BatchSize int `toml:"batch-size"`
BatchTimeout toml.Duration `toml:"batch-timeout"`
ConsistencyLevel string `toml:"consistency-level"`
Templates []string `toml:"templates"`
Tags []string `toml:"tags"`
Separator string `toml:"separator"`
}
// NewConfig returns a new Config with defaults.
func NewConfig() Config {
return Config{
BindAddress: DefaultBindAddress,
Database: DefaultDatabase,
Protocol: DefaultProtocol,
BatchSize: DefaultBatchSize,
BatchTimeout: toml.Duration(DefaultBatchTimeout),
ConsistencyLevel: DefaultConsistencyLevel,
Separator: DefaultSeparator,
}
}
// WithDefaults takes the given config and returns a new config with any required
// default values set.
func (c *Config) WithDefaults() *Config {
d := *c
if d.BindAddress == "" {
d.BindAddress = DefaultBindAddress
}
if d.Database == "" {
d.Database = DefaultDatabase
}
if d.Protocol == "" {
d.Protocol = DefaultProtocol
}
if d.ConsistencyLevel == "" {
d.ConsistencyLevel = DefaultConsistencyLevel
}
if d.Separator == "" {
d.Separator = DefaultSeparator
}
return &d
}
func (c *Config) DefaultTags() tsdb.Tags {
tags := tsdb.Tags{}
for _, t := range c.Tags {
parts := strings.Split(t, "=")
tags[parts[0]] = parts[1]
}
return tags
}
func (c *Config) Validate() error {
if err := c.validateTemplates(); err != nil {
return err
}
if err := c.validateTags(); err != nil {
return err
}
return nil
}
func (c *Config) validateTemplates() error {
// map to keep track of filters we see
filters := map[string]struct{}{}
for i, t := range c.Templates {
parts := strings.Fields(t)
// Ensure template string is non-empty
if len(parts) == 0 {
return fmt.Errorf("missing template at position: %d", i)
}
if len(parts) == 1 && parts[0] == "" {
return fmt.Errorf("missing template at position: %d", i)
}
if len(parts) > 3 {
return fmt.Errorf("invalid template format: '%s'", t)
}
template := t
filter := ""
tags := ""
if len(parts) >= 2 {
// We could have <filter> <template> or <template> <tags>. Equals is only allowed in
// tags section.
if strings.Contains(parts[1], "=") {
template = parts[0]
tags = parts[1]
} else {
filter = parts[0]
template = parts[1]
}
}
if len(parts) == 3 {
tags = parts[2]
}
// Validate the template has one and only one measurement
if err := c.validateTemplate(template); err != nil {
return err
}
// Prevent duplicate filters in the config
if _, ok := filters[filter]; ok {
return fmt.Errorf("duplicate filter '%s' found at position: %d", filter, i)
}
filters[filter] = struct{}{}
if filter != "" {
// Validate filter expression is valid
if err := c.validateFilter(filter); err != nil {
return err
}
}
if tags != "" {
// Validate tags
for _, tagStr := range strings.Split(tags, ",") {
if err := c.validateTag(tagStr); err != nil {
return err
}
}
}
}
return nil
}
func (c *Config) validateTags() error {
for _, t := range c.Tags {
if err := c.validateTag(t); err != nil {
return err
}
}
return nil
}
func (c *Config) validateTemplate(template string) error {
hasMeasurement := false
for _, p := range strings.Split(template, ".") {
if p == "measurement" || p == "measurement*" {
hasMeasurement = true
}
}
if !hasMeasurement {
return fmt.Errorf("no measurement in template `%s`", template)
}
return nil
}
func (c *Config) validateFilter(filter string) error {
for _, p := range strings.Split(filter, ".") {
if p == "" {
return fmt.Errorf("filter contains blank section: %s", filter)
}
if strings.Contains(p, "*") && p != "*" {
return fmt.Errorf("invalid filter wildcard section: %s", filter)
}
}
return nil
}
func (c *Config) validateTag(keyValue string) error {
parts := strings.Split(keyValue, "=")
if len(parts) != 2 {
return fmt.Errorf("invalid template tags: '%s'", keyValue)
}
if parts[0] == "" || parts[1] == "" {
return fmt.Errorf("invalid template tags: %s'", keyValue)
}
return nil
}

164
Godeps/_workspace/src/github.com/influxdb/influxdb/services/graphite/config_test.go generated vendored
View File

@ -1,164 +0,0 @@
package graphite_test
import (
"testing"
"time"
"github.com/BurntSushi/toml"
"github.com/influxdb/influxdb/services/graphite"
)
func TestConfig_Parse(t *testing.T) {
// Parse configuration.
var c graphite.Config
if _, err := toml.Decode(`
bind-address = ":8080"
database = "mydb"
enabled = true
protocol = "tcp"
batch-size=100
batch-timeout="1s"
consistency-level="one"
templates=["servers.* .host.measurement*"]
tags=["region=us-east"]
`, &c); err != nil {
t.Fatal(err)
}
// Validate configuration.
if c.BindAddress != ":8080" {
t.Fatalf("unexpected bind address: %s", c.BindAddress)
} else if c.Database != "mydb" {
t.Fatalf("unexpected database selected: %s", c.Database)
} else if c.Enabled != true {
t.Fatalf("unexpected graphite enabled: %v", c.Enabled)
} else if c.Protocol != "tcp" {
t.Fatalf("unexpected graphite protocol: %s", c.Protocol)
} else if c.BatchSize != 100 {
t.Fatalf("unexpected graphite batch size: %d", c.BatchSize)
} else if time.Duration(c.BatchTimeout) != time.Second {
t.Fatalf("unexpected graphite batch timeout: %v", c.BatchTimeout)
} else if c.ConsistencyLevel != "one" {
t.Fatalf("unexpected graphite consistency setting: %s", c.ConsistencyLevel)
}
if len(c.Templates) != 1 && c.Templates[0] != "servers.* .host.measurement*" {
t.Fatalf("unexpected graphite templates setting: %v", c.Templates)
}
if len(c.Tags) != 1 && c.Tags[0] != "regsion=us-east" {
t.Fatalf("unexpected graphite templates setting: %v", c.Tags)
}
}
func TestConfigValidateEmptyTemplate(t *testing.T) {
c := graphite.NewConfig()
c.Templates = []string{""}
if err := c.Validate(); err == nil {
t.Errorf("config validate expected error. got nil")
}
c.Templates = []string{" "}
if err := c.Validate(); err == nil {
t.Errorf("config validate expected error. got nil")
}
}
func TestConfigValidateTooManyField(t *testing.T) {
c := graphite.NewConfig()
c.Templates = []string{"a measurement b c"}
if err := c.Validate(); err == nil {
t.Errorf("config validate expected error. got nil")
}
}
func TestConfigValidateTemplatePatterns(t *testing.T) {
c := graphite.NewConfig()
c.Templates = []string{"*measurement"}
if err := c.Validate(); err == nil {
t.Errorf("config validate expected error. got nil")
}
c.Templates = []string{".host.region"}
if err := c.Validate(); err == nil {
t.Errorf("config validate expected error. got nil")
}
}
func TestConfigValidateFilter(t *testing.T) {
c := graphite.NewConfig()
c.Templates = []string{".server measurement*"}
if err := c.Validate(); err == nil {
t.Errorf("config validate expected error. got nil")
}
c.Templates = []string{". .server measurement*"}
if err := c.Validate(); err == nil {
t.Errorf("config validate expected error. got nil")
}
c.Templates = []string{"server* measurement*"}
if err := c.Validate(); err == nil {
t.Errorf("config validate expected error. got nil")
}
}
func TestConfigValidateTemplateTags(t *testing.T) {
c := graphite.NewConfig()
c.Templates = []string{"*.server measurement* foo"}
if err := c.Validate(); err == nil {
t.Errorf("config validate expected error. got nil")
}
c.Templates = []string{"*.server measurement* foo=bar="}
if err := c.Validate(); err == nil {
t.Errorf("config validate expected error. got nil")
}
c.Templates = []string{"*.server measurement* foo=bar,"}
if err := c.Validate(); err == nil {
t.Errorf("config validate expected error. got nil")
}
c.Templates = []string{"*.server measurement* ="}
if err := c.Validate(); err == nil {
t.Errorf("config validate expected error. got nil")
}
}
func TestConfigValidateDefaultTags(t *testing.T) {
c := graphite.NewConfig()
c.Tags = []string{"foo"}
if err := c.Validate(); err == nil {
t.Errorf("config validate expected error. got nil")
}
c.Tags = []string{"foo=bar="}
if err := c.Validate(); err == nil {
t.Errorf("config validate expected error. got nil")
}
c.Tags = []string{"foo=bar", ""}
if err := c.Validate(); err == nil {
t.Errorf("config validate expected error. got nil")
}
c.Tags = []string{"="}
if err := c.Validate(); err == nil {
t.Errorf("config validate expected error. got nil")
}
}
func TestConfigValidateFilterDuplicates(t *testing.T) {
c := graphite.NewConfig()
c.Templates = []string{"foo measurement*", "foo .host.measurement"}
if err := c.Validate(); err == nil {
t.Errorf("config validate expected error. got nil")
}
// duplicate default templates
c.Templates = []string{"measurement*", ".host.measurement"}
if err := c.Validate(); err == nil {
t.Errorf("config validate expected error. got nil")
}
}

342
Godeps/_workspace/src/github.com/influxdb/influxdb/services/graphite/parser.go generated vendored
View File

@ -1,342 +0,0 @@
package graphite
import (
"fmt"
"math"
"sort"
"strconv"
"strings"
"time"
"github.com/influxdb/influxdb/tsdb"
)
var defaultTemplate *template
func init() {
var err error
defaultTemplate, err = NewTemplate("measurement*", nil, DefaultSeparator)
if err != nil {
panic(err)
}
}
// Parser encapsulates a Graphite Parser.
type Parser struct {
matcher *matcher
tags tsdb.Tags
}
// Options are configurable values that can be provided to a Parser
type Options struct {
Separator string
Templates []string
DefaultTags tsdb.Tags
}
// NewParserWithOptions returns a graphite parser using the given options
func NewParserWithOptions(options Options) (*Parser, error) {
matcher := newMatcher()
matcher.AddDefaultTemplate(defaultTemplate)
for _, pattern := range options.Templates {
template := pattern
filter := ""
// Format is [filter] <template> [tag1=value1,tag2=value2]
parts := strings.Fields(pattern)
if len(parts) >= 2 {
if strings.Contains(parts[1], "=") {
template = parts[0]
} else {
filter = parts[0]
template = parts[1]
}
}
// Parse out the default tags specific to this template
tags := tsdb.Tags{}
if strings.Contains(parts[len(parts)-1], "=") {
tagStrs := strings.Split(parts[len(parts)-1], ",")
for _, kv := range tagStrs {
parts := strings.Split(kv, "=")
tags[parts[0]] = parts[1]
}
}
tmpl, err := NewTemplate(template, tags, options.Separator)
if err != nil {
return nil, err
}
matcher.Add(filter, tmpl)
}
return &Parser{matcher: matcher, tags: options.DefaultTags}, nil
}
// NewParser returns a GraphiteParser instance.
func NewParser(templates []string, defaultTags tsdb.Tags) (*Parser, error) {
return NewParserWithOptions(
Options{
Templates: templates,
DefaultTags: defaultTags,
Separator: DefaultSeparator,
})
}
// Parse performs Graphite parsing of a single line.
func (p *Parser) Parse(line string) (tsdb.Point, error) {
// Break into 3 fields (name, value, timestamp).
fields := strings.Fields(line)
if len(fields) != 2 && len(fields) != 3 {
return nil, fmt.Errorf("received %q which doesn't have required fields", line)
}
// decode the name and tags
matcher := p.matcher.Match(fields[0])
measurement, tags := matcher.Apply(fields[0])
// Could not extract measurement, use the raw value
if measurement == "" {
measurement = fields[0]
}
// Parse value.
v, err := strconv.ParseFloat(fields[1], 64)
if err != nil {
return nil, fmt.Errorf(`field "%s" value: %s`, fields[0], err)
}
fieldValues := map[string]interface{}{"value": v}
// If no 3rd field, use now as timestamp
timestamp := time.Now().UTC()
if len(fields) == 3 {
// Parse timestamp.
unixTime, err := strconv.ParseFloat(fields[2], 64)
if err != nil {
return nil, fmt.Errorf(`field "%s" time: %s`, fields[0], err)
}
// -1 is a special value that gets converted to current UTC time
// See https://github.com/graphite-project/carbon/issues/54
if unixTime != float64(-1) {
// Check if we have fractional seconds
timestamp = time.Unix(int64(unixTime), int64((unixTime-math.Floor(unixTime))*float64(time.Second)))
}
}
// Set the default tags on the point if they are not already set
for k, v := range p.tags {
if _, ok := tags[k]; !ok {
tags[k] = v
}
}
point := tsdb.NewPoint(measurement, tags, fieldValues, timestamp)
return point, nil
}
// template represents a pattern and tags to map a graphite metric string to a influxdb Point
type template struct {
tags []string
defaultTags tsdb.Tags
greedyMeasurement bool
separator string
}
func NewTemplate(pattern string, defaultTags tsdb.Tags, separator string) (*template, error) {
tags := strings.Split(pattern, ".")
hasMeasurement := false
template := &template{tags: tags, defaultTags: defaultTags, separator: separator}
for _, tag := range tags {
if strings.HasPrefix(tag, "measurement") {
hasMeasurement = true
}
if tag == "measurement*" {
template.greedyMeasurement = true
}
}
if !hasMeasurement {
return nil, fmt.Errorf("no measurement specified for template. %q", pattern)
}
return template, nil
}
// Apply extracts the template fields form the given line and returns the measurement
// name and tags
func (t *template) Apply(line string) (string, map[string]string) {
fields := strings.Split(line, ".")
var (
measurement []string
tags = make(map[string]string)
)
// Set any default tags
for k, v := range t.defaultTags {
tags[k] = v
}
for i, tag := range t.tags {
if i >= len(fields) {
continue
}
if tag == "measurement" {
measurement = append(measurement, fields[i])
} else if tag == "measurement*" {
measurement = append(measurement, fields[i:]...)
break
} else if tag != "" {
tags[tag] = fields[i]
}
}
return strings.Join(measurement, t.separator), tags
}
// matcher determines which template should be applied to a given metric
// based on a filter tree.
type matcher struct {
root *node
defaultTemplate *template
}
func newMatcher() *matcher {
return &matcher{
root: &node{},
}
}
// Add inserts the template in the filter tree based the given filter
func (m *matcher) Add(filter string, template *template) {
if filter == "" {
m.AddDefaultTemplate(template)
return
}
m.root.Insert(filter, template)
}
func (m *matcher) AddDefaultTemplate(template *template) {
m.defaultTemplate = template
}
// Match returns the template that matches the given graphite line
func (m *matcher) Match(line string) *template {
tmpl := m.root.Search(line)
if tmpl != nil {
return tmpl
}
return m.defaultTemplate
}
// node is an item in a sorted k-ary tree. Each child is sorted by its value.
// The special value of "*", is always last.
type node struct {
value string
children nodes
template *template
}
func (n *node) insert(values []string, template *template) {
// Add the end, set the template
if len(values) == 0 {
n.template = template
return
}
// See if the the current element already exists in the tree. If so, insert the
// into that sub-tree
for _, v := range n.children {
if v.value == values[0] {
v.insert(values[1:], template)
return
}
}
// New element, add it to the tree and sort the children
newNode := &node{value: values[0]}
n.children = append(n.children, newNode)
sort.Sort(&n.children)
// Now insert the rest of the tree into the new element
newNode.insert(values[1:], template)
}
// Insert inserts the given string template into the tree. The filter string is separated
// on "." and each part is used as the path in the tree.
func (n *node) Insert(filter string, template *template) {
n.insert(strings.Split(filter, "."), template)
}
func (n *node) search(lineParts []string) *template {
// Nothing to search
if len(lineParts) == 0 || len(n.children) == 0 {
return n.template
}
// If last element is a wildcard, don't include in this search since it's sorted
// to the end but lexicographically it would not always be and sort.Search assumes
// the slice is sorted.
length := len(n.children)
if n.children[length-1].value == "*" {
length -= 1
}
// Find the index of child with an exact match
i := sort.Search(length, func(i int) bool {
return n.children[i].value >= lineParts[0]
})
// Found an exact match, so search that child sub-tree
if i < len(n.children) && n.children[i].value == lineParts[0] {
return n.children[i].search(lineParts[1:])
}
// Not an exact match, see if we have a wildcard child to search
if n.children[len(n.children)-1].value == "*" {
return n.children[len(n.children)-1].search(lineParts[1:])
}
return n.template
}
func (n *node) Search(line string) *template {
return n.search(strings.Split(line, "."))
}
type nodes []*node
// Less returns a boolean indicating whether the filter at position j
// is less than the filter at position k. Filters are order by string
// comparison of each component parts. A wildcard value "*" is never
// less than a non-wildcard value.
//
// For example, the filters:
// "*.*"
// "servers.*"
// "servers.localhost"
// "*.localhost"
//
// Would be sorted as:
// "servers.localhost"
// "servers.*"
// "*.localhost"
// "*.*"
func (n *nodes) Less(j, k int) bool {
if (*n)[j].value == "*" && (*n)[k].value != "*" {
return false
}
if (*n)[j].value != "*" && (*n)[k].value == "*" {
return true
}
return (*n)[j].value < (*n)[k].value
}
func (n *nodes) Swap(i, j int) { (*n)[i], (*n)[j] = (*n)[j], (*n)[i] }
func (n *nodes) Len() int { return len(*n) }

548
Godeps/_workspace/src/github.com/influxdb/influxdb/services/graphite/parser_test.go generated vendored
View File

@ -1,548 +0,0 @@
package graphite_test
import (
"math"
"strconv"
"testing"
"time"
"github.com/influxdb/influxdb/services/graphite"
"github.com/influxdb/influxdb/tsdb"
)
func BenchmarkParse(b *testing.B) {
p, err := graphite.NewParser([]string{
"*.* .wrong.measurement*",
"servers.* .host.measurement*",
"servers.localhost .host.measurement*",
"*.localhost .host.measurement*",
"*.*.cpu .host.measurement*",
"a.b.c .host.measurement*",
"influxd.*.foo .host.measurement*",
"prod.*.mem .host.measurement*",
}, nil)
if err != nil {
b.Fatalf("unexpected error creating parser, got %v", err)
}
for i := 0; i < b.N; i++ {
p.Parse("servers.localhost.cpu.load 11 1435077219")
}
}
func TestTemplateApply(t *testing.T) {
var tests = []struct {
test string
input string
template string
measurement string
tags map[string]string
err string
}{
{
test: "metric only",
input: "cpu",
template: "measurement",
measurement: "cpu",
},
{
test: "metric with single series",
input: "cpu.server01",
template: "measurement.hostname",
measurement: "cpu",
tags: map[string]string{"hostname": "server01"},
},
{
test: "metric with multiple series",
input: "cpu.us-west.server01",
template: "measurement.region.hostname",
measurement: "cpu",
tags: map[string]string{"hostname": "server01", "region": "us-west"},
},
{
test: "no metric",
tags: make(map[string]string),
err: `no measurement specified for template. ""`,
},
{
test: "ignore unnamed",
input: "foo.cpu",
template: "measurement",
measurement: "foo",
tags: make(map[string]string),
},
{
test: "name shorter than template",
input: "foo",
template: "measurement.A.B.C",
measurement: "foo",
tags: make(map[string]string),
},
{
test: "wildcard measurement at end",
input: "prod.us-west.server01.cpu.load",
template: "env.zone.host.measurement*",
measurement: "cpu.load",
tags: map[string]string{"env": "prod", "zone": "us-west", "host": "server01"},
},
{
test: "skip fields",
input: "ignore.us-west.ignore-this-too.cpu.load",
template: ".zone..measurement*",
measurement: "cpu.load",
tags: map[string]string{"zone": "us-west"},
},
}
for _, test := range tests {
tmpl, err := graphite.NewTemplate(test.template, nil, graphite.DefaultSeparator)
if errstr(err) != test.err {
t.Fatalf("err does not match. expected %v, got %v", test.err, err)
}
if err != nil {
// If we erred out,it was intended and the following tests won't work
continue
}
measurement, tags := tmpl.Apply(test.input)
if measurement != test.measurement {
t.Fatalf("name parse failer. expected %v, got %v", test.measurement, measurement)
}
if len(tags) != len(test.tags) {
t.Fatalf("unexpected number of tags. expected %v, got %v", test.tags, tags)
}
for k, v := range test.tags {
if tags[k] != v {
t.Fatalf("unexpected tag value for tags[%s]. expected %q, got %q", k, v, tags[k])
}
}
}
}
func TestParseMissingMeasurement(t *testing.T) {
_, err := graphite.NewParser([]string{"a.b.c"}, nil)
if err == nil {
t.Fatalf("expected error creating parser, got nil")
}
}
func TestParse(t *testing.T) {
testTime := time.Now().Round(time.Second)
epochTime := testTime.Unix()
strTime := strconv.FormatInt(epochTime, 10)
var tests = []struct {
test string
input string
measurement string
tags map[string]string
value float64
time time.Time
template string
err string
}{
{
test: "normal case",
input: `cpu.foo.bar 50 ` + strTime,
template: "measurement.foo.bar",
measurement: "cpu",
tags: map[string]string{
"foo": "foo",
"bar": "bar",
},
value: 50,
time: testTime,
},
{
test: "metric only with float value",
input: `cpu 50.554 ` + strTime,
measurement: "cpu",
template: "measurement",
value: 50.554,
time: testTime,
},
{
test: "missing metric",
input: `1419972457825`,
template: "measurement",
err: `received "1419972457825" which doesn't have required fields`,
},
{
test: "should error parsing invalid float",
input: `cpu 50.554z 1419972457825`,
template: "measurement",
err: `field "cpu" value: strconv.ParseFloat: parsing "50.554z": invalid syntax`,
},
{
test: "should error parsing invalid int",
input: `cpu 50z 1419972457825`,
template: "measurement",
err: `field "cpu" value: strconv.ParseFloat: parsing "50z": invalid syntax`,
},
{
test: "should error parsing invalid time",
input: `cpu 50.554 14199724z57825`,
template: "measurement",
err: `field "cpu" time: strconv.ParseFloat: parsing "14199724z57825": invalid syntax`,
},
}
for _, test := range tests {
p, err := graphite.NewParser([]string{test.template}, nil)
if err != nil {
t.Fatalf("unexpected error creating graphite parser: %v", err)
}
point, err := p.Parse(test.input)
if errstr(err) != test.err {
t.Fatalf("err does not match. expected %v, got %v", test.err, err)
}
if err != nil {
// If we erred out,it was intended and the following tests won't work
continue
}
if point.Name() != test.measurement {
t.Fatalf("name parse failer. expected %v, got %v", test.measurement, point.Name())
}
if len(point.Tags()) != len(test.tags) {
t.Fatalf("tags len mismatch. expected %d, got %d", len(test.tags), len(point.Tags()))
}
f := point.Fields()["value"].(float64)
if point.Fields()["value"] != f {
t.Fatalf("floatValue value mismatch. expected %v, got %v", test.value, f)
}
if point.Time().UnixNano()/1000000 != test.time.UnixNano()/1000000 {
t.Fatalf("time value mismatch. expected %v, got %v", test.time.UnixNano(), point.Time().UnixNano())
}
}
}
func TestParseNaN(t *testing.T) {
p, err := graphite.NewParser([]string{"measurement*"}, nil)
if err != nil {
t.Fatalf("unexpected error creating parser, got %v", err)
}
pt, err := p.Parse("servers.localhost.cpu_load NaN 1435077219")
if err != nil {
t.Fatalf("parse error: %v", err)
}
exp := tsdb.NewPoint("servers.localhost.cpu_load",
tsdb.Tags{},
tsdb.Fields{"value": math.NaN()},
time.Unix(1435077219, 0))
if exp.String() != pt.String() {
t.Errorf("parse mismatch: got %v, exp %v", pt.String(), exp.String())
}
if !math.IsNaN(pt.Fields()["value"].(float64)) {
t.Errorf("parse value mismatch: expected NaN")
}
}
func TestFilterMatchDefault(t *testing.T) {
p, err := graphite.NewParser([]string{"servers.localhost .host.measurement*"}, nil)
if err != nil {
t.Fatalf("unexpected error creating parser, got %v", err)
}
exp := tsdb.NewPoint("miss.servers.localhost.cpu_load",
tsdb.Tags{},
tsdb.Fields{"value": float64(11)},
time.Unix(1435077219, 0))
pt, err := p.Parse("miss.servers.localhost.cpu_load 11 1435077219")
if err != nil {
t.Fatalf("parse error: %v", err)
}
if exp.String() != pt.String() {
t.Errorf("parse mismatch: got %v, exp %v", pt.String(), exp.String())
}
}
func TestFilterMatchMultipleMeasurement(t *testing.T) {
p, err := graphite.NewParser([]string{"servers.localhost .host.measurement.measurement*"}, nil)
if err != nil {
t.Fatalf("unexpected error creating parser, got %v", err)
}
exp := tsdb.NewPoint("cpu.cpu_load.10",
tsdb.Tags{"host": "localhost"},
tsdb.Fields{"value": float64(11)},
time.Unix(1435077219, 0))
pt, err := p.Parse("servers.localhost.cpu.cpu_load.10 11 1435077219")
if err != nil {
t.Fatalf("parse error: %v", err)
}
if exp.String() != pt.String() {
t.Errorf("parse mismatch: got %v, exp %v", pt.String(), exp.String())
}
}
func TestFilterMatchMultipleMeasurementSeparator(t *testing.T) {
p, err := graphite.NewParserWithOptions(graphite.Options{
Templates: []string{"servers.localhost .host.measurement.measurement*"},
Separator: "_",
})
if err != nil {
t.Fatalf("unexpected error creating parser, got %v", err)
}
exp := tsdb.NewPoint("cpu_cpu_load_10",
tsdb.Tags{"host": "localhost"},
tsdb.Fields{"value": float64(11)},
time.Unix(1435077219, 0))
pt, err := p.Parse("servers.localhost.cpu.cpu_load.10 11 1435077219")
if err != nil {
t.Fatalf("parse error: %v", err)
}
if exp.String() != pt.String() {
t.Errorf("parse mismatch: got %v, exp %v", pt.String(), exp.String())
}
}
func TestFilterMatchSingle(t *testing.T) {
p, err := graphite.NewParser([]string{"servers.localhost .host.measurement*"}, nil)
if err != nil {
t.Fatalf("unexpected error creating parser, got %v", err)
}
exp := tsdb.NewPoint("cpu_load",
tsdb.Tags{"host": "localhost"},
tsdb.Fields{"value": float64(11)},
time.Unix(1435077219, 0))
pt, err := p.Parse("servers.localhost.cpu_load 11 1435077219")
if err != nil {
t.Fatalf("parse error: %v", err)
}
if exp.String() != pt.String() {
t.Errorf("parse mismatch: got %v, exp %v", pt.String(), exp.String())
}
}
func TestParseNoMatch(t *testing.T) {
p, err := graphite.NewParser([]string{"servers.*.cpu .host.measurement.cpu.measurement"}, nil)
if err != nil {
t.Fatalf("unexpected error creating parser, got %v", err)
}
exp := tsdb.NewPoint("servers.localhost.memory.VmallocChunk",
tsdb.Tags{},
tsdb.Fields{"value": float64(11)},
time.Unix(1435077219, 0))
pt, err := p.Parse("servers.localhost.memory.VmallocChunk 11 1435077219")
if err != nil {
t.Fatalf("parse error: %v", err)
}
if exp.String() != pt.String() {
t.Errorf("parse mismatch: got %v, exp %v", pt.String(), exp.String())
}
}
func TestFilterMatchWildcard(t *testing.T) {
p, err := graphite.NewParser([]string{"servers.* .host.measurement*"}, nil)
if err != nil {
t.Fatalf("unexpected error creating parser, got %v", err)
}
exp := tsdb.NewPoint("cpu_load",
tsdb.Tags{"host": "localhost"},
tsdb.Fields{"value": float64(11)},
time.Unix(1435077219, 0))
pt, err := p.Parse("servers.localhost.cpu_load 11 1435077219")
if err != nil {
t.Fatalf("parse error: %v", err)
}
if exp.String() != pt.String() {
t.Errorf("parse mismatch: got %v, exp %v", pt.String(), exp.String())
}
}
func TestFilterMatchExactBeforeWildcard(t *testing.T) {
p, err := graphite.NewParser([]string{
"servers.* .wrong.measurement*",
"servers.localhost .host.measurement*"}, nil)
if err != nil {
t.Fatalf("unexpected error creating parser, got %v", err)
}
exp := tsdb.NewPoint("cpu_load",
tsdb.Tags{"host": "localhost"},
tsdb.Fields{"value": float64(11)},
time.Unix(1435077219, 0))
pt, err := p.Parse("servers.localhost.cpu_load 11 1435077219")
if err != nil {
t.Fatalf("parse error: %v", err)
}
if exp.String() != pt.String() {
t.Errorf("parse mismatch: got %v, exp %v", pt.String(), exp.String())
}
}
func TestFilterMatchMostLongestFilter(t *testing.T) {
p, err := graphite.NewParser([]string{
"*.* .wrong.measurement*",
"servers.* .wrong.measurement*",
"servers.localhost .wrong.measurement*",
"servers.localhost.cpu .host.resource.measurement*", // should match this
"*.localhost .wrong.measurement*",
}, nil)
if err != nil {
t.Fatalf("unexpected error creating parser, got %v", err)
}
exp := tsdb.NewPoint("cpu_load",
tsdb.Tags{"host": "localhost", "resource": "cpu"},
tsdb.Fields{"value": float64(11)},
time.Unix(1435077219, 0))
pt, err := p.Parse("servers.localhost.cpu.cpu_load 11 1435077219")
if err != nil {
t.Fatalf("parse error: %v", err)
}
if exp.String() != pt.String() {
t.Errorf("parse mismatch: got %v, exp %v", pt.String(), exp.String())
}
}
func TestFilterMatchMultipleWildcards(t *testing.T) {
p, err := graphite.NewParser([]string{
"*.* .wrong.measurement*",
"servers.* .host.measurement*", // should match this
"servers.localhost .wrong.measurement*",
"*.localhost .wrong.measurement*",
}, nil)
if err != nil {
t.Fatalf("unexpected error creating parser, got %v", err)
}
exp := tsdb.NewPoint("cpu_load",
tsdb.Tags{"host": "server01"},
tsdb.Fields{"value": float64(11)},
time.Unix(1435077219, 0))
pt, err := p.Parse("servers.server01.cpu_load 11 1435077219")
if err != nil {
t.Fatalf("parse error: %v", err)
}
if exp.String() != pt.String() {
t.Errorf("parse mismatch: got %v, exp %v", pt.String(), exp.String())
}
}
func TestParseDefaultTags(t *testing.T) {
p, err := graphite.NewParser([]string{"servers.localhost .host.measurement*"}, tsdb.Tags{
"region": "us-east",
"zone": "1c",
"host": "should not set",
})
if err != nil {
t.Fatalf("unexpected error creating parser, got %v", err)
}
exp := tsdb.NewPoint("cpu_load",
tsdb.Tags{"host": "localhost", "region": "us-east", "zone": "1c"},
tsdb.Fields{"value": float64(11)},
time.Unix(1435077219, 0))
pt, err := p.Parse("servers.localhost.cpu_load 11 1435077219")
if err != nil {
t.Fatalf("parse error: %v", err)
}
if exp.String() != pt.String() {
t.Errorf("parse mismatch: got %v, exp %v", pt.String(), exp.String())
}
}
func TestParseDefaultTemplateTags(t *testing.T) {
p, err := graphite.NewParser([]string{"servers.localhost .host.measurement* zone=1c"}, tsdb.Tags{
"region": "us-east",
"host": "should not set",
})
if err != nil {
t.Fatalf("unexpected error creating parser, got %v", err)
}
exp := tsdb.NewPoint("cpu_load",
tsdb.Tags{"host": "localhost", "region": "us-east", "zone": "1c"},
tsdb.Fields{"value": float64(11)},
time.Unix(1435077219, 0))
pt, err := p.Parse("servers.localhost.cpu_load 11 1435077219")
if err != nil {
t.Fatalf("parse error: %v", err)
}
if exp.String() != pt.String() {
t.Errorf("parse mismatch: got %v, exp %v", pt.String(), exp.String())
}
}
func TestParseDefaultTemplateTagsOverridGlobal(t *testing.T) {
p, err := graphite.NewParser([]string{"servers.localhost .host.measurement* zone=1c,region=us-east"}, tsdb.Tags{
"region": "shot not be set",
"host": "should not set",
})
if err != nil {
t.Fatalf("unexpected error creating parser, got %v", err)
}
exp := tsdb.NewPoint("cpu_load",
tsdb.Tags{"host": "localhost", "region": "us-east", "zone": "1c"},
tsdb.Fields{"value": float64(11)},
time.Unix(1435077219, 0))
pt, err := p.Parse("servers.localhost.cpu_load 11 1435077219")
if err != nil {
t.Fatalf("parse error: %v", err)
}
if exp.String() != pt.String() {
t.Errorf("parse mismatch: got %v, exp %v", pt.String(), exp.String())
}
}
func TestParseTemplateWhitespace(t *testing.T) {
p, err := graphite.NewParser([]string{"servers.localhost .host.measurement* zone=1c"}, tsdb.Tags{
"region": "us-east",
"host": "should not set",
})
if err != nil {
t.Fatalf("unexpected error creating parser, got %v", err)
}
exp := tsdb.NewPoint("cpu_load",
tsdb.Tags{"host": "localhost", "region": "us-east", "zone": "1c"},
tsdb.Fields{"value": float64(11)},
time.Unix(1435077219, 0))
pt, err := p.Parse("servers.localhost.cpu_load 11 1435077219")
if err != nil {
t.Fatalf("parse error: %v", err)
}
if exp.String() != pt.String() {
t.Errorf("parse mismatch: got %v, exp %v", pt.String(), exp.String())
}
}

267
Godeps/_workspace/src/github.com/influxdb/influxdb/services/graphite/service.go generated vendored
View File

@ -1,267 +0,0 @@
package graphite
import (
"bufio"
"fmt"
"log"
"math"
"net"
"os"
"strings"
"sync"
"time"
"github.com/influxdb/influxdb/cluster"
"github.com/influxdb/influxdb/meta"
"github.com/influxdb/influxdb/tsdb"
)
const (
udpBufferSize = 65536
leaderWaitTimeout = 30 * time.Second
)
type Service struct {
bindAddress string
database string
protocol string
batchSize int
batchTimeout time.Duration
consistencyLevel cluster.ConsistencyLevel
batcher *tsdb.PointBatcher
parser *Parser
logger *log.Logger
ln net.Listener
addr net.Addr
wg sync.WaitGroup
done chan struct{}
PointsWriter interface {
WritePoints(p *cluster.WritePointsRequest) error
}
MetaStore interface {
WaitForLeader(d time.Duration) error
CreateDatabaseIfNotExists(name string) (*meta.DatabaseInfo, error)
}
}
// NewService returns an instance of the Graphite service.
func NewService(c Config) (*Service, error) {
// Use defaults where necessary.
d := c.WithDefaults()
s := Service{
bindAddress: d.BindAddress,
database: d.Database,
protocol: d.Protocol,
batchSize: d.BatchSize,
batchTimeout: time.Duration(d.BatchTimeout),
logger: log.New(os.Stderr, "[graphite] ", log.LstdFlags),
done: make(chan struct{}),
}
consistencyLevel, err := cluster.ParseConsistencyLevel(d.ConsistencyLevel)
if err != nil {
return nil, err
}
s.consistencyLevel = consistencyLevel
parser, err := NewParserWithOptions(Options{
Templates: d.Templates,
DefaultTags: d.DefaultTags(),
Separator: d.Separator})
if err != nil {
return nil, err
}
s.parser = parser
return &s, nil
}
// Open starts the Graphite input processing data.
func (s *Service) Open() error {
s.logger.Printf("Starting graphite service, batch size %d, batch timeout %s", s.batchSize, s.batchTimeout)
if err := s.MetaStore.WaitForLeader(leaderWaitTimeout); err != nil {
s.logger.Printf("Failed to detect a cluster leader: %s", err.Error())
return err
}
if _, err := s.MetaStore.CreateDatabaseIfNotExists(s.database); err != nil {
s.logger.Printf("Failed to ensure target database %s exists: %s", s.database, err.Error())
return err
}
s.batcher = tsdb.NewPointBatcher(s.batchSize, s.batchTimeout)
s.batcher.Start()
// Start processing batches.
s.wg.Add(1)
go s.processBatches(s.batcher)
var err error
if strings.ToLower(s.protocol) == "tcp" {
s.addr, err = s.openTCPServer()
} else if strings.ToLower(s.protocol) == "udp" {
s.addr, err = s.openUDPServer()
} else {
return fmt.Errorf("unrecognized Graphite input protocol %s", s.protocol)
}
if err != nil {
return err
}
s.logger.Printf("Listening on %s: %s", strings.ToUpper(s.protocol), s.addr.String())
return nil
}
// Close stops all data processing on the Graphite input.
func (s *Service) Close() error {
if s.ln != nil {
s.ln.Close()
}
s.batcher.Stop()
close(s.done)
s.wg.Wait()
s.done = nil
return nil
}
// SetLogger sets the internal logger to the logger passed in.
func (s *Service) SetLogger(l *log.Logger) {
s.logger = l
}
func (s *Service) Addr() net.Addr {
return s.addr
}
// openTCPServer opens the Graphite input in TCP mode and starts processing data.
func (s *Service) openTCPServer() (net.Addr, error) {
ln, err := net.Listen("tcp", s.bindAddress)
if err != nil {
return nil, err
}
s.ln = ln
s.wg.Add(1)
go func() {
defer s.wg.Done()
for {
conn, err := s.ln.Accept()
if opErr, ok := err.(*net.OpError); ok && !opErr.Temporary() {
s.logger.Println("graphite TCP listener closed")
return
}
if err != nil {
s.logger.Println("error accepting TCP connection", err.Error())
continue
}
s.wg.Add(1)
go s.handleTCPConnection(conn)
}
}()
return ln.Addr(), nil
}
// handleTCPConnection services an individual TCP connection for the Graphite input.
func (s *Service) handleTCPConnection(conn net.Conn) {
defer conn.Close()
defer s.wg.Done()
reader := bufio.NewReader(conn)
for {
// Read up to the next newline.
buf, err := reader.ReadBytes('\n')
if err != nil {
return
}
// Trim the buffer, even though there should be no padding
line := strings.TrimSpace(string(buf))
s.handleLine(line)
}
}
// openUDPServer opens the Graphite input in UDP mode and starts processing incoming data.
func (s *Service) openUDPServer() (net.Addr, error) {
addr, err := net.ResolveUDPAddr("udp", s.bindAddress)
if err != nil {
return nil, err
}
conn, err := net.ListenUDP("udp", addr)
if err != nil {
return nil, err
}
buf := make([]byte, udpBufferSize)
s.wg.Add(1)
go func() {
defer s.wg.Done()
for {
n, _, err := conn.ReadFromUDP(buf)
if err != nil {
conn.Close()
return
}
for _, line := range strings.Split(string(buf[:n]), "\n") {
s.handleLine(line)
}
}
}()
return conn.LocalAddr(), nil
}
func (s *Service) handleLine(line string) {
if line == "" {
return
}
// Parse it.
point, err := s.parser.Parse(line)
if err != nil {
s.logger.Printf("unable to parse line: %s", err)
return
}
f, ok := point.Fields()["value"].(float64)
if ok {
// Drop NaN and +/-Inf data points since they are not supported values
if math.IsNaN(f) || math.IsInf(f, 0) {
s.logger.Printf("dropping unsupported value: '%v'", line)
return
}
}
s.batcher.In() <- point
}
// processBatches continually drains the given batcher and writes the batches to the database.
func (s *Service) processBatches(batcher *tsdb.PointBatcher) {
defer s.wg.Done()
for {
select {
case batch := <-batcher.Out():
if err := s.PointsWriter.WritePoints(&cluster.WritePointsRequest{
Database: s.database,
RetentionPolicy: "",
ConsistencyLevel: s.consistencyLevel,
Points: batch,
}); err != nil {
s.logger.Printf("failed to write point batch to database %q: %s", s.database, err)
}
case <-s.done:
return
}
}
}

183
Godeps/_workspace/src/github.com/influxdb/influxdb/services/graphite/service_test.go generated vendored
View File

@ -1,183 +0,0 @@
package graphite_test
import (
"fmt"
"net"
"sync"
"testing"
"time"
"github.com/influxdb/influxdb/cluster"
"github.com/influxdb/influxdb/meta"
"github.com/influxdb/influxdb/services/graphite"
"github.com/influxdb/influxdb/toml"
"github.com/influxdb/influxdb/tsdb"
)
func Test_ServerGraphiteTCP(t *testing.T) {
t.Parallel()
now := time.Now().UTC().Round(time.Second)
config := graphite.NewConfig()
config.Database = "graphitedb"
config.BatchSize = 0 // No batching.
config.BatchTimeout = toml.Duration(time.Second)
config.BindAddress = ":0"
service, err := graphite.NewService(config)
if err != nil {
t.Fatalf("failed to create Graphite service: %s", err.Error())
}
// Allow test to wait until points are written.
var wg sync.WaitGroup
wg.Add(1)
pointsWriter := PointsWriter{
WritePointsFn: func(req *cluster.WritePointsRequest) error {
defer wg.Done()
if req.Database != "graphitedb" {
t.Fatalf("unexpected database: %s", req.Database)
} else if req.RetentionPolicy != "" {
t.Fatalf("unexpected retention policy: %s", req.RetentionPolicy)
} else if req.Points[0].String() !=
tsdb.NewPoint(
"cpu",
map[string]string{},
map[string]interface{}{"value": 23.456},
time.Unix(now.Unix(), 0)).String() {
}
return nil
},
}
service.PointsWriter = &pointsWriter
dbCreator := DatabaseCreator{}
service.MetaStore = &dbCreator
if err := service.Open(); err != nil {
t.Fatalf("failed to open Graphite service: %s", err.Error())
}
if !dbCreator.Created {
t.Fatalf("failed to create target database")
}
// Connect to the graphite endpoint we just spun up
_, port, _ := net.SplitHostPort(service.Addr().String())
conn, err := net.Dial("tcp", "127.0.0.1:"+port)
if err != nil {
t.Fatal(err)
}
data := []byte(`cpu 23.456 `)
data = append(data, []byte(fmt.Sprintf("%d", now.Unix()))...)
data = append(data, '\n')
_, err = conn.Write(data)
conn.Close()
if err != nil {
t.Fatal(err)
}
wg.Wait()
}
func Test_ServerGraphiteUDP(t *testing.T) {
t.Parallel()
now := time.Now().UTC().Round(time.Second)
config := graphite.NewConfig()
config.Database = "graphitedb"
config.BatchSize = 0 // No batching.
config.BatchTimeout = toml.Duration(time.Second)
config.BindAddress = ":10000"
config.Protocol = "udp"
service, err := graphite.NewService(config)
if err != nil {
t.Fatalf("failed to create Graphite service: %s", err.Error())
}
// Allow test to wait until points are written.
var wg sync.WaitGroup
wg.Add(1)
pointsWriter := PointsWriter{
WritePointsFn: func(req *cluster.WritePointsRequest) error {
defer wg.Done()
if req.Database != "graphitedb" {
t.Fatalf("unexpected database: %s", req.Database)
} else if req.RetentionPolicy != "" {
t.Fatalf("unexpected retention policy: %s", req.RetentionPolicy)
} else if req.Points[0].String() !=
tsdb.NewPoint(
"cpu",
map[string]string{},
map[string]interface{}{"value": 23.456},
time.Unix(now.Unix(), 0)).String() {
t.Fatalf("unexpected points: %#v", req.Points[0].String())
}
return nil
},
}
service.PointsWriter = &pointsWriter
dbCreator := DatabaseCreator{}
service.MetaStore = &dbCreator
if err := service.Open(); err != nil {
t.Fatalf("failed to open Graphite service: %s", err.Error())
}
if !dbCreator.Created {
t.Fatalf("failed to create target database")
}
// Connect to the graphite endpoint we just spun up
_, port, _ := net.SplitHostPort(service.Addr().String())
conn, err := net.Dial("udp", "127.0.0.1:"+port)
if err != nil {
t.Fatal(err)
}
data := []byte(`cpu 23.456 `)
data = append(data, []byte(fmt.Sprintf("%d", now.Unix()))...)
data = append(data, '\n')
_, err = conn.Write(data)
if err != nil {
t.Fatal(err)
}
wg.Wait()
conn.Close()
}
// PointsWriter represents a mock impl of PointsWriter.
type PointsWriter struct {
WritePointsFn func(*cluster.WritePointsRequest) error
}
func (w *PointsWriter) WritePoints(p *cluster.WritePointsRequest) error {
return w.WritePointsFn(p)
}
type DatabaseCreator struct {
Created bool
}
func (d *DatabaseCreator) CreateDatabaseIfNotExists(name string) (*meta.DatabaseInfo, error) {
d.Created = true
return nil, nil
}
func (d *DatabaseCreator) WaitForLeader(t time.Duration) error {
return nil
}
// Test Helpers
func errstr(err error) string {
if err != nil {
return err.Error()
}
return ""
}

44
Godeps/_workspace/src/github.com/influxdb/influxdb/services/hh/config.go generated vendored
View File

@ -1,44 +0,0 @@
package hh
import (
"time"
"github.com/influxdb/influxdb/toml"
)
const (
// DefaultMaxSize is the default maximum size of all hinted handoff queues in bytes.
DefaultMaxSize = 1024 * 1024 * 1024
// DefaultMaxAge is the default maximum amount of time that a hinted handoff write
// can stay in the queue. After this time, the write will be purged.
DefaultMaxAge = 7 * 24 * time.Hour
// DefaultRetryRateLimit is the default rate that hinted handoffs will be retried.
// The rate is in bytes per second and applies across all nodes when retried. A
// value of 0 disables the rate limit.
DefaultRetryRateLimit = 0
// DefaultRetryInterval is the default amout of time the system waits before
// attempting to flush hinted handoff queues.
DefaultRetryInterval = time.Second
)
type Config struct {
Enabled bool `toml:"enabled"`
Dir string `toml:"dir"`
MaxSize int64 `toml:"max-size"`
MaxAge toml.Duration `toml:"max-age"`
RetryRateLimit int64 `toml:"retry-rate-limit"`
RetryInterval toml.Duration `toml:"retry-interval"`
}
func NewConfig() Config {
return Config{
Enabled: true,
MaxSize: DefaultMaxSize,
MaxAge: toml.Duration(DefaultMaxAge),
RetryRateLimit: DefaultRetryRateLimit,
RetryInterval: toml.Duration(DefaultRetryInterval),
}
}

45
Godeps/_workspace/src/github.com/influxdb/influxdb/services/hh/config_test.go generated vendored
View File

@ -1,45 +0,0 @@
package hh_test
import (
"testing"
"time"
"github.com/BurntSushi/toml"
"github.com/influxdb/influxdb/services/hh"
)
func TestConfigParse(t *testing.T) {
// Parse configuration.
var c hh.Config
if _, err := toml.Decode(`
enabled = false
retry-interval = "10m"
max-size=2048
max-age="20m"
retry-rate-limit=1000
`, &c); err != nil {
t.Fatal(err)
}
// Validate configuration.
if exp := true; c.Enabled == true {
t.Fatalf("unexpected enabled: got %v, exp %v", c.Enabled, exp)
}
if exp := 10 * time.Minute; c.RetryInterval.String() != exp.String() {
t.Fatalf("unexpected retry interval: got %v, exp %v", c.RetryInterval, exp)
}
if exp := 20 * time.Minute; c.MaxAge.String() != exp.String() {
t.Fatalf("unexpected max age: got %v, exp %v", c.MaxAge, exp)
}
if exp := int64(2048); c.MaxSize != exp {
t.Fatalf("unexpected retry interval: got %v, exp %v", c.MaxSize, exp)
}
if exp := int64(1000); c.RetryRateLimit != exp {
t.Fatalf("unexpected retry rate limit: got %v, exp %v", c.RetryRateLimit, exp)
}
}

5
Godeps/_workspace/src/github.com/influxdb/influxdb/services/hh/doc.go generated vendored
View File

@ -1,5 +0,0 @@
/*
Package hh implements a hinted handoff for writes
*/
package hh

61
Godeps/_workspace/src/github.com/influxdb/influxdb/services/hh/limiter.go generated vendored
View File

@ -1,61 +0,0 @@
package hh
import "time"
type limiter struct {
count int64
limit int64
start time.Time
delay float64
}
// NewRateLimiter returns a new limiter configured to restrict a process to the limit per second.
// limit is the maximum amount that can be used per second. The limit should be > 0. A limit
// <= 0, will not limit the processes.
func NewRateLimiter(limit int64) *limiter {
return &limiter{
start: time.Now(),
limit: limit,
delay: 0.5,
}
}
// Update updates the amount used
func (t *limiter) Update(count int) {
t.count += int64(count)
}
// Delay returns the amount of time, up to 1 second, that caller should wait
// to maintain the configured rate
func (t *limiter) Delay() time.Duration {
if t.limit > 0 {
delta := time.Now().Sub(t.start).Seconds()
rate := int64(float64(t.count) / delta)
// Determine how far off from the max rate we are
delayAdj := float64((t.limit - rate)) / float64(t.limit)
// Don't adjust by more than 1 second at a time
delayAdj = t.clamp(delayAdj, -1, 1)
t.delay -= delayAdj
if t.delay < 0 {
t.delay = 0
}
return time.Duration(t.delay) * time.Second
}
return time.Duration(0)
}
func (t *limiter) clamp(value, min, max float64) float64 {
if value < min {
return min
}
if value > max {
return max
}
return value
}

47
Godeps/_workspace/src/github.com/influxdb/influxdb/services/hh/limiter_test.go generated vendored
View File

@ -1,47 +0,0 @@
package hh
import (
"testing"
"time"
)
func TestLimiter(t *testing.T) {
l := NewRateLimiter(0)
l.Update(500)
if l.Delay().Nanoseconds() != 0 {
t.Errorf("limiter with no limit mismatch: got %v, exp 0", l.Delay())
}
}
func TestLimiterWithinLimit(t *testing.T) {
if testing.Short() {
t.Skip("Shipping TestLimiterWithinLimit")
}
l := NewRateLimiter(1000)
for i := 0; i < 100; i++ {
// 50 ever 100ms = 500/s which should be within the rate
l.Update(50)
l.Delay()
time.Sleep(100 * time.Millisecond)
}
// Should not have any delay
delay := l.Delay().Seconds()
if exp := int(0); int(delay) != exp {
t.Errorf("limiter rate mismatch: got %v, exp %v", int(delay), exp)
}
}
func TestLimiterExceeded(t *testing.T) {
l := NewRateLimiter(1000)
for i := 0; i < 10; i++ {
l.Update(200)
l.Delay()
}
delay := l.Delay().Seconds()
if int(delay) == 0 {
t.Errorf("limiter rate mismatch. expected non-zero delay")
}
}

218
Godeps/_workspace/src/github.com/influxdb/influxdb/services/hh/processor.go generated vendored
View File

@ -1,218 +0,0 @@
package hh
import (
"encoding/binary"
"fmt"
"io/ioutil"
"log"
"os"
"path/filepath"
"strconv"
"sync"
"time"
"github.com/influxdb/influxdb/tsdb"
)
type Processor struct {
mu sync.RWMutex
dir string
maxSize int64
maxAge time.Duration
retryRateLimit int64
queues map[uint64]*queue
writer shardWriter
Logger *log.Logger
}
type ProcessorOptions struct {
MaxSize int64
RetryRateLimit int64
}
func NewProcessor(dir string, writer shardWriter, options ProcessorOptions) (*Processor, error) {
p := &Processor{
dir: dir,
queues: map[uint64]*queue{},
writer: writer,
Logger: log.New(os.Stderr, "[handoff] ", log.LstdFlags),
}
p.setOptions(options)
// Create the root directory if it doesn't already exist.
if err := os.MkdirAll(dir, 0700); err != nil {
return nil, fmt.Errorf("mkdir all: %s", err)
}
if err := p.loadQueues(); err != nil {
return p, err
}
return p, nil
}
func (p *Processor) setOptions(options ProcessorOptions) {
p.maxSize = DefaultMaxSize
if options.MaxSize != 0 {
p.maxSize = options.MaxSize
}
p.retryRateLimit = DefaultRetryRateLimit
if options.RetryRateLimit != 0 {
p.retryRateLimit = options.RetryRateLimit
}
}
func (p *Processor) loadQueues() error {
files, err := ioutil.ReadDir(p.dir)
if err != nil {
return err
}
for _, file := range files {
nodeID, err := strconv.ParseUint(file.Name(), 10, 64)
if err != nil {
return err
}
if _, err := p.addQueue(nodeID); err != nil {
return err
}
}
return nil
}
func (p *Processor) addQueue(nodeID uint64) (*queue, error) {
p.mu.Lock()
defer p.mu.Unlock()
path := filepath.Join(p.dir, strconv.FormatUint(nodeID, 10))
if err := os.MkdirAll(path, 0700); err != nil {
return nil, err
}
queue, err := newQueue(path, p.maxSize)
if err != nil {
return nil, err
}
if err := queue.Open(); err != nil {
return nil, err
}
p.queues[nodeID] = queue
return queue, nil
}
func (p *Processor) WriteShard(shardID, ownerID uint64, points []tsdb.Point) error {
queue, ok := p.queues[ownerID]
if !ok {
var err error
if queue, err = p.addQueue(ownerID); err != nil {
return err
}
}
b := p.marshalWrite(shardID, points)
return queue.Append(b)
}
func (p *Processor) Process() error {
p.mu.RLock()
defer p.mu.RUnlock()
res := make(chan error, len(p.queues))
for nodeID, q := range p.queues {
go func(nodeID uint64, q *queue) {
// Log how many writes we successfully sent at the end
var sent int
start := time.Now()
defer func(start time.Time) {
if sent > 0 {
p.Logger.Printf("%d queued writes sent to node %d in %s", sent, nodeID, time.Since(start))
}
}(start)
limiter := NewRateLimiter(p.retryRateLimit)
for {
// Get the current block from the queue
buf, err := q.Current()
if err != nil {
res <- nil
break
}
// unmarshal the byte slice back to shard ID and points
shardID, points, err := p.unmarshalWrite(buf)
if err != nil {
p.Logger.Printf("unmarshal write failed: %v", err)
if err := q.Advance(); err != nil {
res <- err
}
return
}
// Try to send the write to the node
if err := p.writer.WriteShard(shardID, nodeID, points); err != nil && tsdb.IsRetryable(err) {
p.Logger.Printf("remote write failed: %v", err)
res <- nil
break
}
// If we get here, the write succeeded so advance the queue to the next item
if err := q.Advance(); err != nil {
res <- err
return
}
sent += 1
// Update how many bytes we've sent
limiter.Update(len(buf))
// Block to maintain the throughput rate
time.Sleep(limiter.Delay())
}
}(nodeID, q)
}
for range p.queues {
err := <-res
if err != nil {
return err
}
}
return nil
}
func (p *Processor) marshalWrite(shardID uint64, points []tsdb.Point) []byte {
b := make([]byte, 8)
binary.BigEndian.PutUint64(b, shardID)
for _, p := range points {
b = append(b, []byte(p.String())...)
b = append(b, '\n')
}
return b
}
func (p *Processor) unmarshalWrite(b []byte) (uint64, []tsdb.Point, error) {
if len(b) < 8 {
return 0, nil, fmt.Errorf("too short: len = %d", len(b))
}
ownerID := binary.BigEndian.Uint64(b[:8])
points, err := tsdb.ParsePoints(b[8:])
return ownerID, points, err
}
func (p *Processor) PurgeOlderThan(when time.Duration) error {
p.mu.Lock()
defer p.mu.Unlock()
for _, queue := range p.queues {
if err := queue.PurgeOlderThan(time.Now().Add(-when)); err != nil {
return err
}
}
return nil
}

80
Godeps/_workspace/src/github.com/influxdb/influxdb/services/hh/processor_test.go generated vendored
View File

@ -1,80 +0,0 @@
package hh
import (
"io/ioutil"
"testing"
"time"
"github.com/influxdb/influxdb/tsdb"
)
type fakeShardWriter struct {
ShardWriteFn func(shardID, nodeID uint64, points []tsdb.Point) error
}
func (f *fakeShardWriter) WriteShard(shardID, nodeID uint64, points []tsdb.Point) error {
return f.ShardWriteFn(shardID, nodeID, points)
}
func TestProcessorProcess(t *testing.T) {
dir, err := ioutil.TempDir("", "processor_test")
if err != nil {
t.Fatalf("failed to create temp dir: %v", err)
}
// expected data to be queue and sent to the shardWriter
var expShardID, expNodeID, count = uint64(100), uint64(200), 0
pt := tsdb.NewPoint("cpu", tsdb.Tags{"foo": "bar"}, tsdb.Fields{"value": 1.0}, time.Unix(0, 0))
sh := &fakeShardWriter{
ShardWriteFn: func(shardID, nodeID uint64, points []tsdb.Point) error {
count += 1
if shardID != expShardID {
t.Errorf("Process() shardID mismatch: got %v, exp %v", shardID, expShardID)
}
if nodeID != expNodeID {
t.Errorf("Process() nodeID mismatch: got %v, exp %v", nodeID, expNodeID)
}
if exp := 1; len(points) != exp {
t.Fatalf("Process() points mismatch: got %v, exp %v", len(points), exp)
}
if points[0].String() != pt.String() {
t.Fatalf("Process() points mismatch:\n got %v\n exp %v", points[0].String(), pt.String())
}
return nil
},
}
p, err := NewProcessor(dir, sh, ProcessorOptions{MaxSize: 1024})
if err != nil {
t.Fatalf("Process() failed to create processor: %v", err)
}
// This should queue the writes
if err := p.WriteShard(expShardID, expNodeID, []tsdb.Point{pt}); err != nil {
t.Fatalf("Process() failed to write points: %v", err)
}
// This should send the write to the shard writer
if err := p.Process(); err != nil {
t.Fatalf("Process() failed to write points: %v", err)
}
if exp := 1; count != exp {
t.Fatalf("Process() write count mismatch: got %v, exp %v", count, exp)
}
// Queue should be empty so no writes should be send again
if err := p.Process(); err != nil {
t.Fatalf("Process() failed to write points: %v", err)
}
// Count should stay the same
if exp := 1; count != exp {
t.Fatalf("Process() write count mismatch: got %v, exp %v", count, exp)
}
}

666
Godeps/_workspace/src/github.com/influxdb/influxdb/services/hh/queue.go generated vendored
View File

@ -1,666 +0,0 @@
package hh
import (
"encoding/binary"
"fmt"
"io"
"io/ioutil"
"os"
"path/filepath"
"strconv"
"sync"
"time"
)
var (
ErrNotOpen = fmt.Errorf("queue not open")
ErrQueueFull = fmt.Errorf("queue is full")
ErrSegmentFull = fmt.Errorf("segment is full")
)
const (
defaultSegmentSize = 10 * 1024 * 1024
footerSize = 8
)
// queue is a bounded, disk-backed, append-only type that combines queue and
// log semantics. byte slices can be appended and read back in-order.
// The queue maintains a pointer to the current head
// byte slice and can re-read from the head until it has been advanced.
//
// Internally, the queue writes byte slices to multiple segment files so
// that disk space can be reclaimed. When a segment file is larger than
// the max segment size, a new file is created. Segments are removed
// after their head pointer has advanced past the last entry. The first
// segment is the head, and the last segment is the tail. Reads are from
// the head segment and writes tail segment.
//
// queues can have a max size configured such that when the size of all
// segments on disk exceeds the size, write will fail.
//
// ┌─────┐
// │Head │
// ├─────┘
// │
// ▼
// ┌─────────────────┐ ┌─────────────────┐┌─────────────────┐
// │Segment 1 - 10MB │ │Segment 2 - 10MB ││Segment 3 - 10MB │
// └─────────────────┘ └─────────────────┘└─────────────────┘
// ▲
// │
// │
// ┌─────┐
// │Tail │
// └─────┘
type queue struct {
mu sync.RWMutex
// Directory to create segments
dir string
// The head and tail segments. Reads are from the beginning of head,
// writes are appended to the tail.
head, tail *segment
// The maximum size in bytes of a segment file before a new one should be created
maxSegmentSize int64
// The maximum size allowed in bytes of all segments before writes will return
// an error
maxSize int64
// The segments that exist on disk
segments segments
}
type segments []*segment
// newQueue create a queue that will store segments in dir and that will
// consume more than maxSize on disk.
func newQueue(dir string, maxSize int64) (*queue, error) {
return &queue{
dir: dir,
maxSegmentSize: defaultSegmentSize,
maxSize: maxSize,
segments: segments{},
}, nil
}
// Open opens the queue for reading and writing
func (l *queue) Open() error {
l.mu.Lock()
defer l.mu.Unlock()
segments, err := l.loadSegments()
if err != nil {
return err
}
l.segments = segments
if len(l.segments) == 0 {
_, err := l.addSegment()
if err != nil {
return err
}
}
l.head = l.segments[0]
l.tail = l.segments[len(l.segments)-1]
// If the head has been fully advanced and the segment size is modified,
// existing segments an get stuck and never allow clients to advance further.
// This advances the segment if the current head is already at the end.
_, err = l.head.current()
if err == io.EOF {
return l.trimHead()
}
return nil
}
// Close stops the queue for reading and writing
func (l *queue) Close() error {
l.mu.Lock()
defer l.mu.Unlock()
for _, s := range l.segments {
if err := s.close(); err != nil {
return err
}
}
l.head = nil
l.tail = nil
l.segments = nil
return nil
}
// SetMaxSegmentSize updates the max segment size for new and existing
// segments.
func (l *queue) SetMaxSegmentSize(size int64) error {
l.mu.Lock()
defer l.mu.Unlock()
l.maxSegmentSize = size
for _, s := range l.segments {
s.SetMaxSegmentSize(size)
}
if l.tail.diskUsage() >= l.maxSegmentSize {
segment, err := l.addSegment()
if err != nil {
return err
}
l.tail = segment
}
return nil
}
func (l *queue) PurgeOlderThan(when time.Time) error {
l.mu.Lock()
defer l.mu.Unlock()
// Add a new empty segment so old ones can be reclaimed
if _, err := l.addSegment(); err != nil {
return err
}
cutoff := when.Truncate(time.Second)
for {
mod, err := l.head.lastModified()
if err != nil {
return err
}
if mod.After(cutoff) || mod.Equal(cutoff) {
return nil
}
if err := l.trimHead(); err != nil {
return err
}
}
}
// diskUsage returns the total size on disk used by the queue
func (l *queue) diskUsage() int64 {
var size int64
for _, s := range l.segments {
size += s.diskUsage()
}
return size
}
// addSegment creates a new empty segment file
func (l *queue) addSegment() (*segment, error) {
nextID, err := l.nextSegmentID()
if err != nil {
return nil, err
}
segment, err := newSegment(filepath.Join(l.dir, strconv.FormatUint(nextID, 10)), l.maxSegmentSize)
if err != nil {
return nil, err
}
l.segments = append(l.segments, segment)
return segment, nil
}
// loadSegments loads all segments on disk
func (l *queue) loadSegments() (segments, error) {
segments := []*segment{}
files, err := ioutil.ReadDir(l.dir)
if err != nil {
return segments, err
}
for _, segment := range files {
// Segments should be files. Skip anything that is not a dir.
if segment.IsDir() {
continue
}
// Segments file names are all numeric
_, err := strconv.ParseUint(segment.Name(), 10, 64)
if err != nil {
continue
}
segment, err := newSegment(filepath.Join(l.dir, segment.Name()), l.maxSegmentSize)
if err != nil {
return segments, err
}
segments = append(segments, segment)
}
return segments, nil
}
// nextSegmentID returns the next segment ID that is free
func (l *queue) nextSegmentID() (uint64, error) {
segments, err := ioutil.ReadDir(l.dir)
if err != nil {
return 0, err
}
var maxID uint64
for _, segment := range segments {
// Segments should be files. Skip anything that is not a dir.
if segment.IsDir() {
continue
}
// Segments file names are all numeric
segmentID, err := strconv.ParseUint(segment.Name(), 10, 64)
if err != nil {
continue
}
if segmentID > maxID {
maxID = segmentID
}
}
return maxID + 1, nil
}
// Append appends a byte slice to the end of the queue
func (l *queue) Append(b []byte) error {
l.mu.Lock()
defer l.mu.Unlock()
if l.tail == nil {
return ErrNotOpen
}
if l.diskUsage()+int64(len(b)) > l.maxSize {
return ErrQueueFull
}
// Append the entry to the tail, if the segment is full,
// try to create new segment and retry the append
if err := l.tail.append(b); err == ErrSegmentFull {
segment, err := l.addSegment()
if err != nil {
return err
}
l.tail = segment
return l.tail.append(b)
}
return nil
}
// Current returns the current byte slice at the head of the queue
func (l *queue) Current() ([]byte, error) {
if l.head == nil {
return nil, ErrNotOpen
}
return l.head.current()
}
// Advance moves the head point to the next byte slice in the queue
func (l *queue) Advance() error {
l.mu.Lock()
defer l.mu.Unlock()
if l.head == nil {
return ErrNotOpen
}
err := l.head.advance()
if err == io.EOF {
if err := l.trimHead(); err != nil {
return err
}
}
return nil
}
func (l *queue) trimHead() error {
if len(l.segments) > 1 {
l.segments = l.segments[1:]
if err := l.head.close(); err != nil {
return err
}
if err := os.Remove(l.head.path); err != nil {
return err
}
l.head = l.segments[0]
}
return nil
}
// Segment is a queue using a single file. The structure of a segment is a series
// lengths + block with a single footer point to the position in the segment of the
// current head block.
//
// ┌──────────────────────────┐ ┌──────────────────────────┐ ┌────────────┐
// │ Block 1 │ │ Block 2 │ │ Footer │
// └──────────────────────────┘ └──────────────────────────┘ └────────────┘
// ┌────────────┐┌────────────┐ ┌────────────┐┌────────────┐ ┌────────────┐
// │Block 1 Len ││Block 1 Body│ │Block 2 Len ││Block 2 Body│ │Head Offset │
// │ 8 bytes ││ N bytes │ │ 8 bytes ││ N bytes │ │ 8 bytes │
// └────────────┘└────────────┘ └────────────┘└────────────┘ └────────────┘
//
// The footer holds the pointer to the head entry at the end of the segment to allow writes
// to seek to the end and write sequentially (vs having to seek back to the beginning of
// the segment to update the head pointer). Reads must seek to the end then back into the
// segment offset stored in the footer.
//
// Segments store arbitrary byte slices and leave the serialization to the caller. Segments
// are created with a max size and will block writes when the segment is full.
type segment struct {
mu sync.RWMutex
size int64
file *os.File
path string
pos int64
currentSize int64
maxSize int64
}
func newSegment(path string, maxSize int64) (*segment, error) {
f, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR, 0600)
if err != nil {
return nil, err
}
stats, err := os.Stat(path)
if err != nil {
return nil, err
}
s := &segment{file: f, path: path, size: stats.Size(), maxSize: maxSize}
if err := s.open(); err != nil {
return nil, err
}
return s, nil
}
func (l *segment) open() error {
l.mu.Lock()
defer l.mu.Unlock()
// If it's a new segment then write the location of the current record in this segment
if l.size == 0 {
l.pos = 0
l.currentSize = 0
if err := l.writeUint64(uint64(l.pos)); err != nil {
return err
}
if err := l.file.Sync(); err != nil {
return err
}
l.size = footerSize
return nil
}
// Existing segment so read the current position and the size of the current block
if err := l.seekEnd(-footerSize); err != nil {
return err
}
pos, err := l.readUint64()
if err != nil {
return err
}
l.pos = int64(pos)
if err := l.seekToCurrent(); err != nil {
return err
}
// If we're at the end of the segment, don't read the current block size,
// it's 0.
if l.pos < l.size-footerSize {
currentSize, err := l.readUint64()
if err != nil {
return err
}
l.currentSize = int64(currentSize)
}
return nil
}
// append adds byte slice to the end of segment
func (l *segment) append(b []byte) error {
l.mu.Lock()
defer l.mu.Unlock()
if l.file == nil {
return ErrNotOpen
}
if l.size+int64(len(b)) > l.maxSize {
return ErrSegmentFull
}
if err := l.seekEnd(-footerSize); err != nil {
return err
}
if err := l.writeUint64(uint64(len(b))); err != nil {
return err
}
if err := l.writeBytes(b); err != nil {
return err
}
if err := l.writeUint64(uint64(l.pos)); err != nil {
return err
}
if err := l.file.Sync(); err != nil {
return err
}
if l.currentSize == 0 {
l.currentSize = int64(len(b))
}
l.size += int64(len(b)) + 8 // uint64 for slice length
return nil
}
// current returns byte slice that the current segment points
func (l *segment) current() ([]byte, error) {
l.mu.Lock()
defer l.mu.Unlock()
if int64(l.pos) == l.size-8 {
return nil, io.EOF
}
if err := l.seekToCurrent(); err != nil {
return nil, err
}
// read the record size
sz, err := l.readUint64()
if err != nil {
return nil, err
}
l.currentSize = int64(sz)
if int64(sz) > l.maxSize {
return nil, fmt.Errorf("record size out of range: max %d: got %d", l.maxSize, sz)
}
b := make([]byte, sz)
if err := l.readBytes(b); err != nil {
return nil, err
}
return b, nil
}
// advance advances the current value pointer
func (l *segment) advance() error {
l.mu.Lock()
defer l.mu.Unlock()
if l.file == nil {
return ErrNotOpen
}
// If we're at the end of the file, can't advance
if int64(l.pos) == l.size-footerSize {
l.currentSize = 0
return io.EOF
}
if err := l.seekEnd(-footerSize); err != nil {
return err
}
pos := l.pos + l.currentSize + 8
if err := l.writeUint64(uint64(pos)); err != nil {
return err
}
if err := l.file.Sync(); err != nil {
return err
}
l.pos = pos
if err := l.seekToCurrent(); err != nil {
return err
}
sz, err := l.readUint64()
if err != nil {
return err
}
l.currentSize = int64(sz)
if int64(l.pos) == l.size-footerSize {
l.currentSize = 0
return io.EOF
}
return nil
}
func (l *segment) close() error {
l.mu.Lock()
defer l.mu.Unlock()
if err := l.file.Close(); err != nil {
return err
}
l.file = nil
return nil
}
func (l *segment) lastModified() (time.Time, error) {
l.mu.RLock()
defer l.mu.RUnlock()
stats, err := os.Stat(l.file.Name())
if err != nil {
return time.Time{}, err
}
return stats.ModTime(), nil
}
func (l *segment) diskUsage() int64 {
l.mu.RLock()
defer l.mu.RUnlock()
return l.size
}
func (l *segment) SetMaxSegmentSize(size int64) {
l.mu.Lock()
defer l.mu.Unlock()
l.maxSize = size
}
func (l *segment) seekToCurrent() error {
return l.seek(int64(l.pos))
}
func (l *segment) seek(pos int64) error {
n, err := l.file.Seek(pos, os.SEEK_SET)
if err != nil {
return err
}
if n != pos {
return fmt.Errorf("bad seek. exp %v, got %v", 0, n)
}
return nil
}
func (l *segment) seekEnd(pos int64) error {
_, err := l.file.Seek(pos, os.SEEK_END)
if err != nil {
return err
}
return nil
}
func (l *segment) filePos() int64 {
n, _ := l.file.Seek(0, os.SEEK_CUR)
return n
}
func (l *segment) readUint64() (uint64, error) {
b := make([]byte, 8)
if err := l.readBytes(b); err != nil {
return 0, err
}
return btou64(b), nil
}
func (l *segment) writeUint64(sz uint64) error {
return l.writeBytes(u64tob(sz))
}
func (l *segment) writeBytes(b []byte) error {
n, err := l.file.Write(b)
if err != nil {
return err
}
if n != len(b) {
return fmt.Errorf("short write. got %d, exp %d", n, len(b))
}
return nil
}
func (l *segment) readBytes(b []byte) error {
n, err := l.file.Read(b)
if err != nil {
return err
}
if n != len(b) {
return fmt.Errorf("bad read. exp %v, got %v", 0, n)
}
return nil
}
func u64tob(v uint64) []byte {
b := make([]byte, 8)
binary.BigEndian.PutUint64(b, v)
return b
}
func btou64(b []byte) uint64 {
return binary.BigEndian.Uint64(b)
}

327
Godeps/_workspace/src/github.com/influxdb/influxdb/services/hh/queue_test.go generated vendored
View File

@ -1,327 +0,0 @@
package hh
import (
"fmt"
"io"
"io/ioutil"
"os"
"path/filepath"
"testing"
"time"
)
func BenchmarkQueueAppend(b *testing.B) {
dir, err := ioutil.TempDir("", "hh_queue")
if err != nil {
b.Fatalf("failed to create temp dir: %v", err)
}
defer os.RemoveAll(dir)
q, err := newQueue(dir, 1024*1024*1024)
if err != nil {
b.Fatalf("failed to create queue: %v", err)
}
if err := q.Open(); err != nil {
b.Fatalf("failed to open queue: %v", err)
}
for i := 0; i < b.N; i++ {
if err := q.Append([]byte(fmt.Sprintf("%d", i))); err != nil {
println(q.diskUsage())
b.Fatalf("Queue.Append failed: %v", err)
}
}
}
func TestQueueAppendOne(t *testing.T) {
dir, err := ioutil.TempDir("", "hh_queue")
if err != nil {
t.Fatalf("failed to create temp dir: %v", err)
}
defer os.RemoveAll(dir)
q, err := newQueue(dir, 1024)
if err != nil {
t.Fatalf("failed to create queue: %v", err)
}
if err := q.Open(); err != nil {
t.Fatalf("failed to open queue: %v", err)
}
if err := q.Append([]byte("test")); err != nil {
t.Fatalf("Queue.Append failed: %v", err)
}
exp := filepath.Join(dir, "1")
stats, err := os.Stat(exp)
if os.IsNotExist(err) {
t.Fatalf("Queue.Append file not exists. exp %v to exist", exp)
}
// 8 byte header ptr + 8 byte record len + record len
if exp := int64(8 + 8 + 4); stats.Size() != exp {
t.Fatalf("Queue.Append file size mismatch. got %v, exp %v", stats.Size(), exp)
}
cur, err := q.Current()
if err != nil {
t.Fatalf("Queue.Current failed: %v", err)
}
if exp := "test"; string(cur) != exp {
t.Errorf("Queue.Current mismatch: got %v, exp %v", string(cur), exp)
}
}
func TestQueueAppendMultiple(t *testing.T) {
dir, err := ioutil.TempDir("", "hh_queue")
if err != nil {
t.Fatalf("failed to create temp dir: %v", err)
}
defer os.RemoveAll(dir)
q, err := newQueue(dir, 1024)
if err != nil {
t.Fatalf("failed to create queue: %v", err)
}
if err := q.Open(); err != nil {
t.Fatalf("failed to open queue: %v", err)
}
if err := q.Append([]byte("one")); err != nil {
t.Fatalf("Queue.Append failed: %v", err)
}
if err := q.Append([]byte("two")); err != nil {
t.Fatalf("Queue.Append failed: %v", err)
}
for _, exp := range []string{"one", "two"} {
cur, err := q.Current()
if err != nil {
t.Fatalf("Queue.Current failed: %v", err)
}
if string(cur) != exp {
t.Errorf("Queue.Current mismatch: got %v, exp %v", string(cur), exp)
}
if err := q.Advance(); err != nil {
t.Fatalf("Queue.Advance failed: %v", err)
}
}
}
func TestQueueAdvancePastEnd(t *testing.T) {
dir, err := ioutil.TempDir("", "hh_queue")
if err != nil {
t.Fatalf("failed to create temp dir: %v", err)
}
defer os.RemoveAll(dir)
// create the queue
q, err := newQueue(dir, 1024)
if err != nil {
t.Fatalf("failed to create queue: %v", err)
}
if err := q.Open(); err != nil {
t.Fatalf("failed to open queue: %v", err)
}
// append one entry, should go to the first segment
if err := q.Append([]byte("one")); err != nil {
t.Fatalf("Queue.Append failed: %v", err)
}
// set the segment size low to force a new segment to be created
q.SetMaxSegmentSize(12)
// Should go into a new segment
if err := q.Append([]byte("two")); err != nil {
t.Fatalf("Queue.Append failed: %v", err)
}
// should read from first segment
cur, err := q.Current()
if err != nil {
t.Fatalf("Queue.Current failed: %v", err)
}
if exp := "one"; string(cur) != exp {
t.Errorf("Queue.Current mismatch: got %v, exp %v", string(cur), exp)
}
if err := q.Advance(); err != nil {
t.Fatalf("Queue.Advance failed: %v", err)
}
// ensure the first segment file is removed since we've advanced past the end
_, err = os.Stat(filepath.Join(dir, "1"))
if !os.IsNotExist(err) {
t.Fatalf("Queue.Advance should have removed the segment")
}
// should read from second segment
cur, err = q.Current()
if err != nil {
t.Fatalf("Queue.Current failed: %v", err)
}
if exp := "two"; string(cur) != exp {
t.Errorf("Queue.Current mismatch: got %v, exp %v", string(cur), exp)
}
_, err = os.Stat(filepath.Join(dir, "2"))
if os.IsNotExist(err) {
t.Fatalf("Queue.Advance should have removed the segment")
}
if err := q.Advance(); err != nil {
t.Fatalf("Queue.Advance failed: %v", err)
}
cur, err = q.Current()
if err != io.EOF {
t.Fatalf("Queue.Current should have returned error")
}
}
func TestQueueFull(t *testing.T) {
dir, err := ioutil.TempDir("", "hh_queue")
if err != nil {
t.Fatalf("failed to create temp dir: %v", err)
}
defer os.RemoveAll(dir)
// create the queue
q, err := newQueue(dir, 10)
if err != nil {
t.Fatalf("failed to create queue: %v", err)
}
if err := q.Open(); err != nil {
t.Fatalf("failed to open queue: %v", err)
}
if err := q.Append([]byte("one")); err != ErrQueueFull {
t.Fatalf("Queue.Append expected to return queue full")
}
}
func TestQueueReopen(t *testing.T) {
dir, err := ioutil.TempDir("", "hh_queue")
if err != nil {
t.Fatalf("failed to create temp dir: %v", err)
}
defer os.RemoveAll(dir)
// create the queue
q, err := newQueue(dir, 1024)
if err != nil {
t.Fatalf("failed to create queue: %v", err)
}
if err := q.Open(); err != nil {
t.Fatalf("failed to open queue: %v", err)
}
if err := q.Append([]byte("one")); err != nil {
t.Fatalf("Queue.Append failed: %v", err)
}
cur, err := q.Current()
if err != nil {
t.Fatalf("Queue.Current failed: %v", err)
}
if exp := "one"; string(cur) != exp {
t.Errorf("Queue.Current mismatch: got %v, exp %v", string(cur), exp)
}
// close and re-open the queue
if err := q.Close(); err != nil {
t.Fatalf("Queue.Close failed: %v", err)
}
if err := q.Open(); err != nil {
t.Fatalf("failed to re-open queue: %v", err)
}
// Make sure we can read back the last current value
cur, err = q.Current()
if err != nil {
t.Fatalf("Queue.Current failed: %v", err)
}
if exp := "one"; string(cur) != exp {
t.Errorf("Queue.Current mismatch: got %v, exp %v", string(cur), exp)
}
if err := q.Append([]byte("two")); err != nil {
t.Fatalf("Queue.Append failed: %v", err)
}
if err := q.Advance(); err != nil {
t.Fatalf("Queue.Advance failed: %v", err)
}
cur, err = q.Current()
if err != nil {
t.Fatalf("Queue.Current failed: %v", err)
}
if exp := "two"; string(cur) != exp {
t.Errorf("Queue.Current mismatch: got %v, exp %v", string(cur), exp)
}
}
func TestPurgeQueue(t *testing.T) {
if testing.Short() {
t.Skip("Skipping purge queue")
}
dir, err := ioutil.TempDir("", "hh_queue")
if err != nil {
t.Fatalf("failed to create temp dir: %v", err)
}
defer os.RemoveAll(dir)
// create the queue
q, err := newQueue(dir, 1024)
if err != nil {
t.Fatalf("failed to create queue: %v", err)
}
if err := q.Open(); err != nil {
t.Fatalf("failed to open queue: %v", err)
}
if err := q.Append([]byte("one")); err != nil {
t.Fatalf("Queue.Append failed: %v", err)
}
cur, err := q.Current()
if err != nil {
t.Fatalf("Queue.Current failed: %v", err)
}
if exp := "one"; string(cur) != exp {
t.Errorf("Queue.Current mismatch: got %v, exp %v", string(cur), exp)
}
time.Sleep(time.Second)
if err := q.PurgeOlderThan(time.Now()); err != nil {
t.Errorf("Queue.PurgeOlderThan failed: %v", err)
}
_, err = q.Current()
if err != io.EOF {
t.Fatalf("Queue.Current expected io.EOF, got: %v", err)
}
}

136
Godeps/_workspace/src/github.com/influxdb/influxdb/services/hh/service.go generated vendored
View File

@ -1,136 +0,0 @@
package hh
import (
"fmt"
"io"
"log"
"os"
"sync"
"time"
"github.com/influxdb/influxdb/tsdb"
)
var ErrHintedHandoffDisabled = fmt.Errorf("hinted handoff disabled")
type Service struct {
mu sync.RWMutex
wg sync.WaitGroup
closing chan struct{}
Logger *log.Logger
cfg Config
ShardWriter shardWriter
HintedHandoff interface {
WriteShard(shardID, ownerID uint64, points []tsdb.Point) error
Process() error
PurgeOlderThan(when time.Duration) error
}
}
type shardWriter interface {
WriteShard(shardID, ownerID uint64, points []tsdb.Point) error
}
// NewService returns a new instance of Service.
func NewService(c Config, w shardWriter) *Service {
s := &Service{
cfg: c,
Logger: log.New(os.Stderr, "[handoff] ", log.LstdFlags),
}
processor, err := NewProcessor(c.Dir, w, ProcessorOptions{
MaxSize: c.MaxSize,
RetryRateLimit: c.RetryRateLimit,
})
if err != nil {
s.Logger.Fatalf("Failed to start hinted handoff processor: %v", err)
}
processor.Logger = s.Logger
s.HintedHandoff = processor
return s
}
func (s *Service) Open() error {
s.Logger.Printf("Starting hinted handoff service")
s.mu.Lock()
defer s.mu.Unlock()
s.closing = make(chan struct{})
s.Logger.Printf("Using data dir: %v", s.cfg.Dir)
s.wg.Add(2)
go s.retryWrites()
go s.expireWrites()
return nil
}
func (s *Service) Close() error {
s.mu.Lock()
defer s.mu.Unlock()
if s.closing != nil {
close(s.closing)
}
s.wg.Wait()
return nil
}
// SetLogger sets the internal logger to the logger passed in.
func (s *Service) SetLogger(l *log.Logger) {
s.Logger = l
}
// WriteShard queues the points write for shardID to node ownerID to handoff queue
func (s *Service) WriteShard(shardID, ownerID uint64, points []tsdb.Point) error {
if !s.cfg.Enabled {
return ErrHintedHandoffDisabled
}
return s.HintedHandoff.WriteShard(shardID, ownerID, points)
}
func (s *Service) retryWrites() {
defer s.wg.Done()
ticker := time.NewTicker(time.Duration(s.cfg.RetryInterval))
defer ticker.Stop()
for {
select {
case <-s.closing:
return
case <-ticker.C:
if err := s.HintedHandoff.Process(); err != nil && err != io.EOF {
s.Logger.Printf("retried write failed: %v", err)
}
}
}
}
// expireWrites will cause the handoff queues to remove writes that are older
// than the configured threshold
func (s *Service) expireWrites() {
defer s.wg.Done()
ticker := time.NewTicker(time.Hour)
defer ticker.Stop()
for {
select {
case <-s.closing:
return
case <-ticker.C:
if err := s.HintedHandoff.PurgeOlderThan(time.Duration(s.cfg.MaxAge)); err != nil {
s.Logger.Printf("purge write failed: %v", err)
}
}
}
}
// purgeWrites will cause the handoff queues to remove writes that are no longer
// valid. e.g. queued writes for a node that has been removed
func (s *Service) purgeWrites() {
panic("not implemented")
}

22
Godeps/_workspace/src/github.com/influxdb/influxdb/services/httpd/config.go generated vendored
View File

@ -1,22 +0,0 @@
package httpd
type Config struct {
Enabled bool `toml:"enabled"`
BindAddress string `toml:"bind-address"`
AuthEnabled bool `toml:"auth-enabled"`
LogEnabled bool `toml:"log-enabled"`
WriteTracing bool `toml:"write-tracing"`
PprofEnabled bool `toml:"pprof-enabled"`
HttpsEnabled bool `toml:"https-enabled"`
HttpsCertificate string `toml:"https-certificate"`
}
func NewConfig() Config {
return Config{
Enabled: true,
BindAddress: ":8086",
LogEnabled: true,
HttpsEnabled: false,
HttpsCertificate: "/etc/ssl/influxdb.pem",
}
}

52
Godeps/_workspace/src/github.com/influxdb/influxdb/services/httpd/config_test.go generated vendored
View File

@ -1,52 +0,0 @@
package httpd_test
import (
"testing"
"github.com/BurntSushi/toml"
"github.com/influxdb/influxdb/services/httpd"
)
func TestConfig_Parse(t *testing.T) {
// Parse configuration.
var c httpd.Config
if _, err := toml.Decode(`
enabled = true
bind-address = ":8080"
auth-enabled = true
log-enabled = true
write-tracing = true
pprof-enabled = true
https-enabled = true
https-certificate = "/dev/null"
`, &c); err != nil {
t.Fatal(err)
}
// Validate configuration.
if c.Enabled != true {
t.Fatalf("unexpected enabled: %v", c.Enabled)
} else if c.BindAddress != ":8080" {
t.Fatalf("unexpected bind address: %s", c.BindAddress)
} else if c.AuthEnabled != true {
t.Fatalf("unexpected auth enabled: %v", c.AuthEnabled)
} else if c.LogEnabled != true {
t.Fatalf("unexpected log enabled: %v", c.LogEnabled)
} else if c.WriteTracing != true {
t.Fatalf("unexpected write tracing: %v", c.WriteTracing)
} else if c.PprofEnabled != true {
t.Fatalf("unexpected pprof enabled: %v", c.PprofEnabled)
} else if c.HttpsEnabled != true {
t.Fatalf("unexpected https enabled: %v", c.HttpsEnabled)
} else if c.HttpsCertificate != "/dev/null" {
t.Fatalf("unexpected https certificate: %v", c.HttpsCertificate)
}
}
func TestConfig_WriteTracing(t *testing.T) {
c := httpd.Config{WriteTracing: true}
s := httpd.NewService(c)
if !s.Handler.WriteTrace {
t.Fatalf("write tracing was not set")
}
}

853
Godeps/_workspace/src/github.com/influxdb/influxdb/services/httpd/handler.go generated vendored
View File

@ -1,853 +0,0 @@
package httpd
import (
"bytes"
"compress/gzip"
"encoding/json"
"errors"
"fmt"
"io"
"io/ioutil"
"log"
"net/http"
"net/http/pprof"
"os"
"strconv"
"strings"
"time"
"github.com/bmizerany/pat"
"github.com/influxdb/influxdb"
"github.com/influxdb/influxdb/client"
"github.com/influxdb/influxdb/cluster"
"github.com/influxdb/influxdb/influxql"
"github.com/influxdb/influxdb/meta"
"github.com/influxdb/influxdb/services/continuous_querier"
"github.com/influxdb/influxdb/tsdb"
"github.com/influxdb/influxdb/uuid"
)
const (
// With raw data queries, mappers will read up to this amount before sending results back to the engine.
// This is the default size in the number of values returned in a raw query. Could be many more bytes depending on fields returned.
DefaultChunkSize = 10000
)
// TODO: Standard response headers (see: HeaderHandler)
// TODO: Compression (see: CompressionHeaderHandler)
// TODO: Check HTTP response codes: 400, 401, 403, 409.
type route struct {
name string
method string
pattern string
gzipped bool
log bool
handlerFunc interface{}
}
// Handler represents an HTTP handler for the InfluxDB server.
type Handler struct {
mux *pat.PatternServeMux
requireAuthentication bool
Version string
MetaStore interface {
Database(name string) (*meta.DatabaseInfo, error)
Authenticate(username, password string) (ui *meta.UserInfo, err error)
Users() ([]meta.UserInfo, error)
}
QueryExecutor interface {
Authorize(u *meta.UserInfo, q *influxql.Query, db string) error
ExecuteQuery(q *influxql.Query, db string, chunkSize int) (<-chan *influxql.Result, error)
}
PointsWriter interface {
WritePoints(p *cluster.WritePointsRequest) error
}
ContinuousQuerier continuous_querier.ContinuousQuerier
Logger *log.Logger
loggingEnabled bool // Log every HTTP access.
WriteTrace bool // Detailed logging of write path
}
// NewHandler returns a new instance of handler with routes.
func NewHandler(requireAuthentication, loggingEnabled, writeTrace bool) *Handler {
h := &Handler{
mux: pat.New(),
requireAuthentication: requireAuthentication,
Logger: log.New(os.Stderr, "[http] ", log.LstdFlags),
loggingEnabled: loggingEnabled,
WriteTrace: writeTrace,
}
h.SetRoutes([]route{
route{
"query", // Satisfy CORS checks.
"OPTIONS", "/query", true, true, h.serveOptions,
},
route{
"query", // Query serving route.
"GET", "/query", true, true, h.serveQuery,
},
route{
"write", // Satisfy CORS checks.
"OPTIONS", "/write", true, true, h.serveOptions,
},
route{
"write", // Data-ingest route.
"POST", "/write", true, true, h.serveWrite,
},
route{ // Ping
"ping",
"GET", "/ping", true, true, h.servePing,
},
route{ // Ping
"ping-head",
"HEAD", "/ping", true, true, h.servePing,
},
route{ // Tell data node to run CQs that should be run
"process_continuous_queries",
"POST", "/data/process_continuous_queries", false, false, h.serveProcessContinuousQueries,
},
})
return h
}
func (h *Handler) SetRoutes(routes []route) {
for _, r := range routes {
var handler http.Handler
// If it's a handler func that requires authorization, wrap it in authorization
if hf, ok := r.handlerFunc.(func(http.ResponseWriter, *http.Request, *meta.UserInfo)); ok {
handler = authenticate(hf, h, h.requireAuthentication)
}
// This is a normal handler signature and does not require authorization
if hf, ok := r.handlerFunc.(func(http.ResponseWriter, *http.Request)); ok {
handler = http.HandlerFunc(hf)
}
if r.gzipped {
handler = gzipFilter(handler)
}
handler = versionHeader(handler, h)
handler = cors(handler)
handler = requestID(handler)
if h.loggingEnabled && r.log {
handler = logging(handler, r.name, h.Logger)
}
handler = recovery(handler, r.name, h.Logger) // make sure recovery is always last
h.mux.Add(r.method, r.pattern, handler)
}
}
// ServeHTTP responds to HTTP request to the handler.
func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
// FIXME(benbjohnson): Add pprof enabled flag.
if strings.HasPrefix(r.URL.Path, "/debug/pprof") {
switch r.URL.Path {
case "/debug/pprof/cmdline":
pprof.Cmdline(w, r)
case "/debug/pprof/profile":
pprof.Profile(w, r)
case "/debug/pprof/symbol":
pprof.Symbol(w, r)
default:
pprof.Index(w, r)
}
return
}
h.mux.ServeHTTP(w, r)
}
func (h *Handler) serveProcessContinuousQueries(w http.ResponseWriter, r *http.Request, user *meta.UserInfo) {
// If the continuous query service isn't configured, return 404.
if h.ContinuousQuerier == nil {
w.WriteHeader(http.StatusNotImplemented)
return
}
q := r.URL.Query()
// Get the database name (blank means all databases).
db := q.Get("db")
// Get the name of the CQ to run (blank means run all).
name := q.Get("name")
// Pass the request to the CQ service.
if err := h.ContinuousQuerier.Run(db, name); err != nil {
w.WriteHeader(http.StatusBadRequest)
return
}
w.WriteHeader(http.StatusNoContent)
}
// serveQuery parses an incoming query and, if valid, executes the query.
func (h *Handler) serveQuery(w http.ResponseWriter, r *http.Request, user *meta.UserInfo) {
q := r.URL.Query()
pretty := q.Get("pretty") == "true"
qp := strings.TrimSpace(q.Get("q"))
if qp == "" {
httpError(w, `missing required parameter "q"`, pretty, http.StatusBadRequest)
return
}
epoch := strings.TrimSpace(q.Get("epoch"))
p := influxql.NewParser(strings.NewReader(qp))
db := q.Get("db")
// Parse query from query string.
query, err := p.ParseQuery()
if err != nil {
httpError(w, "error parsing query: "+err.Error(), pretty, http.StatusBadRequest)
return
}
// Sanitize statements with passwords.
for _, s := range query.Statements {
switch stmt := s.(type) {
case *influxql.CreateUserStatement:
sanitize(r, stmt.Password)
case *influxql.SetPasswordUserStatement:
sanitize(r, stmt.Password)
}
}
// Check authorization.
if h.requireAuthentication {
err = h.QueryExecutor.Authorize(user, query, db)
if err != nil {
httpError(w, "error authorizing query: "+err.Error(), pretty, http.StatusUnauthorized)
return
}
}
// Parse chunk size. Use default if not provided or unparsable.
chunked := (q.Get("chunked") == "true")
chunkSize := DefaultChunkSize
if chunked {
if n, err := strconv.ParseInt(q.Get("chunk_size"), 10, 64); err == nil {
chunkSize = int(n)
}
}
// Execute query.
w.Header().Add("content-type", "application/json")
results, err := h.QueryExecutor.ExecuteQuery(query, db, chunkSize)
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
return
}
// if we're not chunking, this will be the in memory buffer for all results before sending to client
resp := Response{Results: make([]*influxql.Result, 0)}
// Status header is OK once this point is reached.
w.WriteHeader(http.StatusOK)
// pull all results from the channel
for r := range results {
// Ignore nil results.
if r == nil {
continue
}
// if requested, convert result timestamps to epoch
if epoch != "" {
convertToEpoch(r, epoch)
}
// Write out result immediately if chunked.
if chunked {
w.Write(MarshalJSON(Response{
Results: []*influxql.Result{r},
}, pretty))
w.(http.Flusher).Flush()
continue
}
// It's not chunked so buffer results in memory.
// Results for statements need to be combined together.
// We need to check if this new result is for the same statement as
// the last result, or for the next statement
l := len(resp.Results)
if l == 0 {
resp.Results = append(resp.Results, r)
} else if resp.Results[l-1].StatementID == r.StatementID {
cr := resp.Results[l-1]
lastSeries := cr.Series[len(cr.Series)-1]
rowsMerged := 0
for _, row := range r.Series {
if !lastSeries.SameSeries(row) {
// Next row is for a different series than last.
break
}
// Values are for the same series, so append them.
lastSeries.Values = append(lastSeries.Values, row.Values...)
rowsMerged++
}
// Append remaining rows as new rows.
r.Series = r.Series[rowsMerged:]
cr.Series = append(cr.Series, r.Series...)
} else {
resp.Results = append(resp.Results, r)
}
}
// If it's not chunked we buffered everything in memory, so write it out
if !chunked {
w.Write(MarshalJSON(resp, pretty))
}
}
func (h *Handler) serveWrite(w http.ResponseWriter, r *http.Request, user *meta.UserInfo) {
// Handle gzip decoding of the body
body := r.Body
if r.Header.Get("Content-encoding") == "gzip" {
b, err := gzip.NewReader(r.Body)
if err != nil {
h.writeError(w, influxql.Result{Err: err}, http.StatusBadRequest)
return
}
body = b
}
defer body.Close()
b, err := ioutil.ReadAll(body)
if err != nil {
if h.WriteTrace {
h.Logger.Print("write handler unable to read bytes from request body")
}
h.writeError(w, influxql.Result{Err: err}, http.StatusBadRequest)
return
}
if h.WriteTrace {
h.Logger.Printf("write body received by handler: %s", string(b))
}
if r.Header.Get("Content-Type") == "application/json" {
h.serveWriteJSON(w, r, b, user)
return
}
h.serveWriteLine(w, r, b, user)
}
// serveWriteJSON receives incoming series data in JSON and writes it to the database.
func (h *Handler) serveWriteJSON(w http.ResponseWriter, r *http.Request, body []byte, user *meta.UserInfo) {
var bp client.BatchPoints
var dec *json.Decoder
dec = json.NewDecoder(bytes.NewReader(body))
if err := dec.Decode(&bp); err != nil {
if err.Error() == "EOF" {
w.WriteHeader(http.StatusOK)
return
}
resultError(w, influxql.Result{Err: err}, http.StatusBadRequest)
return
}
if bp.Database == "" {
resultError(w, influxql.Result{Err: fmt.Errorf("database is required")}, http.StatusBadRequest)
return
}
if di, err := h.MetaStore.Database(bp.Database); err != nil {
resultError(w, influxql.Result{Err: fmt.Errorf("metastore database error: %s", err)}, http.StatusInternalServerError)
return
} else if di == nil {
resultError(w, influxql.Result{Err: fmt.Errorf("database not found: %q", bp.Database)}, http.StatusNotFound)
return
}
if h.requireAuthentication && user == nil {
resultError(w, influxql.Result{Err: fmt.Errorf("user is required to write to database %q", bp.Database)}, http.StatusUnauthorized)
return
}
if h.requireAuthentication && !user.Authorize(influxql.WritePrivilege, bp.Database) {
resultError(w, influxql.Result{Err: fmt.Errorf("%q user is not authorized to write to database %q", user.Name, bp.Database)}, http.StatusUnauthorized)
return
}
points, err := NormalizeBatchPoints(bp)
if err != nil {
resultError(w, influxql.Result{Err: err}, http.StatusBadRequest)
return
}
// Convert the json batch struct to a points writer struct
if err := h.PointsWriter.WritePoints(&cluster.WritePointsRequest{
Database: bp.Database,
RetentionPolicy: bp.RetentionPolicy,
ConsistencyLevel: cluster.ConsistencyLevelOne,
Points: points,
}); influxdb.IsClientError(err) {
resultError(w, influxql.Result{Err: err}, http.StatusBadRequest)
return
} else if err != nil {
resultError(w, influxql.Result{Err: err}, http.StatusInternalServerError)
return
}
w.WriteHeader(http.StatusNoContent)
}
func (h *Handler) writeError(w http.ResponseWriter, result influxql.Result, statusCode int) {
w.WriteHeader(statusCode)
w.Write([]byte(result.Err.Error()))
w.Write([]byte("\n"))
}
// serveWriteLine receives incoming series data in line protocol format and writes it to the database.
func (h *Handler) serveWriteLine(w http.ResponseWriter, r *http.Request, body []byte, user *meta.UserInfo) {
// Some clients may not set the content-type header appropriately and send JSON with a non-json
// content-type. If the body looks JSON, try to handle it as as JSON instead
if len(body) > 0 {
var i int
for {
// JSON requests must start w/ an opening bracket
if body[i] == '{' {
h.serveWriteJSON(w, r, body, user)
return
}
// check that the byte is in the standard ascii code range
if body[i] > 32 {
break
}
i += 1
}
}
precision := r.FormValue("precision")
if precision == "" {
precision = "n"
}
points, err := tsdb.ParsePointsWithPrecision(body, time.Now().UTC(), precision)
if err != nil {
if err.Error() == "EOF" {
w.WriteHeader(http.StatusOK)
return
}
h.writeError(w, influxql.Result{Err: err}, http.StatusBadRequest)
return
}
database := r.FormValue("db")
if database == "" {
h.writeError(w, influxql.Result{Err: fmt.Errorf("database is required")}, http.StatusBadRequest)
return
}
if di, err := h.MetaStore.Database(database); err != nil {
h.writeError(w, influxql.Result{Err: fmt.Errorf("metastore database error: %s", err)}, http.StatusInternalServerError)
return
} else if di == nil {
h.writeError(w, influxql.Result{Err: fmt.Errorf("database not found: %q", database)}, http.StatusNotFound)
return
}
if h.requireAuthentication && user == nil {
h.writeError(w, influxql.Result{Err: fmt.Errorf("user is required to write to database %q", database)}, http.StatusUnauthorized)
return
}
if h.requireAuthentication && !user.Authorize(influxql.WritePrivilege, database) {
h.writeError(w, influxql.Result{Err: fmt.Errorf("%q user is not authorized to write to database %q", user.Name, database)}, http.StatusUnauthorized)
return
}
// Determine required consistency level.
consistency := cluster.ConsistencyLevelOne
switch r.Form.Get("consistency") {
case "all":
consistency = cluster.ConsistencyLevelAll
case "any":
consistency = cluster.ConsistencyLevelAny
case "one":
consistency = cluster.ConsistencyLevelOne
case "quorum":
consistency = cluster.ConsistencyLevelQuorum
}
// Write points.
if err := h.PointsWriter.WritePoints(&cluster.WritePointsRequest{
Database: database,
RetentionPolicy: r.FormValue("rp"),
ConsistencyLevel: consistency,
Points: points,
}); influxdb.IsClientError(err) {
h.writeError(w, influxql.Result{Err: err}, http.StatusBadRequest)
return
} else if err != nil {
h.writeError(w, influxql.Result{Err: err}, http.StatusInternalServerError)
return
}
w.WriteHeader(http.StatusNoContent)
}
// serveOptions returns an empty response to comply with OPTIONS pre-flight requests
func (h *Handler) serveOptions(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusNoContent)
}
// servePing returns a simple response to let the client know the server is running.
func (h *Handler) servePing(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusNoContent)
}
// convertToEpoch converts result timestamps from time.Time to the specified epoch.
func convertToEpoch(r *influxql.Result, epoch string) {
divisor := int64(1)
switch epoch {
case "u":
divisor = int64(time.Microsecond)
case "ms":
divisor = int64(time.Millisecond)
case "s":
divisor = int64(time.Second)
case "m":
divisor = int64(time.Minute)
case "h":
divisor = int64(time.Hour)
}
for _, s := range r.Series {
for _, v := range s.Values {
if ts, ok := v[0].(time.Time); ok {
v[0] = ts.UnixNano() / divisor
}
}
}
}
// MarshalJSON will marshal v to JSON. Pretty prints if pretty is true.
func MarshalJSON(v interface{}, pretty bool) []byte {
var b []byte
var err error
if pretty {
b, err = json.MarshalIndent(v, "", " ")
} else {
b, err = json.Marshal(v)
}
if err != nil {
return []byte(err.Error())
}
return b
}
type Point struct {
Name string `json:"name"`
Time time.Time `json:"time"`
Tags map[string]string `json:"tags"`
Fields map[string]interface{} `json:"fields"`
}
type Batch struct {
Database string `json:"database"`
RetentionPolicy string `json:"retentionPolicy"`
Points []Point `json:"points"`
}
// httpError writes an error to the client in a standard format.
func httpError(w http.ResponseWriter, error string, pretty bool, code int) {
w.Header().Add("content-type", "application/json")
w.WriteHeader(code)
response := Response{Err: errors.New(error)}
var b []byte
if pretty {
b, _ = json.MarshalIndent(response, "", " ")
} else {
b, _ = json.Marshal(response)
}
w.Write(b)
}
func resultError(w http.ResponseWriter, result influxql.Result, code int) {
w.Header().Add("content-type", "application/json")
w.WriteHeader(code)
_ = json.NewEncoder(w).Encode(&result)
}
// Filters and filter helpers
// parseCredentials returns the username and password encoded in
// a request. The credentials may be present as URL query params, or as
// a Basic Authentication header.
// as params: http://127.0.0.1/query?u=username&p=password
// as basic auth: http://username:password@127.0.0.1
func parseCredentials(r *http.Request) (string, string, error) {
q := r.URL.Query()
if u, p := q.Get("u"), q.Get("p"); u != "" && p != "" {
return u, p, nil
}
if u, p, ok := r.BasicAuth(); ok {
return u, p, nil
}
return "", "", fmt.Errorf("unable to parse Basic Auth credentials")
}
// authenticate wraps a handler and ensures that if user credentials are passed in
// an attempt is made to authenticate that user. If authentication fails, an error is returned.
//
// There is one exception: if there are no users in the system, authentication is not required. This
// is to facilitate bootstrapping of a system with authentication enabled.
func authenticate(inner func(http.ResponseWriter, *http.Request, *meta.UserInfo), h *Handler, requireAuthentication bool) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
// Return early if we are not authenticating
if !requireAuthentication {
inner(w, r, nil)
return
}
var user *meta.UserInfo
// Retrieve user list.
uis, err := h.MetaStore.Users()
if err != nil {
httpError(w, err.Error(), false, http.StatusInternalServerError)
return
}
// TODO corylanou: never allow this in the future without users
if requireAuthentication && len(uis) > 0 {
username, password, err := parseCredentials(r)
if err != nil {
httpError(w, err.Error(), false, http.StatusUnauthorized)
return
}
if username == "" {
httpError(w, "username required", false, http.StatusUnauthorized)
return
}
user, err = h.MetaStore.Authenticate(username, password)
if err != nil {
httpError(w, err.Error(), false, http.StatusUnauthorized)
return
}
}
inner(w, r, user)
})
}
type gzipResponseWriter struct {
io.Writer
http.ResponseWriter
}
func (w gzipResponseWriter) Write(b []byte) (int, error) {
return w.Writer.Write(b)
}
func (w gzipResponseWriter) Flush() {
w.Writer.(*gzip.Writer).Flush()
}
// determines if the client can accept compressed responses, and encodes accordingly
func gzipFilter(inner http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if !strings.Contains(r.Header.Get("Accept-Encoding"), "gzip") {
inner.ServeHTTP(w, r)
return
}
w.Header().Set("Content-Encoding", "gzip")
gz := gzip.NewWriter(w)
defer gz.Close()
gzw := gzipResponseWriter{Writer: gz, ResponseWriter: w}
inner.ServeHTTP(gzw, r)
})
}
// versionHeader takes a HTTP handler and returns a HTTP handler
// and adds the X-INFLUXBD-VERSION header to outgoing responses.
func versionHeader(inner http.Handler, h *Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Add("X-InfluxDB-Version", h.Version)
inner.ServeHTTP(w, r)
})
}
// cors responds to incoming requests and adds the appropriate cors headers
// TODO: corylanou: add the ability to configure this in our config
func cors(inner http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if origin := r.Header.Get("Origin"); origin != "" {
w.Header().Set(`Access-Control-Allow-Origin`, origin)
w.Header().Set(`Access-Control-Allow-Methods`, strings.Join([]string{
`DELETE`,
`GET`,
`OPTIONS`,
`POST`,
`PUT`,
}, ", "))
w.Header().Set(`Access-Control-Allow-Headers`, strings.Join([]string{
`Accept`,
`Accept-Encoding`,
`Authorization`,
`Content-Length`,
`Content-Type`,
`X-CSRF-Token`,
`X-HTTP-Method-Override`,
}, ", "))
}
if r.Method == "OPTIONS" {
return
}
inner.ServeHTTP(w, r)
})
}
func requestID(inner http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
uid := uuid.TimeUUID()
r.Header.Set("Request-Id", uid.String())
w.Header().Set("Request-Id", r.Header.Get("Request-Id"))
inner.ServeHTTP(w, r)
})
}
func logging(inner http.Handler, name string, weblog *log.Logger) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
start := time.Now()
l := &responseLogger{w: w}
inner.ServeHTTP(l, r)
logLine := buildLogLine(l, r, start)
weblog.Println(logLine)
})
}
func recovery(inner http.Handler, name string, weblog *log.Logger) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
start := time.Now()
l := &responseLogger{w: w}
inner.ServeHTTP(l, r)
if err := recover(); err != nil {
logLine := buildLogLine(l, r, start)
logLine = fmt.Sprintf(`%s [err:%s]`, logLine, err)
weblog.Println(logLine)
}
})
}
// Response represents a list of statement results.
type Response struct {
Results []*influxql.Result
Err error
}
// MarshalJSON encodes a Response struct into JSON.
func (r Response) MarshalJSON() ([]byte, error) {
// Define a struct that outputs "error" as a string.
var o struct {
Results []*influxql.Result `json:"results,omitempty"`
Err string `json:"error,omitempty"`
}
// Copy fields to output struct.
o.Results = r.Results
if r.Err != nil {
o.Err = r.Err.Error()
}
return json.Marshal(&o)
}
// UnmarshalJSON decodes the data into the Response struct
func (r *Response) UnmarshalJSON(b []byte) error {
var o struct {
Results []*influxql.Result `json:"results,omitempty"`
Err string `json:"error,omitempty"`
}
err := json.Unmarshal(b, &o)
if err != nil {
return err
}
r.Results = o.Results
if o.Err != "" {
r.Err = errors.New(o.Err)
}
return nil
}
// Error returns the first error from any statement.
// Returns nil if no errors occurred on any statements.
func (r *Response) Error() error {
if r.Err != nil {
return r.Err
}
for _, rr := range r.Results {
if rr.Err != nil {
return rr.Err
}
}
return nil
}
// NormalizeBatchPoints returns a slice of Points, created by populating individual
// points within the batch, which do not have times or tags, with the top-level
// values.
func NormalizeBatchPoints(bp client.BatchPoints) ([]tsdb.Point, error) {
points := []tsdb.Point{}
for _, p := range bp.Points {
if p.Time.IsZero() {
if bp.Time.IsZero() {
p.Time = time.Now()
} else {
p.Time = bp.Time
}
}
if p.Precision == "" && bp.Precision != "" {
p.Precision = bp.Precision
}
p.Time = client.SetPrecision(p.Time, p.Precision)
if len(bp.Tags) > 0 {
if p.Tags == nil {
p.Tags = make(map[string]string)
}
for k := range bp.Tags {
if p.Tags[k] == "" {
p.Tags[k] = bp.Tags[k]
}
}
}
if p.Measurement == "" {
return points, fmt.Errorf("missing measurement")
}
if len(p.Fields) == 0 {
return points, fmt.Errorf("missing fields")
}
// Need to convert from a client.Point to a influxdb.Point
points = append(points, tsdb.NewPoint(p.Measurement, p.Tags, p.Fields, p.Time))
}
return points, nil
}

448
Godeps/_workspace/src/github.com/influxdb/influxdb/services/httpd/handler_test.go generated vendored
View File

@ -1,448 +0,0 @@
package httpd_test
import (
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"net/http/httptest"
"reflect"
"regexp"
"testing"
"time"
"github.com/influxdb/influxdb/client"
"github.com/influxdb/influxdb/influxql"
"github.com/influxdb/influxdb/meta"
"github.com/influxdb/influxdb/services/httpd"
"github.com/influxdb/influxdb/tsdb"
)
func TestBatchWrite_UnmarshalEpoch(t *testing.T) {
now := time.Now().UTC()
tests := []struct {
name string
epoch int64
precision string
expected time.Time
}{
{
name: "nanoseconds",
epoch: now.UnixNano(),
precision: "n",
expected: now,
},
{
name: "microseconds",
epoch: now.Round(time.Microsecond).UnixNano() / int64(time.Microsecond),
precision: "u",
expected: now.Round(time.Microsecond),
},
{
name: "milliseconds",
epoch: now.Round(time.Millisecond).UnixNano() / int64(time.Millisecond),
precision: "ms",
expected: now.Round(time.Millisecond),
},
{
name: "seconds",
epoch: now.Round(time.Second).UnixNano() / int64(time.Second),
precision: "s",
expected: now.Round(time.Second),
},
{
name: "minutes",
epoch: now.Round(time.Minute).UnixNano() / int64(time.Minute),
precision: "m",
expected: now.Round(time.Minute),
},
{
name: "hours",
epoch: now.Round(time.Hour).UnixNano() / int64(time.Hour),
precision: "h",
expected: now.Round(time.Hour),
},
{
name: "max int64",
epoch: 9223372036854775807,
precision: "n",
expected: time.Unix(0, 9223372036854775807),
},
{
name: "100 years from now",
epoch: now.Add(time.Hour * 24 * 365 * 100).UnixNano(),
precision: "n",
expected: now.Add(time.Hour * 24 * 365 * 100),
},
}
for _, test := range tests {
t.Logf("testing %q\n", test.name)
data := []byte(fmt.Sprintf(`{"time": %d, "precision":"%s"}`, test.epoch, test.precision))
t.Logf("json: %s", string(data))
var bp client.BatchPoints
err := json.Unmarshal(data, &bp)
if err != nil {
t.Fatalf("unexpected error. expected: %v, actual: %v", nil, err)
}
if !bp.Time.Equal(test.expected) {
t.Fatalf("Unexpected time. expected: %v, actual: %v", test.expected, bp.Time)
}
}
}
func TestBatchWrite_UnmarshalRFC(t *testing.T) {
now := time.Now()
tests := []struct {
name string
rfc string
now time.Time
expected time.Time
}{
{
name: "RFC3339Nano",
rfc: time.RFC3339Nano,
now: now,
expected: now,
},
{
name: "RFC3339",
rfc: time.RFC3339,
now: now.Round(time.Second),
expected: now.Round(time.Second),
},
}
for _, test := range tests {
t.Logf("testing %q\n", test.name)
ts := test.now.Format(test.rfc)
data := []byte(fmt.Sprintf(`{"time": %q}`, ts))
t.Logf("json: %s", string(data))
var bp client.BatchPoints
err := json.Unmarshal(data, &bp)
if err != nil {
t.Fatalf("unexpected error. exptected: %v, actual: %v", nil, err)
}
if !bp.Time.Equal(test.expected) {
t.Fatalf("Unexpected time. expected: %v, actual: %v", test.expected, bp.Time)
}
}
}
// Ensure the handler returns results from a query (including nil results).
func TestHandler_Query(t *testing.T) {
h := NewHandler(false)
h.QueryExecutor.ExecuteQueryFn = func(q *influxql.Query, db string, chunkSize int) (<-chan *influxql.Result, error) {
if q.String() != `SELECT * FROM bar` {
t.Fatalf("unexpected query: %s", q.String())
} else if db != `foo` {
t.Fatalf("unexpected db: %s", db)
}
return NewResultChan(
&influxql.Result{StatementID: 1, Series: influxql.Rows{{Name: "series0"}}},
&influxql.Result{StatementID: 2, Series: influxql.Rows{{Name: "series1"}}},
nil,
), nil
}
w := httptest.NewRecorder()
h.ServeHTTP(w, MustNewJSONRequest("GET", "/query?db=foo&q=SELECT+*+FROM+bar", nil))
if w.Code != http.StatusOK {
t.Fatalf("unexpected status: %d", w.Code)
} else if w.Body.String() != `{"results":[{"series":[{"name":"series0"}]},{"series":[{"name":"series1"}]}]}` {
t.Fatalf("unexpected body: %s", w.Body.String())
}
}
// Ensure the handler merges results from the same statement.
func TestHandler_Query_MergeResults(t *testing.T) {
h := NewHandler(false)
h.QueryExecutor.ExecuteQueryFn = func(q *influxql.Query, db string, chunkSize int) (<-chan *influxql.Result, error) {
return NewResultChan(
&influxql.Result{StatementID: 1, Series: influxql.Rows{{Name: "series0"}}},
&influxql.Result{StatementID: 1, Series: influxql.Rows{{Name: "series1"}}},
), nil
}
w := httptest.NewRecorder()
h.ServeHTTP(w, MustNewJSONRequest("GET", "/query?db=foo&q=SELECT+*+FROM+bar", nil))
if w.Code != http.StatusOK {
t.Fatalf("unexpected status: %d", w.Code)
} else if w.Body.String() != `{"results":[{"series":[{"name":"series0"},{"name":"series1"}]}]}` {
t.Fatalf("unexpected body: %s", w.Body.String())
}
}
// Ensure the handler can parse chunked and chunk size query parameters.
func TestHandler_Query_Chunked(t *testing.T) {
h := NewHandler(false)
h.QueryExecutor.ExecuteQueryFn = func(q *influxql.Query, db string, chunkSize int) (<-chan *influxql.Result, error) {
if chunkSize != 2 {
t.Fatalf("unexpected chunk size: %d", chunkSize)
}
return NewResultChan(
&influxql.Result{StatementID: 1, Series: influxql.Rows{{Name: "series0"}}},
&influxql.Result{StatementID: 1, Series: influxql.Rows{{Name: "series1"}}},
), nil
}
w := httptest.NewRecorder()
h.ServeHTTP(w, MustNewJSONRequest("GET", "/query?db=foo&q=SELECT+*+FROM+bar&chunked=true&chunk_size=2", nil))
if w.Code != http.StatusOK {
t.Fatalf("unexpected status: %d", w.Code)
} else if w.Body.String() != `{"results":[{"series":[{"name":"series0"}]}]}{"results":[{"series":[{"name":"series1"}]}]}` {
t.Fatalf("unexpected body: %s", w.Body.String())
}
}
// Ensure the handler returns a status 400 if the query is not passed in.
func TestHandler_Query_ErrQueryRequired(t *testing.T) {
h := NewHandler(false)
w := httptest.NewRecorder()
h.ServeHTTP(w, MustNewJSONRequest("GET", "/query", nil))
if w.Code != http.StatusBadRequest {
t.Fatalf("unexpected status: %d", w.Code)
} else if w.Body.String() != `{"error":"missing required parameter \"q\""}` {
t.Fatalf("unexpected body: %s", w.Body.String())
}
}
// Ensure the handler returns a status 400 if the query cannot be parsed.
func TestHandler_Query_ErrInvalidQuery(t *testing.T) {
h := NewHandler(false)
w := httptest.NewRecorder()
h.ServeHTTP(w, MustNewJSONRequest("GET", "/query?q=SELECT", nil))
if w.Code != http.StatusBadRequest {
t.Fatalf("unexpected status: %d", w.Code)
} else if w.Body.String() != `{"error":"error parsing query: found EOF, expected identifier, string, number, bool at line 1, char 8"}` {
t.Fatalf("unexpected body: %s", w.Body.String())
}
}
// Ensure the handler returns a status 401 if the user is not authorized.
// func TestHandler_Query_ErrUnauthorized(t *testing.T) {
// h := NewHandler(false)
// h.QueryExecutor.AuthorizeFn = func(u *meta.UserInfo, q *influxql.Query, db string) error {
// return errors.New("marker")
// }
// w := httptest.NewRecorder()
// h.ServeHTTP(w, MustNewJSONRequest("GET", "/query?u=bar&db=foo&q=SHOW+SERIES+FROM+bar", nil))
// if w.Code != http.StatusUnauthorized {
// t.Fatalf("unexpected status: %d", w.Code)
// }
// }
// Ensure the handler returns a status 500 if an error is returned from the query executor.
func TestHandler_Query_ErrExecuteQuery(t *testing.T) {
h := NewHandler(false)
h.QueryExecutor.ExecuteQueryFn = func(q *influxql.Query, db string, chunkSize int) (<-chan *influxql.Result, error) {
return nil, errors.New("marker")
}
w := httptest.NewRecorder()
h.ServeHTTP(w, MustNewJSONRequest("GET", "/query?db=foo&q=SHOW+SERIES+FROM+bar", nil))
if w.Code != http.StatusInternalServerError {
t.Fatalf("unexpected status: %d", w.Code)
}
}
// Ensure the handler returns a status 200 if an error is returned in the result.
func TestHandler_Query_ErrResult(t *testing.T) {
h := NewHandler(false)
h.QueryExecutor.ExecuteQueryFn = func(q *influxql.Query, db string, chunkSize int) (<-chan *influxql.Result, error) {
return NewResultChan(&influxql.Result{Err: errors.New("measurement not found")}), nil
}
w := httptest.NewRecorder()
h.ServeHTTP(w, MustNewJSONRequest("GET", "/query?db=foo&q=SHOW+SERIES+from+bin", nil))
if w.Code != http.StatusOK {
t.Fatalf("unexpected status: %d", w.Code)
} else if w.Body.String() != `{"results":[{"error":"measurement not found"}]}` {
t.Fatalf("unexpected body: %s", w.Body.String())
}
}
func TestMarshalJSON_NoPretty(t *testing.T) {
if b := httpd.MarshalJSON(struct {
Name string `json:"name"`
}{Name: "foo"}, false); string(b) != `{"name":"foo"}` {
t.Fatalf("unexpected bytes: %s", b)
}
}
func TestMarshalJSON_Pretty(t *testing.T) {
if b := httpd.MarshalJSON(struct {
Name string `json:"name"`
}{Name: "foo"}, true); string(b) != "{\n \"name\": \"foo\"\n}" {
t.Fatalf("unexpected bytes: %q", string(b))
}
}
func TestMarshalJSON_Error(t *testing.T) {
if b := httpd.MarshalJSON(&invalidJSON{}, true); string(b) != "json: error calling MarshalJSON for type *httpd_test.invalidJSON: marker" {
t.Fatalf("unexpected bytes: %q", string(b))
}
}
type invalidJSON struct{}
func (*invalidJSON) MarshalJSON() ([]byte, error) { return nil, errors.New("marker") }
func TestNormalizeBatchPoints(t *testing.T) {
now := time.Now()
tests := []struct {
name string
bp client.BatchPoints
p []tsdb.Point
err string
}{
{
name: "default",
bp: client.BatchPoints{
Points: []client.Point{
{Measurement: "cpu", Tags: map[string]string{"region": "useast"}, Time: now, Fields: map[string]interface{}{"value": 1.0}},
},
},
p: []tsdb.Point{
tsdb.NewPoint("cpu", map[string]string{"region": "useast"}, map[string]interface{}{"value": 1.0}, now),
},
},
{
name: "merge time",
bp: client.BatchPoints{
Time: now,
Points: []client.Point{
{Measurement: "cpu", Tags: map[string]string{"region": "useast"}, Fields: map[string]interface{}{"value": 1.0}},
},
},
p: []tsdb.Point{
tsdb.NewPoint("cpu", map[string]string{"region": "useast"}, map[string]interface{}{"value": 1.0}, now),
},
},
{
name: "merge tags",
bp: client.BatchPoints{
Tags: map[string]string{"day": "monday"},
Points: []client.Point{
{Measurement: "cpu", Tags: map[string]string{"region": "useast"}, Time: now, Fields: map[string]interface{}{"value": 1.0}},
{Measurement: "memory", Time: now, Fields: map[string]interface{}{"value": 2.0}},
},
},
p: []tsdb.Point{
tsdb.NewPoint("cpu", map[string]string{"day": "monday", "region": "useast"}, map[string]interface{}{"value": 1.0}, now),
tsdb.NewPoint("memory", map[string]string{"day": "monday"}, map[string]interface{}{"value": 2.0}, now),
},
},
}
for _, test := range tests {
t.Logf("running test %q", test.name)
p, e := httpd.NormalizeBatchPoints(test.bp)
if test.err == "" && e != nil {
t.Errorf("unexpected error %v", e)
} else if test.err != "" && e == nil {
t.Errorf("expected error %s, got <nil>", test.err)
} else if e != nil && test.err != e.Error() {
t.Errorf("unexpected error. expected: %s, got %v", test.err, e)
}
if !reflect.DeepEqual(p, test.p) {
t.Logf("expected: %+v", test.p)
t.Logf("got: %+v", p)
t.Error("failed to normalize.")
}
}
}
// NewHandler represents a test wrapper for httpd.Handler.
type Handler struct {
*httpd.Handler
MetaStore HandlerMetaStore
QueryExecutor HandlerQueryExecutor
TSDBStore HandlerTSDBStore
}
// NewHandler returns a new instance of Handler.
func NewHandler(requireAuthentication bool) *Handler {
h := &Handler{
Handler: httpd.NewHandler(requireAuthentication, true, false),
}
h.Handler.MetaStore = &h.MetaStore
h.Handler.QueryExecutor = &h.QueryExecutor
h.Handler.Version = "0.0.0"
return h
}
// HandlerMetaStore is a mock implementation of Handler.MetaStore.
type HandlerMetaStore struct {
DatabaseFn func(name string) (*meta.DatabaseInfo, error)
AuthenticateFn func(username, password string) (ui *meta.UserInfo, err error)
UsersFn func() ([]meta.UserInfo, error)
}
func (s *HandlerMetaStore) Database(name string) (*meta.DatabaseInfo, error) {
return s.DatabaseFn(name)
}
func (s *HandlerMetaStore) Authenticate(username, password string) (ui *meta.UserInfo, err error) {
return s.AuthenticateFn(username, password)
}
func (s *HandlerMetaStore) Users() ([]meta.UserInfo, error) {
return s.UsersFn()
}
// HandlerQueryExecutor is a mock implementation of Handler.QueryExecutor.
type HandlerQueryExecutor struct {
AuthorizeFn func(u *meta.UserInfo, q *influxql.Query, db string) error
ExecuteQueryFn func(q *influxql.Query, db string, chunkSize int) (<-chan *influxql.Result, error)
}
func (e *HandlerQueryExecutor) Authorize(u *meta.UserInfo, q *influxql.Query, db string) error {
return e.AuthorizeFn(u, q, db)
}
func (e *HandlerQueryExecutor) ExecuteQuery(q *influxql.Query, db string, chunkSize int) (<-chan *influxql.Result, error) {
return e.ExecuteQueryFn(q, db, chunkSize)
}
// HandlerTSDBStore is a mock implementation of Handler.TSDBStore
type HandlerTSDBStore struct {
CreateMapperFn func(shardID uint64, query string, chunkSize int) (tsdb.Mapper, error)
}
func (h *HandlerTSDBStore) CreateMapper(shardID uint64, query string, chunkSize int) (tsdb.Mapper, error) {
return h.CreateMapperFn(shardID, query, chunkSize)
}
// MustNewRequest returns a new HTTP request. Panic on error.
func MustNewRequest(method, urlStr string, body io.Reader) *http.Request {
r, err := http.NewRequest(method, urlStr, body)
if err != nil {
panic(err.Error())
}
return r
}
// MustNewRequest returns a new HTTP request with the content type set. Panic on error.
func MustNewJSONRequest(method, urlStr string, body io.Reader) *http.Request {
r := MustNewRequest(method, urlStr, body)
r.Header.Set("Content-Type", "application/json")
return r
}
// matchRegex returns true if a s matches pattern.
func matchRegex(pattern, s string) bool {
return regexp.MustCompile(pattern).MatchString(s)
}
// NewResultChan returns a channel that sends all results and then closes.
func NewResultChan(results ...*influxql.Result) <-chan *influxql.Result {
ch := make(chan *influxql.Result, len(results))
for _, r := range results {
ch <- r
}
close(ch)
return ch
}

153
Godeps/_workspace/src/github.com/influxdb/influxdb/services/httpd/response_logger.go generated vendored
View File

@ -1,153 +0,0 @@
package httpd
import (
"fmt"
"net"
"net/http"
"strconv"
"strings"
"time"
)
type loggingResponseWriter interface {
http.ResponseWriter
Status() int
Size() int
}
// responseLogger is wrapper of http.ResponseWriter that keeps track of its HTTP status
// code and body size
type responseLogger struct {
w http.ResponseWriter
status int
size int
}
func (l *responseLogger) Header() http.Header {
return l.w.Header()
}
func (l *responseLogger) Flush() {
l.w.(http.Flusher).Flush()
}
func (l *responseLogger) Write(b []byte) (int, error) {
if l.status == 0 {
// Set status if WriteHeader has not been called
l.status = http.StatusOK
}
size, err := l.w.Write(b)
l.size += size
return size, err
}
func (l *responseLogger) WriteHeader(s int) {
l.w.WriteHeader(s)
l.status = s
}
func (l *responseLogger) Status() int {
if l.status == 0 {
// This can happen if we never actually write data, but only set response headers.
l.status = http.StatusOK
}
return l.status
}
func (l *responseLogger) Size() int {
return l.size
}
// redact any occurrence of a password parameter, 'p'
func redactPassword(r *http.Request) {
q := r.URL.Query()
if p := q.Get("p"); p != "" {
q.Set("p", "[REDACTED]")
r.URL.RawQuery = q.Encode()
}
}
// Common Log Format: http://en.wikipedia.org/wiki/Common_Log_Format
// buildLogLine creates a common log format
// in addition to the common fields, we also append referrer, user agent and request ID
func buildLogLine(l *responseLogger, r *http.Request, start time.Time) string {
redactPassword(r)
username := parseUsername(r)
host, _, err := net.SplitHostPort(r.RemoteAddr)
if err != nil {
host = r.RemoteAddr
}
uri := r.URL.RequestURI()
referer := r.Referer()
userAgent := r.UserAgent()
fields := []string{
host,
"-",
detect(username, "-"),
fmt.Sprintf("[%s]", start.Format("02/Jan/2006:15:04:05 -0700")),
r.Method,
uri,
r.Proto,
detect(strconv.Itoa(l.Status()), "-"),
strconv.Itoa(l.Size()),
detect(referer, "-"),
detect(userAgent, "-"),
r.Header.Get("Request-Id"),
fmt.Sprintf("%s", time.Since(start)),
}
return strings.Join(fields, " ")
}
// detect detects the first presense of a non blank string and returns it
func detect(values ...string) string {
for _, v := range values {
if v != "" {
return v
}
}
return ""
}
// parses the username either from the url or auth header
func parseUsername(r *http.Request) string {
var (
username = ""
url = r.URL
)
// get username from the url if passed there
if url.User != nil {
if name := url.User.Username(); name != "" {
username = name
}
}
// Try to get the username from the query param 'u'
q := url.Query()
if u := q.Get("u"); u != "" {
username = u
}
// Try to get it from the authorization header if set there
if username == "" {
if u, _, ok := r.BasicAuth(); ok {
username = u
}
}
return username
}
// Sanitize passwords from query string for logging.
func sanitize(r *http.Request, s string) {
r.URL.RawQuery = strings.Replace(r.URL.RawQuery, s, "[REDACTED]", -1)
}

Some files were not shown because too many files have changed in this diff Show More