Use graphite parser for templating, godep update to head

Godeps/Godeps.json

@@ -101,34 +101,9 @@
 			"Rev": "d1e82c1ec3f15ee991f7cc7ffd5b67ff6f5bbaee"
 		},
 		{
-			"ImportPath": "github.com/influxdb/influxdb/client",
+			"ImportPath": "github.com/influxdb/influxdb",
-			"Comment": "v0.9.3",
+			"Comment": "v0.9.4-rc1-457-g883d32c",
-			"Rev": "5d42b212fca8facfe9db0c83822f09b88be643ec"
+			"Rev": "883d32cfd06e8cf14e6d9fc75dbe7b7b92345623"
-		},
-		{
-			"ImportPath": "github.com/influxdb/influxdb/influxql",
-			"Comment": "v0.9.3",
-			"Rev": "5d42b212fca8facfe9db0c83822f09b88be643ec"
-		},
-		{
-			"ImportPath": "github.com/influxdb/influxdb/meta",
-			"Comment": "v0.9.3",
-			"Rev": "5d42b212fca8facfe9db0c83822f09b88be643ec"
-		},
-		{
-			"ImportPath": "github.com/influxdb/influxdb/snapshot",
-			"Comment": "v0.9.3",
-			"Rev": "5d42b212fca8facfe9db0c83822f09b88be643ec"
-		},
-		{
-			"ImportPath": "github.com/influxdb/influxdb/toml",
-			"Comment": "v0.9.3",
-			"Rev": "5d42b212fca8facfe9db0c83822f09b88be643ec"
-		},
-		{
-			"ImportPath": "github.com/influxdb/influxdb/tsdb",
-			"Comment": "v0.9.3",
-			"Rev": "5d42b212fca8facfe9db0c83822f09b88be643ec"
 		},
 		{
 			"ImportPath": "github.com/lib/pq",
@@ -268,6 +243,10 @@
 			"Comment": "v1.x.x",
 			"Rev": "8aca6ba2cc6e873299617d730fac0d7f6593113a"
 		},
+		{
+			"ImportPath": "gopkg.in/fatih/pool.v2",
+			"Rev": "cba550ebf9bce999a02e963296d4bc7a486cb715"
+		},
 		{
 			"ImportPath": "gopkg.in/mgo.v2",
 			"Comment": "r2015.06.03-3-g3569c88",

Godeps/_workspace/src/github.com/influxdb/influxdb/.gitignore

@@ -0,0 +1,72 @@
+*~
+src/
+
+config.json
+/bin/
+
+TAGS
+
+# vim temp files
+*.swp
+
+*.test
+/query/a.out*
+.DS_Store
+
+# ignore generated files.
+cmd/influxd/version.go
+
+# executables
+
+influx_stress
+**/influx_stress
+!**/influx_stress/
+
+influxd
+**/influxd
+!**/influxd/
+
+influx
+**/influx
+!**/influx/
+
+influxdb
+**/influxdb
+!**/influxdb/
+
+/benchmark-tool
+/main
+/benchmark-storage
+godef
+gosym
+gocode
+inspect-raft
+
+# dependencies
+out_rpm/
+packages/
+
+# autconf
+autom4te.cache/
+config.log
+config.status
+Makefile
+
+# log file
+influxdb.log
+benchmark.log
+
+# config file
+config.toml
+
+# test data files
+integration/migration_data/
+
+# goide project files
+.idea
+
+# goconvey config files
+*.goconvey
+
+// Ingnore SourceGraph directory
+.srclib-store/

Godeps/_workspace/src/github.com/influxdb/influxdb/CONTRIBUTING.md

@@ -0,0 +1,247 @@
+Contributing to InfluxDB
+========================
+
+Bug reports
+---------------
+Before you file an issue, please search existing issues in case it has already been filed, or perhaps even fixed. If you file an issue, please include the following.
+* Full details of your operating system (or distribution) e.g. 64-bit Ubuntu 14.04.
+* The version of InfluxDB you are running
+* Whether you installed it using a pre-built package, or built it from source.
+* A small test case, if applicable, that demonstrates the issues.
+
+Remember the golden rule of bug reports: **The easier you make it for us to reproduce the problem, the faster it will get fixed.**
+If you have never written a bug report before, or if you want to brush up on your bug reporting skills, we recommend reading [Simon Tatham's essay "How to Report Bugs Effectively."](http://www.chiark.greenend.org.uk/~sgtatham/bugs.html)
+
+Test cases should be in the form of `curl` commands. For example:
+```
+# create database
+curl -G http://localhost:8086/query --data-urlencode "q=CREATE DATABASE mydb"
+
+# create retention policy
+curl -G http://localhost:8086/query --data-urlencode "q=CREATE RETENTION POLICY myrp ON mydb DURATION 365d REPLICATION 1 DEFAULT"
+
+# write data
+curl -X POST http://localhost:8086/write --data-urlencode "db=mydb" --data-binary "cpu,region=useast,host=server_1,service=redis value=61" 
+
+# Delete a Measurement
+curl -G http://localhost:8086/query  --data-urlencode 'db=mydb' --data-urlencode 'q=DROP MEASUREMENT cpu'
+
+# Query the Measurement
+# Bug: expected it to return no data, but data comes back.
+curl -G http://localhost:8086/query  --data-urlencode 'db=mydb' --data-urlencode 'q=SELECT * from cpu'
+```
+**If you don't include a clear test case like this, your issue may not be investigated, and may even be closed**. If writing the data is too difficult, please zip up your data directory and include a link to it in your bug report.
+
+Please note that issues are *not the place to file general questions* such as "how do I use collectd with InfluxDB?" Questions of this nature should be sent to the [Google Group](https://groups.google.com/forum/#!forum/influxdb), not filed as issues. Issues like this will be closed.
+
+Feature requests
+---------------
+We really like to receive feature requests, as it helps us prioritize our work. Please be clear about your requirements, as incomplete feature requests may simply be closed if we don't understand what you would like to see added to InfluxDB.
+
+Contributing to the source code
+---------------
+
+InfluxDB follows standard Go project structure. This means that all
+your go development are done in `$GOPATH/src`. GOPATH can be any
+directory under which InfluxDB and all its dependencies will be
+cloned. For more details on recommended go project's structure, see
+[How to Write Go Code](http://golang.org/doc/code.html) and
+[Go: Best Practices for Production Environments](http://peter.bourgon.org/go-in-production/), or you can just follow
+the steps below.
+
+Submitting a pull request
+------------
+To submit a pull request you should fork the InfluxDB repository, and make your change on a feature branch of your fork. Then generate a pull request from your branch against *master* of the InfluxDB repository. Include in your pull request details of your change -- the why *and* the how -- as well as the testing your performed. Also, be sure to run the test suite with your change in place. Changes that cause tests to fail cannot be merged.
+
+There will usually be some back and forth as we finalize the change, but once that completes it may be merged.
+
+To assist in review for the PR, please add the following to your pull request comment:
+
+```md
+- [ ] CHANGELOG.md updated
+- [ ] Rebased/mergable
+- [ ] Tests pass
+- [ ] Sign [CLA](http://influxdb.com/community/cla.html) (if not already signed)
+```
+
+Use of third-party packages
+------------
+A third-party package is defined as one that is not part of the standard Go distribution. Generally speaking we prefer to minimize our use of third-party packages, and avoid them unless absolutely necessarly. We'll often write a little bit of code rather than pull in a third-party package. Of course, we do use some third-party packages -- most importantly we use [BoltDB](https://github.com/boltdb/bolt) as the storage engine. So to maximise the chance your change will be accepted by us, use only the standard libaries, or the third-party packages we have decided to use.
+
+For rationale, check out the post [The Case Against Third Party Libraries](http://blog.gopheracademy.com/advent-2014/case-against-3pl/).
+
+Signing the CLA
+---------------
+
+If you are going to be contributing back to InfluxDB please take a
+second to sign our CLA, which can be found
+[on our website](http://influxdb.com/community/cla.html).
+
+Installing Go
+-------------
+InfluxDB requires Go 1.5 or greater.
+
+At InfluxDB we find gvm, a Go version manager, useful for installing Go. For instructions
+on how to install it see [the gvm page on github](https://github.com/moovweb/gvm).
+
+After installing gvm you can install and set the default go version by
+running the following:
+
+    gvm install go1.5
+    gvm use go1.5 --default
+
+Revision Control Systems
+-------------
+Go has the ability to import remote packages via revision control systems with the `go get` command.  To ensure that you can retrieve any remote package, be sure to install the following rcs software to your system.
+Currently the project only depends on `git` and `mercurial`.
+
+* [Install Git](http://git-scm.com/book/en/Getting-Started-Installing-Git)
+* [Install Mercurial](http://mercurial.selenic.com/wiki/Download)
+
+Getting the source
+------
+Setup the project structure and fetch the repo like so:
+
+    mkdir $HOME/gocodez
+    export GOPATH=$HOME/gocodez
+    go get github.com/influxdb/influxdb
+
+You can add the line `export GOPATH=$HOME/gocodez` to your bash/zsh file to be set for every shell instead of having to manually run it everytime.
+
+Cloning a fork
+-------------
+If you wish to work with fork of InfluxDB, your own fork for example, you must still follow the directory structure above. But instead of cloning the main repo, instead clone your fork. Follow the steps below to work with a fork:
+
+    export GOPATH=$HOME/gocodez
+    mkdir -p $GOPATH/src/github.com/influxdb
+    cd $GOPATH/src/github.com/influxdb
+    git clone git@github.com:<username>/influxdb
+
+Retaining the directory structure `$GOPATH/src/github.com/influxdb` is necessary so that Go imports work correctly.
+
+Pre-commit checks
+-------------
+
+We have a pre commit hook to make sure code is formatted properly
+and vetted before you commit any changes. We strongly recommend using the pre
+commit hook to guard against accidentally committing unformatted
+code. To use the pre-commit hook, run the following:
+
+    cd $GOPATH/src/github.com/influxdb/influxdb
+    cp .hooks/pre-commit .git/hooks/
+
+In case the commit is rejected because it's not formatted you can run
+the following to format the code:
+
+```
+go fmt ./...
+go vet ./...
+```
+
+To install go vet, run the following command:
+```
+go get golang.org/x/tools/cmd/vet
+```
+
+NOTE: If you have not installed mercurial, the above command will fail.  See [Revision Control Systems](#revision-control-systems) above.
+
+For more information on `go vet`, [read the GoDoc](https://godoc.org/golang.org/x/tools/cmd/vet).
+
+Build and Test
+-----
+
+Make sure you have Go installed and the project structure as shown above. To then build the project, execute the following commands:
+
+```bash
+cd $GOPATH/src/github.com/influxdb
+go get -u -f -t ./...
+go build ./...
+```
+
+To then install the binaries, run the following command. They can be found in `$GOPATH/bin`. Please note that the InfluxDB binary is named `influxd`, not `influxdb`.
+
+```bash
+go install ./...
+```
+
+To set the version and commit flags during the build pass the following to the build command:
+
+```bash
+-ldflags="-X main.version=$VERSION -X main.branch=$BRANCH -X main.commit=$COMMIT -X main.buildTime=$TIME"
+```
+
+where `$VERSION` is the version, `$BRANCH` is the branch, `$COMMIT` is the git commit hash, and `$TIME` is the build timestamp.
+
+If you want to build packages, see `package.sh` help:
+```bash
+package.sh -h
+```
+
+To run the tests, execute the following command:
+
+```bash
+cd $GOPATH/src/github.com/influxdb/influxdb
+go test -v ./...
+
+# run tests that match some pattern
+go test -run=TestDatabase . -v
+
+# run tests and show coverage
+go test -coverprofile /tmp/cover . && go tool cover -html /tmp/cover
+```
+
+To install go cover, run the following command:
+```
+go get golang.org/x/tools/cmd/cover
+```
+
+Generated Google Protobuf code
+-----------------
+Most changes to the source do not require that the generated protocol buffer code be changed. But if you need to modify the protocol buffer code, you'll first need to install the protocol buffers toolchain.
+
+First install the [protocol buffer compiler](https://developers.google.com/protocol-buffers/
+) 2.6.1 or later for your OS:
+
+Then install the go plugins:
+
+```bash
+go get github.com/gogo/protobuf/proto
+go get github.com/gogo/protobuf/protoc-gen-gogo
+go get github.com/gogo/protobuf/gogoproto
+```
+
+Finally run, `go generate` after updating any `*.proto` file:
+
+```bash
+go generate ./...
+```
+**Trouleshooting**
+
+If generating the protobuf code is failing for you, check each of the following:
+ * Ensure the protobuf library can be found. Make sure that `LD_LIBRRARY_PATH` includes the directory in which the library `libprotoc.so` has been installed.
+ * Ensure the command `protoc-gen-gogo`, found in `GOPATH/bin`, is on your path. This can be done by adding `GOPATH/bin` to `PATH`.
+
+Profiling
+-----
+When troubleshooting problems with CPU or memory the Go toolchain can be helpful. You can start InfluxDB with CPU or memory profiling turned on. For example:
+
+```sh
+# start influx with profiling
+./influxd -cpuprofile influxd.prof
+# run queries, writes, whatever you're testing
+# open up pprof
+go tool pprof influxd influxd.prof
+# once inside run "web", opens up browser with the CPU graph
+# can also run "web <function name>" to zoom in. Or "list <function name>" to see specific lines
+```
+
+Continuous Integration testing
+-----
+InfluxDB uses CirceCI for continuous integration testing. To see how the code is built and tested, check out [this file](https://github.com/influxdb/influxdb/blob/master/circle-test.sh). It closely follows the build and test process outlined above. You can see the exact version of Go InfluxDB uses for testing by consulting that file.
+
+Useful links
+------------
+- [Useful techniques in Go](http://arslan.io/ten-useful-techniques-in-go)
+- [Go in production](http://peter.bourgon.org/go-in-production/)
+- [Principles of designing Go APIs with channels](https://inconshreveable.com/07-08-2014/principles-of-designing-go-apis-with-channels/)
+- [Common mistakes in Golang](http://soryy.com/blog/2014/common-mistakes-with-go-lang/). Especially this section `Loops, Closures, and Local Variables`

Godeps/_workspace/src/github.com/influxdb/influxdb/DOCKER.md

@@ -0,0 +1,44 @@
+# Docker Setup
+========================
+
+This document describes how to build and run a minimal InfluxDB container under Docker.   Currently, it has only been tested for local development and assumes that you have a working docker environment.
+
+## Building Image
+
+To build a docker image for InfluxDB from your current checkout, run the following:
+
+```
+$ ./build-docker.sh
+```
+
+This script uses the `golang:1.5` image to build a fully static binary of `influxd` and then adds it to a minimal `scratch` image.
+
+To build the image using a different version of go:
+
+```
+$ GO_VER=1.4.2 ./build-docker.sh
+```
+
+Available version can be found [here](https://hub.docker.com/_/golang/).
+
+## Single Node Container
+
+This will start an interactive, single-node, that publishes the containers port `8086` and `8088` to the hosts ports `8086` and `8088` respectively.  This is identical to starting `influxd` manually.
+
+```
+$ docker run -it -p 8086:8086 -p 8088:8088 influxdb
+```
+
+## Multi-Node Cluster
+
+This will create a simple 3-node cluster.  The data is stored within the container and will be lost when the container is removed.  This is only useful for test clusters.
+
+The `HOST_IP` env variable should be your host IP if running under linux or the virtualbox VM IP if running under OSX.  On OSX, this would be something like: `$(docker-machine ip dev)` or `$(boot2docker ip)` depending on which docker tool you are using.
+
+```
+$ export HOST_IP=<your host/VM IP>
+$ docker run -it -p 8086:8086 -p 8088:8088 influxdb -hostname $HOST_IP:8088
+$ docker run -it -p 8186:8086 -p 8188:8088 influxdb -hostname $HOST_IP:8188 -join $HOST_IP:8088
+$ docker run -it -p 8286:8086 -p 8288:8088 influxdb -hostname $HOST_IP:8288 -join $HOST_IP:8088
+```
+

Godeps/_workspace/src/github.com/influxdb/influxdb/Dockerfile

@@ -0,0 +1,24 @@
+FROM busybox:ubuntu-14.04
+
+MAINTAINER Jason Wilder "<jason@influxdb.com>"
+
+# admin, http, udp, cluster, graphite, opentsdb, collectd
+EXPOSE 8083 8086 8086/udp 8088 2003 4242 25826
+
+WORKDIR /app
+
+# copy binary into image
+COPY influxd /app/
+
+# Add influxd to the PATH
+ENV PATH=/app:$PATH
+
+# Generate a default config
+RUN influxd config > /etc/influxdb.toml
+
+# Use /data for all disk storage
+RUN sed -i 's/dir = "\/.*influxdb/dir = "\/data/' /etc/influxdb.toml
+
+VOLUME ["/data"]
+
+ENTRYPOINT ["influxd", "--config", "/etc/influxdb.toml"]

Godeps/_workspace/src/github.com/influxdb/influxdb/Dockerfile_test_ubuntu32

@@ -0,0 +1,12 @@
+FROM 32bit/ubuntu:14.04
+
+RUN apt-get update && apt-get install -y python-software-properties software-properties-common git
+RUN add-apt-repository ppa:evarlast/golang1.5
+RUN apt-get update && apt-get install -y -o Dpkg::Options::="--force-overwrite" golang-go
+
+ENV GOPATH=/root/go
+RUN mkdir -p /root/go/src/github.com/influxdb/influxdb
+RUN mkdir -p /tmp/artifacts
+
+VOLUME /root/go/src/github.com/influxdb/influxdb
+VOLUME /tmp/artifacts

Godeps/_workspace/src/github.com/influxdb/influxdb/LICENSE

@@ -0,0 +1,20 @@
+The MIT License (MIT)
+
+Copyright (c) 2013-2015 Errplane Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+the Software, and to permit persons to whom the Software is furnished to do so,
+subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

Godeps/_workspace/src/github.com/influxdb/influxdb/LICENSE_OF_DEPENDENCIES.md

@@ -0,0 +1,19 @@
+# List
+- github.com/gogo/protobuf/proto [BSD LICENSE](https://github.com/gogo/protobuf/blob/master/LICENSE)
+- gopkg.in/fatih/pool.v2 [MIT LICENSE](https://github.com/fatih/pool/blob/v2.0.0/LICENSE)
+- github.com/BurntSushi/toml [WTFPL LICENSE](https://github.com/BurntSushi/toml/blob/master/COPYING)
+- github.com/peterh/liner [MIT LICENSE](https://github.com/peterh/liner/blob/master/COPYING)
+- github.com/davecgh/go-spew/spew [ISC LICENSE](https://github.com/davecgh/go-spew/blob/master/LICENSE)
+- github.com/hashicorp/raft [MPL LICENSE](https://github.com/hashicorp/raft/blob/master/LICENSE)
+- github.com/rakyll/statik/fs [APACHE LICENSE](https://github.com/rakyll/statik/blob/master/LICENSE)
+- github.com/kimor79/gollectd [BSD LICENSE](https://github.com/kimor79/gollectd/blob/master/LICENSE)
+- github.com/bmizerany/pat [MIT LICENSE](https://github.com/bmizerany/pat#license)
+- react 0.13.3 [BSD LICENSE](https://github.com/facebook/react/blob/master/LICENSE)
+- bootstrap 3.3.5 [MIT LICENSE](https://github.com/twbs/bootstrap/blob/master/LICENSE)
+- jquery 2.1.4 [MIT LICENSE](https://github.com/jquery/jquery/blob/master/LICENSE.txt)
+- glyphicons [LICENSE](http://glyphicons.com/license/)
+- github.com/golang/snappy [BSD LICENSE](https://github.com/golang/snappy/blob/master/LICENSE)
+- github.com/boltdb/bolt [MIT LICENSE](https://github.com/boltdb/bolt/blob/master/LICENSE)
+- collectd.org [ISC LICENSE](https://github.com/collectd/go-collectd/blob/master/LICENSE)
+- golang.org/x/crypto/bcrypt [BSD LICENSE](https://go.googlesource.com/crypto/+/master/LICENSE)
+

Godeps/_workspace/src/github.com/influxdb/influxdb/QUERIES.md

@@ -0,0 +1,180 @@
+The top level name is called a measurement. These names can contain any characters. Then there are field names, field values, tag keys and tag values, which can also contain any characters. However, if the measurement, field, or tag contains any character other than [A-Z,a-z,0-9,_], or if it starts with a digit, it must be double-quoted. Therefore anywhere a measurement name, field name, field value, tag name, or tag value appears it should be wrapped in double quotes.
+
+# Databases & retention policies
+
+```sql
+-- create a database
+CREATE DATABASE <name>
+
+-- create a retention policy
+CREATE RETENTION POLICY <rp-name> ON <db-name> DURATION <duration> REPLICATION <n> [DEFAULT]
+
+-- alter retention policy
+ALTER RETENTION POLICY <rp-name> ON <db-name> (DURATION <duration> | REPLICATION <n> | DEFAULT)+
+
+-- drop a database
+DROP DATABASE <name>
+
+-- drop a retention policy
+DROP RETENTION POLICY <rp-name> ON <db-name>
+```
+where `<duration>` is either `INF` for infinite retention, or an integer followed by the desired unit of time: u,ms,s,m,h,d,w for microseconds, milliseconds, seconds, minutes, hours, days, or weeks, respectively. `<replication>` must be an integer.
+
+If present, `DEFAULT` sets the retention policy as the default retention policy for writes and reads.
+
+# Users and permissions
+
+```sql
+-- create user
+CREATE USER <name> WITH PASSWORD '<password>'
+
+-- grant privilege on a database
+GRANT <privilege> ON <db> TO <user>
+
+-- grant cluster admin privileges
+GRANT ALL [PRIVILEGES] TO <user>
+
+-- revoke privilege
+REVOKE <privilege> ON <db> FROM <user>
+
+-- revoke all privileges for a DB
+REVOKE ALL [PRIVILEGES] ON <db> FROM <user>
+
+-- revoke all privileges including cluster admin
+REVOKE ALL [PRIVILEGES] FROM <user>
+
+-- combine db creation with privilege assignment (user must already exist)
+CREATE DATABASE <name> GRANT <privilege> TO <user>
+CREATE DATABASE <name> REVOKE <privilege> FROM <user>
+
+-- delete a user
+DROP USER <name>
+
+
+```
+where `<privilege> := READ | WRITE | All `. 
+
+Authentication must be enabled in the influxdb.conf file for user permissions to be in effect.
+
+By default, newly created users have no privileges to any databases.
+
+Cluster administration privileges automatically grant full read and write permissions to all databases, regardless of subsequent database-specific privilege revocation statements.
+
+# Select
+
+```sql
+SELECT mean(value) from cpu WHERE host = 'serverA' AND time > now() - 4h GROUP BY time(5m)
+
+SELECT mean(value) from cpu WHERE time > now() - 4h GROUP BY time(5m), region
+```
+
+## Group By
+
+# Delete
+
+# Series
+
+## Destroy
+
+```sql
+DROP MEASUREMENT <name>
+DROP MEASUREMENT cpu WHERE region = 'uswest'
+```
+
+## Show
+
+Show series queries are for pulling out individual series from measurement names and tag data. They're useful for discovery.
+
+```sql
+-- show all databases
+SHOW DATABASES
+
+-- show measurement names
+SHOW MEASUREMENTS
+SHOW MEASUREMENTS LIMIT 15
+SHOW MEASUREMENTS LIMIT 10 OFFSET 40
+SHOW MEASUREMENTS WHERE service = 'redis'
+-- LIMIT and OFFSET can be applied to any of the SHOW type queries
+
+-- show all series across all measurements/tagsets
+SHOW SERIES
+
+-- get a show of all series for any measurements where tag key region = tak value 'uswest'
+SHOW SERIES WHERE region = 'uswest'
+
+SHOW SERIES FROM cpu_load WHERE region = 'uswest' LIMIT 10
+
+-- returns the 100 - 109 rows in the result. In the case of SHOW SERIES, which returns 
+-- series split into measurements. Each series counts as a row. So you could see only a 
+-- single measurement returned, but 10 series within it.
+SHOW SERIES FROM cpu_load WHERE region = 'uswest' LIMIT 10 OFFSET 100
+
+-- show all retention policies on a database
+SHOW RETENTION POLICIES ON mydb
+
+-- get a show of all tag keys across all measurements
+SHOW TAG KEYS
+
+-- show all the tag keys for a given measurement
+SHOW TAG KEYS FROM cpu
+SHOW TAG KEYS FROM temperature, wind_speed
+
+-- show all the tag values. note that a single WHERE TAG KEY = '...' clause is required
+SHOW TAG VALUES WITH TAG KEY = 'region'
+SHOW TAG VALUES FROM cpu WHERE region = 'uswest' WITH TAG KEY = 'host'
+
+-- and you can do stuff against fields
+SHOW FIELD KEYS FROM cpu
+
+-- but you can't do this
+SHOW FIELD VALUES
+-- we don't index field values, so this query should be invalid.
+
+-- show all users
+SHOW USERS
+```
+
+Note that `FROM` and `WHERE` are optional clauses in most of the show series queries.
+
+And the show series output looks like this:
+
+```json
+[
+    {
+        "name": "cpu",
+        "columns": ["id", "region", "host"],
+        "values": [
+            1, "uswest", "servera",
+            2, "uswest", "serverb"
+        ]
+    },
+    {
+        "name": "reponse_time",
+        "columns": ["id", "application", "host"],
+        "values": [
+            3, "myRailsApp", "servera"
+        ]
+    }
+]
+```
+
+# Continuous Queries
+
+Continous queries are going to be inspired by MySQL `TRIGGER` syntax:
+
+http://dev.mysql.com/doc/refman/5.0/en/trigger-syntax.html
+
+Instead of having automatically-assigned ids, named continuous queries allows for some level of duplication prevention,
+particularly in the case where creation is scripted.
+
+## Create
+
+    CREATE CONTINUOUS QUERY <name> AS SELECT ... FROM ...
+
+## Destroy
+
+    DROP CONTINUOUS QUERY <name>
+
+## List
+
+    SHOW CONTINUOUS QUERIES

Godeps/_workspace/src/github.com/influxdb/influxdb/README.md

@@ -0,0 +1,72 @@
+# InfluxDB [![Circle CI](https://circleci.com/gh/influxdb/influxdb/tree/master.svg?style=svg)](https://circleci.com/gh/influxdb/influxdb/tree/master)
+
+## An Open-Source, Distributed, Time Series Database
+
+> InfluxDB v0.9.0 is now out. Going forward, the 0.9.x series of releases will not make breaking API changes or breaking changes to the underlying data storage. However, 0.9.0 clustering should be considered an alpha release.
+
+InfluxDB is an open source **distributed time series database** with
+**no external dependencies**. It's useful for recording metrics,
+events, and performing analytics.
+
+## Features
+
+* Built-in [HTTP API](http://influxdb.com/docs/v0.9/concepts/reading_and_writing_data.html) so you don't have to write any server side code to get up and running.
+* Data can be tagged, allowing very flexible querying.
+* SQL-like query language.
+* Clustering is supported out of the box, so that you can scale horizontally to handle your data.
+* Simple to install and manage, and fast to get data in and out.
+* It aims to answer queries in real-time. That means every data point is
+  indexed as it comes in and is immediately available in queries that
+  should return in < 100ms.
+
+## Getting Started
+*The following directions apply only to the 0.9.0 release or building from the source on master.*
+
+### Building
+
+You don't need to build the project to use it - you can use any of our
+[pre-built packages](http://influxdb.com/download/index.html) to install InfluxDB. That's
+the recommended way to get it running. However, if you want to contribute to the core of InfluxDB, you'll need to build.
+For those adventurous enough, you can
+[follow along on our docs](http://github.com/influxdb/influxdb/blob/master/CONTRIBUTING.md).
+
+### Starting InfluxDB
+* `service influxdb start` if you have installed InfluxDB using an official Debian or RPM package.
+* `systemctl start influxdb` if you have installed InfluxDB using an official Debian or RPM package, and are running a distro with `systemd`. For example, Ubuntu 15 or later.
+* `$GOPATH/bin/influxd` if you have built InfluxDB from source.
+
+### Creating your first database
+
+```
+curl -G 'http://localhost:8086/query' --data-urlencode "q=CREATE DATABASE mydb"
+```
+
+### Insert some data
+```
+curl -XPOST 'http://localhost:8086/write?db=mydb' \
+-d 'cpu,host=server01,region=uswest load=42 1434055562000000000'
+
+curl -XPOST 'http://localhost:8086/write?db=mydb' \
+-d 'cpu,host=server02,region=uswest load=78 1434055562000000000'
+
+curl -XPOST 'http://localhost:8086/write?db=mydb' \
+-d 'cpu,host=server03,region=useast load=15.4 1434055562000000000'
+```
+
+### Query for the data
+```JSON
+curl -G http://localhost:8086/query?pretty=true --data-urlencode "db=mydb" \
+--data-urlencode "q=SELECT * FROM cpu WHERE host='server01' AND time < now() - 1d"
+```
+
+### Analyze the data
+```JSON
+curl -G http://localhost:8086/query?pretty=true --data-urlencode "db=mydb" \
+--data-urlencode "q=SELECT mean(load) FROM cpu WHERE region='uswest'"
+```
+
+## Helpful Links
+
+* Understand the [design goals and motivations of the project](http://influxdb.com/docs/v0.9/introduction/overview.html).
+* Follow the [getting started guide](http://influxdb.com/docs/v0.9/introduction/getting_started.html) to find out how to install InfluxDB, start writing more data, and issue more queries - in just a few minutes.
+* See the  [HTTP API documentation to start writing a library for your favorite language](http://influxdb.com/docs/v0.9/concepts/reading_and_writing_data.html).

Godeps/_workspace/src/github.com/influxdb/influxdb/balancer.go

@@ -0,0 +1,78 @@
+package influxdb
+
+import (
+	"math/rand"
+
+	"github.com/influxdb/influxdb/meta"
+)
+
+// Balancer represents a load-balancing algorithm for a set of nodes
+type Balancer interface {
+	// Next returns the next Node according to the balancing method
+	// or nil if there are no nodes available
+	Next() *meta.NodeInfo
+}
+
+type nodeBalancer struct {
+	nodes []meta.NodeInfo // data nodes to balance between
+	p     int             // current node index
+}
+
+// NewNodeBalancer create a shuffled, round-robin balancer so that
+// multiple instances will return nodes in randomized order and each
+// each returned node will be repeated in a cycle
+func NewNodeBalancer(nodes []meta.NodeInfo) Balancer {
+	// make a copy of the node slice so we can randomize it
+	// without affecting the original instance as well as ensure
+	// that each Balancer returns nodes in a different order
+	b := &nodeBalancer{}
+
+	b.nodes = make([]meta.NodeInfo, len(nodes))
+	copy(b.nodes, nodes)
+
+	b.shuffle()
+	return b
+}
+
+// shuffle randomizes the ordering the balancers available nodes
+func (b *nodeBalancer) shuffle() {
+	for i := range b.nodes {
+		j := rand.Intn(i + 1)
+		b.nodes[i], b.nodes[j] = b.nodes[j], b.nodes[i]
+	}
+}
+
+// online returns a slice of the nodes that are online
+func (b *nodeBalancer) online() []meta.NodeInfo {
+	return b.nodes
+	// now := time.Now().UTC()
+	// up := []meta.NodeInfo{}
+	// for _, n := range b.nodes {
+	// 	if n.OfflineUntil.After(now) {
+	// 		continue
+	// 	}
+	// 	up = append(up, n)
+	// }
+	// return up
+}
+
+// Next returns the next available nodes
+func (b *nodeBalancer) Next() *meta.NodeInfo {
+	// only use online nodes
+	up := b.online()
+
+	// no nodes online
+	if len(up) == 0 {
+		return nil
+	}
+
+	// rollover back to the beginning
+	if b.p >= len(up) {
+		b.p = 0
+	}
+
+	d := &up[b.p]
+	b.p += 1
+
+	return d
+}

Godeps/_workspace/src/github.com/influxdb/influxdb/balancer_test.go

@@ -0,0 +1,115 @@
+package influxdb_test
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/influxdb/influxdb"
+	"github.com/influxdb/influxdb/meta"
+)
+
+func NewNodes() []meta.NodeInfo {
+	var nodes []meta.NodeInfo
+	for i := 1; i <= 2; i++ {
+		nodes = append(nodes, meta.NodeInfo{
+			ID:   uint64(i),
+			Host: fmt.Sprintf("localhost:999%d", i),
+		})
+	}
+	return nodes
+}
+
+func TestBalancerEmptyNodes(t *testing.T) {
+	b := influxdb.NewNodeBalancer([]meta.NodeInfo{})
+	got := b.Next()
+	if got != nil {
+		t.Errorf("expected nil, got %v", got)
+	}
+}
+
+func TestBalancerUp(t *testing.T) {
+	nodes := NewNodes()
+	b := influxdb.NewNodeBalancer(nodes)
+
+	// First node in randomized round-robin order
+	first := b.Next()
+	if first == nil {
+		t.Errorf("expected datanode, got %v", first)
+	}
+
+	// Second node in randomized round-robin order
+	second := b.Next()
+	if second == nil {
+		t.Errorf("expected datanode, got %v", second)
+	}
+
+	// Should never get the same node in order twice
+	if first.ID == second.ID {
+		t.Errorf("expected first != second. got %v = %v", first.ID, second.ID)
+	}
+}
+
+/*
+func TestBalancerDown(t *testing.T) {
+	nodes := NewNodes()
+	b := influxdb.NewNodeBalancer(nodes)
+
+	nodes[0].Down()
+
+	// First node in randomized round-robin order
+	first := b.Next()
+	if first == nil {
+		t.Errorf("expected datanode, got %v", first)
+	}
+
+	// Second node should rollover to the first up node
+	second := b.Next()
+	if second == nil {
+		t.Errorf("expected datanode, got %v", second)
+	}
+
+	// Health node should be returned each time
+	if first.ID != 2 && first.ID != second.ID {
+		t.Errorf("expected first != second. got %v = %v", first.ID, second.ID)
+	}
+}
+*/
+
+/*
+func TestBalancerBackUp(t *testing.T) {
+	nodes := newDataNodes()
+	b := influxdb.NewNodeBalancer(nodes)
+
+	nodes[0].Down()
+
+	for i := 0; i < 3; i++ {
+		got := b.Next()
+		if got == nil {
+			t.Errorf("expected datanode, got %v", got)
+		}
+
+		if exp := uint64(2); got.ID != exp {
+			t.Errorf("wrong node id: exp %v, got %v", exp, got.ID)
+		}
+	}
+
+	nodes[0].Up()
+
+	// First node in randomized round-robin order
+	first := b.Next()
+	if first == nil {
+		t.Errorf("expected datanode, got %v", first)
+	}
+
+	// Second node should rollover to the first up node
+	second := b.Next()
+	if second == nil {
+		t.Errorf("expected datanode, got %v", second)
+	}
+
+	// Should get both nodes returned
+	if first.ID == second.ID {
+		t.Errorf("expected first != second. got %v = %v", first.ID, second.ID)
+	}
+}
+*/

Godeps/_workspace/src/github.com/influxdb/influxdb/build-docker.sh

@@ -0,0 +1,9 @@
+#!/bin/sh
+
+set -e -x
+
+GO_VER=${GO_VER:-1.5}
+
+docker run -it -v "${GOPATH}":/gopath -v "$(pwd)":/app -e "GOPATH=/gopath" -w /app golang:$GO_VER sh -c 'CGO_ENABLED=0 go build -a --installsuffix cgo --ldflags="-s" -o influxd ./cmd/influxd'
+
+docker build -t influxdb .

Godeps/_workspace/src/github.com/influxdb/influxdb/circle-test.sh

@@ -0,0 +1,95 @@
+#!/bin/bash
+#
+# This is the InfluxDB CircleCI test script. Using this script allows total control
+# the environment in which the build and test is run, and matches the official
+# build process for InfluxDB.
+
+BUILD_DIR=$HOME/influxdb-build
+GO_VERSION=go1.5.1
+PARALLELISM="-parallel 256"
+TIMEOUT="-timeout 480s"
+
+# Executes the given statement, and exits if the command returns a non-zero code.
+function exit_if_fail {
+    command=$@
+    echo "Executing '$command'"
+    $command
+    rc=$?
+    if [ $rc -ne 0 ]; then
+        echo "'$command' returned $rc."
+        exit $rc
+    fi
+}
+
+# Check that go fmt has been run.
+function check_go_fmt {
+	fmtcount=`git ls-files | grep '.go$' | xargs gofmt -l 2>&1 | wc -l`
+	if [ $fmtcount -gt 0 ]; then
+	    echo "run 'go fmt ./...' to format your source code."
+	    exit 1
+	fi
+}
+
+# Check that go vet passes.
+function check_go_vet {
+	# Due to the way composites work, vet will fail for some of our tests so we ignore it
+	vetcount=`go tool vet --composites=false ./ 2>&1  | wc -l`
+	if [ $vetcount -gt 0 ]; then
+	    echo "run 'go tool vet --composites=false ./' to see the errors it flags and correct your source code."
+	    exit 1
+	fi
+}
+
+source $HOME/.gvm/scripts/gvm
+exit_if_fail gvm use $GO_VERSION
+
+# Set up the build directory, and then GOPATH.
+exit_if_fail mkdir $BUILD_DIR
+export GOPATH=$BUILD_DIR
+exit_if_fail mkdir -p $GOPATH/src/github.com/influxdb
+
+# Dump some test config to the log.
+echo "Test configuration"
+echo "========================================"
+echo "\$HOME: $HOME"
+echo "\$GOPATH: $GOPATH"
+echo "\$CIRCLE_BRANCH: $CIRCLE_BRANCH"
+
+# Move the checked-out source to a better location.
+exit_if_fail mv $HOME/influxdb $GOPATH/src/github.com/influxdb
+exit_if_fail cd $GOPATH/src/github.com/influxdb/influxdb
+exit_if_fail git branch --set-upstream-to=origin/$CIRCLE_BRANCH $CIRCLE_BRANCH
+
+# Install the code.
+exit_if_fail cd $GOPATH/src/github.com/influxdb/influxdb
+exit_if_fail go get -t -d -v ./...
+exit_if_fail git checkout $CIRCLE_BRANCH # 'go get' switches to master. Who knew? Switch back.
+check_go_fmt
+check_go_vet
+exit_if_fail go build -v ./...
+
+# Run the tests.
+case $CIRCLE_NODE_INDEX in
+    0)
+        go test $PARALLELISM $TIMEOUT -v ./... 2>&1 | tee $CIRCLE_ARTIFACTS/test_logs.txt
+        rc=${PIPESTATUS[0]}
+        ;;
+    1)
+        # 32bit tests.
+        if [[ -e ~/docker/image.tar ]]; then docker load -i ~/docker/image.tar; fi
+        docker build -f Dockerfile_test_ubuntu32 -t ubuntu-32-influxdb-test .
+        mkdir -p ~/docker; docker save ubuntu-32-influxdb-test > ~/docker/image.tar
+        exit_if_fail docker build -f Dockerfile_test_ubuntu32 -t ubuntu-32-influxdb-test .
+        docker run -v $(pwd):/root/go/src/github.com/influxdb/influxdb -e "CI=${CI}" \
+                        -v ${CIRCLE_ARTIFACTS}:/tmp/artifacts \
+                        -t ubuntu-32-influxdb-test bash \
+                        -c "cd /root/go/src/github.com/influxdb/influxdb && go get -t -d -v ./... && go build -v ./... && go test ${PARALLELISM} ${TIMEOUT} -v ./... 2>&1 | tee /tmp/artifacts/test_logs_i386.txt && exit \${PIPESTATUS[0]}"
+        rc=$?
+        ;;
+    2)
+       GORACE="halt_on_error=1" go test $PARALLELISM $TIMEOUT -v -race ./... 2>&1 | tee $CIRCLE_ARTIFACTS/test_logs_race.txt
+       rc=${PIPESTATUS[0]}
+       ;;
+esac
+
+exit $rc

Godeps/_workspace/src/github.com/influxdb/influxdb/circle.yml

@@ -0,0 +1,16 @@
+machine:
+    services:
+        - docker
+    pre:
+        - bash < <(curl -s -S -L https://raw.githubusercontent.com/moovweb/gvm/master/binscripts/gvm-installer)
+        - source $HOME/.gvm/scripts/gvm; gvm install go1.5.1 --binary
+
+dependencies:
+    override:
+        - mkdir -p ~/docker
+    cache_directories:
+        - "~/docker"
+test:
+    override:
+        - bash circle-test.sh:
+            parallel: true

Godeps/_workspace/src/github.com/influxdb/influxdb/client/README.md

@@ -45,7 +45,12 @@ the configuration below.
 package main
 
 import "github.com/influxdb/influxdb/client"
-import "net/url"
+import (
+	  "net/url"
+	  "fmt"
+	  "log"
+	  "os"
+)
 
 const (
 	MyHost        = "localhost"
@@ -190,8 +195,8 @@ for i, row := range res[0].Series[0].Values {
 	if err != nil {
 		log.Fatal(err)
 	}
-	val, err := row[1].(json.Number).Int64()
+	val := row[1].(string)
-	log.Printf("[%2d] %s: %03d\n", i, t.Format(time.Stamp), val)
+	log.Printf("[%2d] %s: %s\n", i, t.Format(time.Stamp), val)
 }
 ```
 

Godeps/_workspace/src/github.com/influxdb/influxdb/client/influxdb.go

@@ -13,8 +13,7 @@ import (
 	"strings"
 	"time"
 
-	"github.com/influxdb/influxdb/influxql"
+	"github.com/influxdb/influxdb/models"
-	"github.com/influxdb/influxdb/tsdb"
 )
 
 const (
@@ -79,6 +78,7 @@ type Config struct {
 	Password  string
 	UserAgent string
 	Timeout   time.Duration
+	Precision string
 }
 
 // NewConfig will create a config to be used in connecting to the client
@@ -95,6 +95,7 @@ type Client struct {
 	password   string
 	httpClient *http.Client
 	userAgent  string
+	precision  string
 }
 
 const (
@@ -112,6 +113,7 @@ func NewClient(c Config) (*Client, error) {
 		password:   c.Password,
 		httpClient: &http.Client{Timeout: c.Timeout},
 		userAgent:  c.UserAgent,
+		precision:  c.Precision,
 	}
 	if client.userAgent == "" {
 		client.userAgent = "InfluxDBClient"
@@ -125,6 +127,11 @@ func (c *Client) SetAuth(u, p string) {
 	c.password = p
 }
 
+// SetPrecision will update the precision
+func (c *Client) SetPrecision(precision string) {
+	c.precision = precision
+}
+
 // Query sends a command to the server and returns the Response
 func (c *Client) Query(q Query) (*Response, error) {
 	u := c.url
@@ -133,6 +140,9 @@ func (c *Client) Query(q Query) (*Response, error) {
 	values := u.Query()
 	values.Set("q", q.Command)
 	values.Set("db", q.Database)
+	if c.precision != "" {
+		values.Set("epoch", c.precision)
+	}
 	u.RawQuery = values.Encode()
 
 	req, err := http.NewRequest("GET", u.String(), nil)
@@ -314,7 +324,7 @@ func (c *Client) Ping() (time.Duration, string, error) {
 
 // Result represents a resultset returned from a single statement.
 type Result struct {
-	Series []influxql.Row
+	Series []models.Row
 	Err    error
 }
 
@@ -322,8 +332,8 @@ type Result struct {
 func (r *Result) MarshalJSON() ([]byte, error) {
 	// Define a struct that outputs "error" as a string.
 	var o struct {
-		Series []influxql.Row `json:"series,omitempty"`
+		Series []models.Row `json:"series,omitempty"`
-		Err    string         `json:"error,omitempty"`
+		Err    string       `json:"error,omitempty"`
 	}
 
 	// Copy fields to output struct.
@@ -338,8 +348,8 @@ func (r *Result) MarshalJSON() ([]byte, error) {
 // UnmarshalJSON decodes the data into the Result struct
 func (r *Result) UnmarshalJSON(b []byte) error {
 	var o struct {
-		Series []influxql.Row `json:"series,omitempty"`
+		Series []models.Row `json:"series,omitempty"`
-		Err    string         `json:"error,omitempty"`
+		Err    string       `json:"error,omitempty"`
 	}
 
 	dec := json.NewDecoder(bytes.NewBuffer(b))
@@ -449,7 +459,11 @@ func (p *Point) MarshalJSON() ([]byte, error) {
 }
 
 func (p *Point) MarshalString() string {
-	return tsdb.NewPoint(p.Measurement, p.Tags, p.Fields, p.Time).String()
+	pt := models.NewPoint(p.Measurement, p.Tags, p.Fields, p.Time)
+	if p.Precision == "" || p.Precision == "ns" || p.Precision == "n" {
+		return pt.String()
+	}
+	return pt.PrecisionString(p.Precision)
 }
 
 // UnmarshalJSON decodes the data into the Point struct

Godeps/_workspace/src/github.com/influxdb/influxdb/client/influxdb_test.go

@@ -498,13 +498,12 @@ func TestBatchPoints_Normal(t *testing.T) {
 }
 
 func TestClient_Timeout(t *testing.T) {
+	done := make(chan bool)
 	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		time.Sleep(1 * time.Second)
+		<-done
-		var data client.Response
-		w.WriteHeader(http.StatusOK)
-		_ = json.NewEncoder(w).Encode(data)
 	}))
 	defer ts.Close()
+	defer func() { done <- true }()
 
 	u, _ := url.Parse(ts.URL)
 	config := client.Config{URL: *u, Timeout: 500 * time.Millisecond}
@@ -517,13 +516,33 @@ func TestClient_Timeout(t *testing.T) {
 	_, err = c.Query(query)
 	if err == nil {
 		t.Fatalf("unexpected success.  expected timeout error")
-	} else if !strings.Contains(err.Error(), "use of closed network connection") {
+	} else if !strings.Contains(err.Error(), "request canceled") &&
-		t.Fatalf("unexpected error.  expected 'use of closed network connection' error, got %v", err)
+		!strings.Contains(err.Error(), "use of closed network connection") {
+		t.Fatalf("unexpected error.  expected 'request canceled' error, got %v", err)
+	}
+}
+
+func TestClient_NoTimeout(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping in short mode")
+	}
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		time.Sleep(1 * time.Second)
+		var data client.Response
+		w.WriteHeader(http.StatusOK)
+		_ = json.NewEncoder(w).Encode(data)
+	}))
+	defer ts.Close()
+
+	u, _ := url.Parse(ts.URL)
+	config := client.Config{URL: *u}
+	c, err := client.NewClient(config)
+	if err != nil {
+		t.Fatalf("unexpected error.  expected %v, actual %v", nil, err)
 	}
 
-	confignotimeout := client.Config{URL: *u}
+	query := client.Query{}
-	cnotimeout, err := client.NewClient(confignotimeout)
+	_, err = c.Query(query)
-	_, err = cnotimeout.Query(query)
 	if err != nil {
 		t.Fatalf("unexpected error.  expected %v, actual %v", nil, err)
 	}

Godeps/_workspace/src/github.com/influxdb/influxdb/cluster/client_pool.go

@@ -0,0 +1,57 @@
+package cluster
+
+import (
+	"net"
+	"sync"
+
+	"gopkg.in/fatih/pool.v2"
+)
+
+type clientPool struct {
+	mu   sync.RWMutex
+	pool map[uint64]pool.Pool
+}
+
+func newClientPool() *clientPool {
+	return &clientPool{
+		pool: make(map[uint64]pool.Pool),
+	}
+}
+
+func (c *clientPool) setPool(nodeID uint64, p pool.Pool) {
+	c.mu.Lock()
+	c.pool[nodeID] = p
+	c.mu.Unlock()
+}
+
+func (c *clientPool) getPool(nodeID uint64) (pool.Pool, bool) {
+	c.mu.RLock()
+	p, ok := c.pool[nodeID]
+	c.mu.RUnlock()
+	return p, ok
+}
+
+func (c *clientPool) size() int {
+	c.mu.RLock()
+	var size int
+	for _, p := range c.pool {
+		size += p.Len()
+	}
+	c.mu.RUnlock()
+	return size
+}
+
+func (c *clientPool) conn(nodeID uint64) (net.Conn, error) {
+	c.mu.RLock()
+	conn, err := c.pool[nodeID].Get()
+	c.mu.RUnlock()
+	return conn, err
+}
+
+func (c *clientPool) close() {
+	c.mu.Lock()
+	for _, p := range c.pool {
+		p.Close()
+	}
+	c.mu.Unlock()
+}

Godeps/_workspace/src/github.com/influxdb/influxdb/cluster/config.go

@@ -0,0 +1,35 @@
+package cluster
+
+import (
+	"time"
+
+	"github.com/influxdb/influxdb/toml"
+)
+
+const (
+	// DefaultWriteTimeout is the default timeout for a complete write to succeed.
+	DefaultWriteTimeout = 5 * time.Second
+
+	// DefaultShardWriterTimeout is the default timeout set on shard writers.
+	DefaultShardWriterTimeout = 5 * time.Second
+
+	// DefaultShardMapperTimeout is the default timeout set on shard mappers.
+	DefaultShardMapperTimeout = 5 * time.Second
+)
+
+// Config represents the configuration for the clustering service.
+type Config struct {
+	ForceRemoteShardMapping bool          `toml:"force-remote-mapping"`
+	WriteTimeout            toml.Duration `toml:"write-timeout"`
+	ShardWriterTimeout      toml.Duration `toml:"shard-writer-timeout"`
+	ShardMapperTimeout      toml.Duration `toml:"shard-mapper-timeout"`
+}
+
+// NewConfig returns an instance of Config with defaults.
+func NewConfig() Config {
+	return Config{
+		WriteTimeout:       toml.Duration(DefaultWriteTimeout),
+		ShardWriterTimeout: toml.Duration(DefaultShardWriterTimeout),
+		ShardMapperTimeout: toml.Duration(DefaultShardMapperTimeout),
+	}
+}

Godeps/_workspace/src/github.com/influxdb/influxdb/cluster/config_test.go

@@ -0,0 +1,27 @@
+package cluster_test
+
+import (
+	"testing"
+	"time"
+
+	"github.com/BurntSushi/toml"
+	"github.com/influxdb/influxdb/cluster"
+)
+
+func TestConfig_Parse(t *testing.T) {
+	// Parse configuration.
+	var c cluster.Config
+	if _, err := toml.Decode(`
+shard-writer-timeout = "10s"
+write-timeout = "20s"
+`, &c); err != nil {
+		t.Fatal(err)
+	}
+
+	// Validate configuration.
+	if time.Duration(c.ShardWriterTimeout) != 10*time.Second {
+		t.Fatalf("unexpected shard-writer timeout: %s", c.ShardWriterTimeout)
+	} else if time.Duration(c.WriteTimeout) != 20*time.Second {
+		t.Fatalf("unexpected write timeout s: %s", c.WriteTimeout)
+	}
+}

Godeps/_workspace/src/github.com/influxdb/influxdb/cluster/internal/data.pb.go

@@ -0,0 +1,154 @@
+// Code generated by protoc-gen-gogo.
+// source: internal/data.proto
+// DO NOT EDIT!
+
+/*
+Package internal is a generated protocol buffer package.
+
+It is generated from these files:
+	internal/data.proto
+
+It has these top-level messages:
+	WriteShardRequest
+	WriteShardResponse
+	MapShardRequest
+	MapShardResponse
+*/
+package internal
+
+import proto "github.com/gogo/protobuf/proto"
+import fmt "fmt"
+import math "math"
+
+// Reference imports to suppress errors if they are not otherwise used.
+var _ = proto.Marshal
+var _ = fmt.Errorf
+var _ = math.Inf
+
+type WriteShardRequest struct {
+	ShardID          *uint64  `protobuf:"varint,1,req,name=ShardID" json:"ShardID,omitempty"`
+	Points           [][]byte `protobuf:"bytes,2,rep,name=Points" json:"Points,omitempty"`
+	XXX_unrecognized []byte   `json:"-"`
+}
+
+func (m *WriteShardRequest) Reset()         { *m = WriteShardRequest{} }
+func (m *WriteShardRequest) String() string { return proto.CompactTextString(m) }
+func (*WriteShardRequest) ProtoMessage()    {}
+
+func (m *WriteShardRequest) GetShardID() uint64 {
+	if m != nil && m.ShardID != nil {
+		return *m.ShardID
+	}
+	return 0
+}
+
+func (m *WriteShardRequest) GetPoints() [][]byte {
+	if m != nil {
+		return m.Points
+	}
+	return nil
+}
+
+type WriteShardResponse struct {
+	Code             *int32  `protobuf:"varint,1,req,name=Code" json:"Code,omitempty"`
+	Message          *string `protobuf:"bytes,2,opt,name=Message" json:"Message,omitempty"`
+	XXX_unrecognized []byte  `json:"-"`
+}
+
+func (m *WriteShardResponse) Reset()         { *m = WriteShardResponse{} }
+func (m *WriteShardResponse) String() string { return proto.CompactTextString(m) }
+func (*WriteShardResponse) ProtoMessage()    {}
+
+func (m *WriteShardResponse) GetCode() int32 {
+	if m != nil && m.Code != nil {
+		return *m.Code
+	}
+	return 0
+}
+
+func (m *WriteShardResponse) GetMessage() string {
+	if m != nil && m.Message != nil {
+		return *m.Message
+	}
+	return ""
+}
+
+type MapShardRequest struct {
+	ShardID          *uint64 `protobuf:"varint,1,req,name=ShardID" json:"ShardID,omitempty"`
+	Query            *string `protobuf:"bytes,2,req,name=Query" json:"Query,omitempty"`
+	ChunkSize        *int32  `protobuf:"varint,3,req,name=ChunkSize" json:"ChunkSize,omitempty"`
+	XXX_unrecognized []byte  `json:"-"`
+}
+
+func (m *MapShardRequest) Reset()         { *m = MapShardRequest{} }
+func (m *MapShardRequest) String() string { return proto.CompactTextString(m) }
+func (*MapShardRequest) ProtoMessage()    {}
+
+func (m *MapShardRequest) GetShardID() uint64 {
+	if m != nil && m.ShardID != nil {
+		return *m.ShardID
+	}
+	return 0
+}
+
+func (m *MapShardRequest) GetQuery() string {
+	if m != nil && m.Query != nil {
+		return *m.Query
+	}
+	return ""
+}
+
+func (m *MapShardRequest) GetChunkSize() int32 {
+	if m != nil && m.ChunkSize != nil {
+		return *m.ChunkSize
+	}
+	return 0
+}
+
+type MapShardResponse struct {
+	Code             *int32   `protobuf:"varint,1,req,name=Code" json:"Code,omitempty"`
+	Message          *string  `protobuf:"bytes,2,opt,name=Message" json:"Message,omitempty"`
+	Data             []byte   `protobuf:"bytes,3,opt,name=Data" json:"Data,omitempty"`
+	TagSets          []string `protobuf:"bytes,4,rep,name=TagSets" json:"TagSets,omitempty"`
+	Fields           []string `protobuf:"bytes,5,rep,name=Fields" json:"Fields,omitempty"`
+	XXX_unrecognized []byte   `json:"-"`
+}
+
+func (m *MapShardResponse) Reset()         { *m = MapShardResponse{} }
+func (m *MapShardResponse) String() string { return proto.CompactTextString(m) }
+func (*MapShardResponse) ProtoMessage()    {}
+
+func (m *MapShardResponse) GetCode() int32 {
+	if m != nil && m.Code != nil {
+		return *m.Code
+	}
+	return 0
+}
+
+func (m *MapShardResponse) GetMessage() string {
+	if m != nil && m.Message != nil {
+		return *m.Message
+	}
+	return ""
+}
+
+func (m *MapShardResponse) GetData() []byte {
+	if m != nil {
+		return m.Data
+	}
+	return nil
+}
+
+func (m *MapShardResponse) GetTagSets() []string {
+	if m != nil {
+		return m.TagSets
+	}
+	return nil
+}
+
+func (m *MapShardResponse) GetFields() []string {
+	if m != nil {
+		return m.Fields
+	}
+	return nil
+}

Godeps/_workspace/src/github.com/influxdb/influxdb/cluster/internal/data.proto

@@ -0,0 +1,25 @@
+package internal;
+
+message WriteShardRequest {
+    required uint64 ShardID = 1;
+    repeated bytes Points = 2;
+}
+
+message WriteShardResponse {
+    required int32 Code = 1;
+    optional string Message = 2;
+}
+
+message MapShardRequest {
+    required uint64 ShardID = 1;
+    required string Query = 2;
+    required int32 ChunkSize = 3;
+}
+
+message MapShardResponse {
+    required int32 Code = 1;
+    optional string Message = 2;
+    optional bytes Data = 3;
+    repeated string TagSets = 4;
+    repeated string Fields = 5;
+}

Godeps/_workspace/src/github.com/influxdb/influxdb/cluster/points_writer.go

@@ -0,0 +1,353 @@
+package cluster
+
+import (
+	"errors"
+	"expvar"
+	"fmt"
+	"log"
+	"os"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/influxdb/influxdb"
+	"github.com/influxdb/influxdb/meta"
+	"github.com/influxdb/influxdb/models"
+	"github.com/influxdb/influxdb/tsdb"
+)
+
+// ConsistencyLevel represent a required replication criteria before a write can
+// be returned as successful
+type ConsistencyLevel int
+
+// The statistics generated by the "write" mdoule
+const (
+	statWriteReq            = "req"
+	statPointWriteReq       = "point_req"
+	statPointWriteReqLocal  = "point_req_local"
+	statPointWriteReqRemote = "point_req_remote"
+	statWriteOK             = "write_ok"
+	statWritePartial        = "write_partial"
+	statWriteTimeout        = "write_timeout"
+	statWriteErr            = "write_error"
+	statWritePointReqHH     = "point_req_hh"
+)
+
+const (
+	// ConsistencyLevelAny allows for hinted hand off, potentially no write happened yet
+	ConsistencyLevelAny ConsistencyLevel = iota
+
+	// ConsistencyLevelOne requires at least one data node acknowledged a write
+	ConsistencyLevelOne
+
+	// ConsistencyLevelQuorum requires a quorum of data nodes to acknowledge a write
+	ConsistencyLevelQuorum
+
+	// ConsistencyLevelAll requires all data nodes to acknowledge a write
+	ConsistencyLevelAll
+)
+
+var (
+	// ErrTimeout is returned when a write times out.
+	ErrTimeout = errors.New("timeout")
+
+	// ErrPartialWrite is returned when a write partially succeeds but does
+	// not meet the requested consistency level.
+	ErrPartialWrite = errors.New("partial write")
+
+	// ErrWriteFailed is returned when no writes succeeded.
+	ErrWriteFailed = errors.New("write failed")
+
+	// ErrInvalidConsistencyLevel is returned when parsing the string version
+	// of a consistency level.
+	ErrInvalidConsistencyLevel = errors.New("invalid consistency level")
+)
+
+// ParseConsistencyLevel converts a consistency level string to the corresponding ConsistencyLevel const
+func ParseConsistencyLevel(level string) (ConsistencyLevel, error) {
+	switch strings.ToLower(level) {
+	case "any":
+		return ConsistencyLevelAny, nil
+	case "one":
+		return ConsistencyLevelOne, nil
+	case "quorum":
+		return ConsistencyLevelQuorum, nil
+	case "all":
+		return ConsistencyLevelAll, nil
+	default:
+		return 0, ErrInvalidConsistencyLevel
+	}
+}
+
+// PointsWriter handles writes across multiple local and remote data nodes.
+type PointsWriter struct {
+	mu           sync.RWMutex
+	closing      chan struct{}
+	WriteTimeout time.Duration
+	Logger       *log.Logger
+
+	MetaStore interface {
+		NodeID() uint64
+		Database(name string) (di *meta.DatabaseInfo, err error)
+		RetentionPolicy(database, policy string) (*meta.RetentionPolicyInfo, error)
+		CreateShardGroupIfNotExists(database, policy string, timestamp time.Time) (*meta.ShardGroupInfo, error)
+		ShardOwner(shardID uint64) (string, string, *meta.ShardGroupInfo)
+	}
+
+	TSDBStore interface {
+		CreateShard(database, retentionPolicy string, shardID uint64) error
+		WriteToShard(shardID uint64, points []models.Point) error
+	}
+
+	ShardWriter interface {
+		WriteShard(shardID, ownerID uint64, points []models.Point) error
+	}
+
+	HintedHandoff interface {
+		WriteShard(shardID, ownerID uint64, points []models.Point) error
+	}
+
+	statMap *expvar.Map
+}
+
+// NewPointsWriter returns a new instance of PointsWriter for a node.
+func NewPointsWriter() *PointsWriter {
+	return &PointsWriter{
+		closing:      make(chan struct{}),
+		WriteTimeout: DefaultWriteTimeout,
+		Logger:       log.New(os.Stderr, "[write] ", log.LstdFlags),
+		statMap:      influxdb.NewStatistics("write", "write", nil),
+	}
+}
+
+// ShardMapping contains a mapping of a shards to a points.
+type ShardMapping struct {
+	Points map[uint64][]models.Point  // The points associated with a shard ID
+	Shards map[uint64]*meta.ShardInfo // The shards that have been mapped, keyed by shard ID
+}
+
+// NewShardMapping creates an empty ShardMapping
+func NewShardMapping() *ShardMapping {
+	return &ShardMapping{
+		Points: map[uint64][]models.Point{},
+		Shards: map[uint64]*meta.ShardInfo{},
+	}
+}
+
+// MapPoint maps a point to shard
+func (s *ShardMapping) MapPoint(shardInfo *meta.ShardInfo, p models.Point) {
+	points, ok := s.Points[shardInfo.ID]
+	if !ok {
+		s.Points[shardInfo.ID] = []models.Point{p}
+	} else {
+		s.Points[shardInfo.ID] = append(points, p)
+	}
+	s.Shards[shardInfo.ID] = shardInfo
+}
+
+// Open opens the communication channel with the point writer
+func (w *PointsWriter) Open() error {
+	w.mu.Lock()
+	defer w.mu.Unlock()
+	if w.closing == nil {
+		w.closing = make(chan struct{})
+	}
+	return nil
+}
+
+// Close closes the communication channel with the point writer
+func (w *PointsWriter) Close() error {
+	w.mu.Lock()
+	defer w.mu.Unlock()
+	if w.closing != nil {
+		close(w.closing)
+		w.closing = nil
+	}
+	return nil
+}
+
+// MapShards maps the points contained in wp to a ShardMapping.  If a point
+// maps to a shard group or shard that does not currently exist, it will be
+// created before returning the mapping.
+func (w *PointsWriter) MapShards(wp *WritePointsRequest) (*ShardMapping, error) {
+
+	// holds the start time ranges for required shard groups
+	timeRanges := map[time.Time]*meta.ShardGroupInfo{}
+
+	rp, err := w.MetaStore.RetentionPolicy(wp.Database, wp.RetentionPolicy)
+	if err != nil {
+		return nil, err
+	}
+	if rp == nil {
+		return nil, influxdb.ErrRetentionPolicyNotFound(wp.RetentionPolicy)
+	}
+
+	for _, p := range wp.Points {
+		timeRanges[p.Time().Truncate(rp.ShardGroupDuration)] = nil
+	}
+
+	// holds all the shard groups and shards that are required for writes
+	for t := range timeRanges {
+		sg, err := w.MetaStore.CreateShardGroupIfNotExists(wp.Database, wp.RetentionPolicy, t)
+		if err != nil {
+			return nil, err
+		}
+		timeRanges[t] = sg
+	}
+
+	mapping := NewShardMapping()
+	for _, p := range wp.Points {
+		sg := timeRanges[p.Time().Truncate(rp.ShardGroupDuration)]
+		sh := sg.ShardFor(p.HashID())
+		mapping.MapPoint(&sh, p)
+	}
+	return mapping, nil
+}
+
+// WritePoints writes across multiple local and remote data nodes according the consistency level.
+func (w *PointsWriter) WritePoints(p *WritePointsRequest) error {
+	w.statMap.Add(statWriteReq, 1)
+	w.statMap.Add(statPointWriteReq, int64(len(p.Points)))
+
+	if p.RetentionPolicy == "" {
+		db, err := w.MetaStore.Database(p.Database)
+		if err != nil {
+			return err
+		} else if db == nil {
+			return influxdb.ErrDatabaseNotFound(p.Database)
+		}
+		p.RetentionPolicy = db.DefaultRetentionPolicy
+	}
+
+	shardMappings, err := w.MapShards(p)
+	if err != nil {
+		return err
+	}
+
+	// Write each shard in it's own goroutine and return as soon
+	// as one fails.
+	ch := make(chan error, len(shardMappings.Points))
+	for shardID, points := range shardMappings.Points {
+		go func(shard *meta.ShardInfo, database, retentionPolicy string, points []models.Point) {
+			ch <- w.writeToShard(shard, p.Database, p.RetentionPolicy, p.ConsistencyLevel, points)
+		}(shardMappings.Shards[shardID], p.Database, p.RetentionPolicy, points)
+	}
+
+	for range shardMappings.Points {
+		select {
+		case <-w.closing:
+			return ErrWriteFailed
+		case err := <-ch:
+			if err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+// writeToShards writes points to a shard and ensures a write consistency level has been met.  If the write
+// partially succeeds, ErrPartialWrite is returned.
+func (w *PointsWriter) writeToShard(shard *meta.ShardInfo, database, retentionPolicy string,
+	consistency ConsistencyLevel, points []models.Point) error {
+	// The required number of writes to achieve the requested consistency level
+	required := len(shard.Owners)
+	switch consistency {
+	case ConsistencyLevelAny, ConsistencyLevelOne:
+		required = 1
+	case ConsistencyLevelQuorum:
+		required = required/2 + 1
+	}
+
+	// response channel for each shard writer go routine
+	type AsyncWriteResult struct {
+		Owner meta.ShardOwner
+		Err   error
+	}
+	ch := make(chan *AsyncWriteResult, len(shard.Owners))
+
+	for _, owner := range shard.Owners {
+		go func(shardID uint64, owner meta.ShardOwner, points []models.Point) {
+			if w.MetaStore.NodeID() == owner.NodeID {
+				w.statMap.Add(statPointWriteReqLocal, int64(len(points)))
+
+				err := w.TSDBStore.WriteToShard(shardID, points)
+				// If we've written to shard that should exist on the current node, but the store has
+				// not actually created this shard, tell it to create it and retry the write
+				if err == tsdb.ErrShardNotFound {
+					err = w.TSDBStore.CreateShard(database, retentionPolicy, shardID)
+					if err != nil {
+						ch <- &AsyncWriteResult{owner, err}
+						return
+					}
+					err = w.TSDBStore.WriteToShard(shardID, points)
+				}
+				ch <- &AsyncWriteResult{owner, err}
+				return
+			}
+
+			w.statMap.Add(statPointWriteReqRemote, int64(len(points)))
+			err := w.ShardWriter.WriteShard(shardID, owner.NodeID, points)
+			if err != nil && tsdb.IsRetryable(err) {
+				// The remote write failed so queue it via hinted handoff
+				w.statMap.Add(statWritePointReqHH, int64(len(points)))
+				hherr := w.HintedHandoff.WriteShard(shardID, owner.NodeID, points)
+
+				// If the write consistency level is ANY, then a successful hinted handoff can
+				// be considered a successful write so send nil to the response channel
+				// otherwise, let the original error propogate to the response channel
+				if hherr == nil && consistency == ConsistencyLevelAny {
+					ch <- &AsyncWriteResult{owner, nil}
+					return
+				}
+			}
+			ch <- &AsyncWriteResult{owner, err}
+
+		}(shard.ID, owner, points)
+	}
+
+	var wrote int
+	timeout := time.After(w.WriteTimeout)
+	var writeError error
+	for range shard.Owners {
+		select {
+		case <-w.closing:
+			return ErrWriteFailed
+		case <-timeout:
+			w.statMap.Add(statWriteTimeout, 1)
+			// return timeout error to caller
+			return ErrTimeout
+		case result := <-ch:
+			// If the write returned an error, continue to the next response
+			if result.Err != nil {
+				w.statMap.Add(statWriteErr, 1)
+				w.Logger.Printf("write failed for shard %d on node %d: %v", shard.ID, result.Owner.NodeID, result.Err)
+
+				// Keep track of the first error we see to return back to the client
+				if writeError == nil {
+					writeError = result.Err
+				}
+				continue
+			}
+
+			wrote++
+
+			// We wrote the required consistency level
+			if wrote >= required {
+				w.statMap.Add(statWriteOK, 1)
+				return nil
+			}
+		}
+	}
+
+	if wrote > 0 {
+		w.statMap.Add(statWritePartial, 1)
+		return ErrPartialWrite
+	}
+
+	if writeError != nil {
+		return fmt.Errorf("write failed: %v", writeError)
+	}
+
+	return ErrWriteFailed
+}

Godeps/_workspace/src/github.com/influxdb/influxdb/cluster/points_writer_test.go

@@ -0,0 +1,464 @@
+package cluster_test
+
+import (
+	"fmt"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/influxdb/influxdb/cluster"
+	"github.com/influxdb/influxdb/meta"
+	"github.com/influxdb/influxdb/models"
+)
+
+// Ensures the points writer maps a single point to a single shard.
+func TestPointsWriter_MapShards_One(t *testing.T) {
+	ms := MetaStore{}
+	rp := NewRetentionPolicy("myp", time.Hour, 3)
+
+	ms.NodeIDFn = func() uint64 { return 1 }
+	ms.RetentionPolicyFn = func(db, retentionPolicy string) (*meta.RetentionPolicyInfo, error) {
+		return rp, nil
+	}
+
+	ms.CreateShardGroupIfNotExistsFn = func(database, policy string, timestamp time.Time) (*meta.ShardGroupInfo, error) {
+		return &rp.ShardGroups[0], nil
+	}
+
+	c := cluster.PointsWriter{MetaStore: ms}
+	pr := &cluster.WritePointsRequest{
+		Database:         "mydb",
+		RetentionPolicy:  "myrp",
+		ConsistencyLevel: cluster.ConsistencyLevelOne,
+	}
+	pr.AddPoint("cpu", 1.0, time.Now(), nil)
+
+	var (
+		shardMappings *cluster.ShardMapping
+		err           error
+	)
+	if shardMappings, err = c.MapShards(pr); err != nil {
+		t.Fatalf("unexpected an error: %v", err)
+	}
+
+	if exp := 1; len(shardMappings.Points) != exp {
+		t.Errorf("MapShards() len mismatch. got %v, exp %v", len(shardMappings.Points), exp)
+	}
+}
+
+// Ensures the points writer maps a multiple points across shard group boundaries.
+func TestPointsWriter_MapShards_Multiple(t *testing.T) {
+	ms := MetaStore{}
+	rp := NewRetentionPolicy("myp", time.Hour, 3)
+	AttachShardGroupInfo(rp, []meta.ShardOwner{
+		{NodeID: 1},
+		{NodeID: 2},
+		{NodeID: 3},
+	})
+	AttachShardGroupInfo(rp, []meta.ShardOwner{
+		{NodeID: 1},
+		{NodeID: 2},
+		{NodeID: 3},
+	})
+
+	ms.NodeIDFn = func() uint64 { return 1 }
+	ms.RetentionPolicyFn = func(db, retentionPolicy string) (*meta.RetentionPolicyInfo, error) {
+		return rp, nil
+	}
+
+	ms.CreateShardGroupIfNotExistsFn = func(database, policy string, timestamp time.Time) (*meta.ShardGroupInfo, error) {
+		for i, sg := range rp.ShardGroups {
+			if timestamp.Equal(sg.StartTime) || timestamp.After(sg.StartTime) && timestamp.Before(sg.EndTime) {
+				return &rp.ShardGroups[i], nil
+			}
+		}
+		panic("should not get here")
+	}
+
+	c := cluster.PointsWriter{MetaStore: ms}
+	pr := &cluster.WritePointsRequest{
+		Database:         "mydb",
+		RetentionPolicy:  "myrp",
+		ConsistencyLevel: cluster.ConsistencyLevelOne,
+	}
+
+	// Three points that range over the shardGroup duration (1h) and should map to two
+	// distinct shards
+	pr.AddPoint("cpu", 1.0, time.Unix(0, 0), nil)
+	pr.AddPoint("cpu", 2.0, time.Unix(0, 0).Add(time.Hour), nil)
+	pr.AddPoint("cpu", 3.0, time.Unix(0, 0).Add(time.Hour+time.Second), nil)
+
+	var (
+		shardMappings *cluster.ShardMapping
+		err           error
+	)
+	if shardMappings, err = c.MapShards(pr); err != nil {
+		t.Fatalf("unexpected an error: %v", err)
+	}
+
+	if exp := 2; len(shardMappings.Points) != exp {
+		t.Errorf("MapShards() len mismatch. got %v, exp %v", len(shardMappings.Points), exp)
+	}
+
+	for _, points := range shardMappings.Points {
+		// First shard shoud have 1 point w/ first point added
+		if len(points) == 1 && points[0].Time() != pr.Points[0].Time() {
+			t.Fatalf("MapShards() value mismatch. got %v, exp %v", points[0].Time(), pr.Points[0].Time())
+		}
+
+		// Second shard shoud have the last two points added
+		if len(points) == 2 && points[0].Time() != pr.Points[1].Time() {
+			t.Fatalf("MapShards() value mismatch. got %v, exp %v", points[0].Time(), pr.Points[1].Time())
+		}
+
+		if len(points) == 2 && points[1].Time() != pr.Points[2].Time() {
+			t.Fatalf("MapShards() value mismatch. got %v, exp %v", points[1].Time(), pr.Points[2].Time())
+		}
+	}
+}
+
+func TestPointsWriter_WritePoints(t *testing.T) {
+	tests := []struct {
+		name            string
+		database        string
+		retentionPolicy string
+		consistency     cluster.ConsistencyLevel
+
+		// the responses returned by each shard write call.  node ID 1 = pos 0
+		err    []error
+		expErr error
+	}{
+		// Consistency one
+		{
+			name:            "write one success",
+			database:        "mydb",
+			retentionPolicy: "myrp",
+			consistency:     cluster.ConsistencyLevelOne,
+			err:             []error{nil, nil, nil},
+			expErr:          nil,
+		},
+		{
+			name:            "write one error",
+			database:        "mydb",
+			retentionPolicy: "myrp",
+			consistency:     cluster.ConsistencyLevelOne,
+			err:             []error{fmt.Errorf("a failure"), fmt.Errorf("a failure"), fmt.Errorf("a failure")},
+			expErr:          fmt.Errorf("write failed: a failure"),
+		},
+
+		// Consistency any
+		{
+			name:            "write any success",
+			database:        "mydb",
+			retentionPolicy: "myrp",
+			consistency:     cluster.ConsistencyLevelAny,
+			err:             []error{fmt.Errorf("a failure"), nil, fmt.Errorf("a failure")},
+			expErr:          nil,
+		},
+		// Consistency all
+		{
+			name:            "write all success",
+			database:        "mydb",
+			retentionPolicy: "myrp",
+			consistency:     cluster.ConsistencyLevelAll,
+			err:             []error{nil, nil, nil},
+			expErr:          nil,
+		},
+		{
+			name:            "write all, 2/3, partial write",
+			database:        "mydb",
+			retentionPolicy: "myrp",
+			consistency:     cluster.ConsistencyLevelAll,
+			err:             []error{nil, fmt.Errorf("a failure"), nil},
+			expErr:          cluster.ErrPartialWrite,
+		},
+		{
+			name:            "write all, 1/3 (failure)",
+			database:        "mydb",
+			retentionPolicy: "myrp",
+			consistency:     cluster.ConsistencyLevelAll,
+			err:             []error{nil, fmt.Errorf("a failure"), fmt.Errorf("a failure")},
+			expErr:          cluster.ErrPartialWrite,
+		},
+
+		// Consistency quorum
+		{
+			name:            "write quorum, 1/3 failure",
+			consistency:     cluster.ConsistencyLevelQuorum,
+			database:        "mydb",
+			retentionPolicy: "myrp",
+			err:             []error{fmt.Errorf("a failure"), fmt.Errorf("a failure"), nil},
+			expErr:          cluster.ErrPartialWrite,
+		},
+		{
+			name:            "write quorum, 2/3 success",
+			database:        "mydb",
+			retentionPolicy: "myrp",
+			consistency:     cluster.ConsistencyLevelQuorum,
+			err:             []error{nil, nil, fmt.Errorf("a failure")},
+			expErr:          nil,
+		},
+		{
+			name:            "write quorum, 3/3 success",
+			database:        "mydb",
+			retentionPolicy: "myrp",
+			consistency:     cluster.ConsistencyLevelQuorum,
+			err:             []error{nil, nil, nil},
+			expErr:          nil,
+		},
+
+		// Error write error
+		{
+			name:            "no writes succeed",
+			database:        "mydb",
+			retentionPolicy: "myrp",
+			consistency:     cluster.ConsistencyLevelOne,
+			err:             []error{fmt.Errorf("a failure"), fmt.Errorf("a failure"), fmt.Errorf("a failure")},
+			expErr:          fmt.Errorf("write failed: a failure"),
+		},
+
+		// Hinted handoff w/ ANY
+		{
+			name:            "hinted handoff write succeed",
+			database:        "mydb",
+			retentionPolicy: "myrp",
+			consistency:     cluster.ConsistencyLevelAny,
+			err:             []error{fmt.Errorf("a failure"), fmt.Errorf("a failure"), fmt.Errorf("a failure")},
+			expErr:          nil,
+		},
+
+		// Write to non-existant database
+		{
+			name:            "write to non-existant database",
+			database:        "doesnt_exist",
+			retentionPolicy: "",
+			consistency:     cluster.ConsistencyLevelAny,
+			err:             []error{nil, nil, nil},
+			expErr:          fmt.Errorf("database not found: doesnt_exist"),
+		},
+	}
+
+	for _, test := range tests {
+
+		pr := &cluster.WritePointsRequest{
+			Database:         test.database,
+			RetentionPolicy:  test.retentionPolicy,
+			ConsistencyLevel: test.consistency,
+		}
+
+		// Three points that range over the shardGroup duration (1h) and should map to two
+		// distinct shards
+		pr.AddPoint("cpu", 1.0, time.Unix(0, 0), nil)
+		pr.AddPoint("cpu", 2.0, time.Unix(0, 0).Add(time.Hour), nil)
+		pr.AddPoint("cpu", 3.0, time.Unix(0, 0).Add(time.Hour+time.Second), nil)
+
+		// copy to prevent data race
+		theTest := test
+		sm := cluster.NewShardMapping()
+		sm.MapPoint(
+			&meta.ShardInfo{ID: uint64(1), Owners: []meta.ShardOwner{
+				{NodeID: 1},
+				{NodeID: 2},
+				{NodeID: 3},
+			}},
+			pr.Points[0])
+		sm.MapPoint(
+			&meta.ShardInfo{ID: uint64(2), Owners: []meta.ShardOwner{
+				{NodeID: 1},
+				{NodeID: 2},
+				{NodeID: 3},
+			}},
+			pr.Points[1])
+		sm.MapPoint(
+			&meta.ShardInfo{ID: uint64(2), Owners: []meta.ShardOwner{
+				{NodeID: 1},
+				{NodeID: 2},
+				{NodeID: 3},
+			}},
+			pr.Points[2])
+
+		// Local cluster.Node ShardWriter
+		// lock on the write increment since these functions get called in parallel
+		var mu sync.Mutex
+		sw := &fakeShardWriter{
+			ShardWriteFn: func(shardID, nodeID uint64, points []models.Point) error {
+				mu.Lock()
+				defer mu.Unlock()
+				return theTest.err[int(nodeID)-1]
+			},
+		}
+
+		store := &fakeStore{
+			WriteFn: func(shardID uint64, points []models.Point) error {
+				mu.Lock()
+				defer mu.Unlock()
+				return theTest.err[0]
+			},
+		}
+
+		hh := &fakeShardWriter{
+			ShardWriteFn: func(shardID, nodeID uint64, points []models.Point) error {
+				return nil
+			},
+		}
+
+		ms := NewMetaStore()
+		ms.DatabaseFn = func(database string) (*meta.DatabaseInfo, error) {
+			return nil, nil
+		}
+		ms.NodeIDFn = func() uint64 { return 1 }
+		c := cluster.NewPointsWriter()
+		c.MetaStore = ms
+		c.ShardWriter = sw
+		c.TSDBStore = store
+		c.HintedHandoff = hh
+
+		err := c.WritePoints(pr)
+		if err == nil && test.expErr != nil {
+			t.Errorf("PointsWriter.WritePoints(): '%s' error: got %v, exp %v", test.name, err, test.expErr)
+		}
+
+		if err != nil && test.expErr == nil {
+			t.Errorf("PointsWriter.WritePoints(): '%s' error: got %v, exp %v", test.name, err, test.expErr)
+		}
+		if err != nil && test.expErr != nil && err.Error() != test.expErr.Error() {
+			t.Errorf("PointsWriter.WritePoints(): '%s' error: got %v, exp %v", test.name, err, test.expErr)
+		}
+	}
+}
+
+var shardID uint64
+
+type fakeShardWriter struct {
+	ShardWriteFn func(shardID, nodeID uint64, points []models.Point) error
+}
+
+func (f *fakeShardWriter) WriteShard(shardID, nodeID uint64, points []models.Point) error {
+	return f.ShardWriteFn(shardID, nodeID, points)
+}
+
+type fakeStore struct {
+	WriteFn       func(shardID uint64, points []models.Point) error
+	CreateShardfn func(database, retentionPolicy string, shardID uint64) error
+}
+
+func (f *fakeStore) WriteToShard(shardID uint64, points []models.Point) error {
+	return f.WriteFn(shardID, points)
+}
+
+func (f *fakeStore) CreateShard(database, retentionPolicy string, shardID uint64) error {
+	return f.CreateShardfn(database, retentionPolicy, shardID)
+}
+
+func NewMetaStore() *MetaStore {
+	ms := &MetaStore{}
+	rp := NewRetentionPolicy("myp", time.Hour, 3)
+	AttachShardGroupInfo(rp, []meta.ShardOwner{
+		{NodeID: 1},
+		{NodeID: 2},
+		{NodeID: 3},
+	})
+	AttachShardGroupInfo(rp, []meta.ShardOwner{
+		{NodeID: 1},
+		{NodeID: 2},
+		{NodeID: 3},
+	})
+
+	ms.RetentionPolicyFn = func(db, retentionPolicy string) (*meta.RetentionPolicyInfo, error) {
+		return rp, nil
+	}
+
+	ms.CreateShardGroupIfNotExistsFn = func(database, policy string, timestamp time.Time) (*meta.ShardGroupInfo, error) {
+		for i, sg := range rp.ShardGroups {
+			if timestamp.Equal(sg.StartTime) || timestamp.After(sg.StartTime) && timestamp.Before(sg.EndTime) {
+				return &rp.ShardGroups[i], nil
+			}
+		}
+		panic("should not get here")
+	}
+	return ms
+}
+
+type MetaStore struct {
+	NodeIDFn                      func() uint64
+	RetentionPolicyFn             func(database, name string) (*meta.RetentionPolicyInfo, error)
+	CreateShardGroupIfNotExistsFn func(database, policy string, timestamp time.Time) (*meta.ShardGroupInfo, error)
+	DatabaseFn                    func(database string) (*meta.DatabaseInfo, error)
+	ShardOwnerFn                  func(shardID uint64) (string, string, *meta.ShardGroupInfo)
+}
+
+func (m MetaStore) NodeID() uint64 { return m.NodeIDFn() }
+
+func (m MetaStore) RetentionPolicy(database, name string) (*meta.RetentionPolicyInfo, error) {
+	return m.RetentionPolicyFn(database, name)
+}
+
+func (m MetaStore) CreateShardGroupIfNotExists(database, policy string, timestamp time.Time) (*meta.ShardGroupInfo, error) {
+	return m.CreateShardGroupIfNotExistsFn(database, policy, timestamp)
+}
+
+func (m MetaStore) Database(database string) (*meta.DatabaseInfo, error) {
+	return m.DatabaseFn(database)
+}
+
+func (m MetaStore) ShardOwner(shardID uint64) (string, string, *meta.ShardGroupInfo) {
+	return m.ShardOwnerFn(shardID)
+}
+
+func NewRetentionPolicy(name string, duration time.Duration, nodeCount int) *meta.RetentionPolicyInfo {
+	shards := []meta.ShardInfo{}
+	owners := []meta.ShardOwner{}
+	for i := 1; i <= nodeCount; i++ {
+		owners = append(owners, meta.ShardOwner{NodeID: uint64(i)})
+	}
+
+	// each node is fully replicated with each other
+	shards = append(shards, meta.ShardInfo{
+		ID:     nextShardID(),
+		Owners: owners,
+	})
+
+	rp := &meta.RetentionPolicyInfo{
+		Name:               "myrp",
+		ReplicaN:           nodeCount,
+		Duration:           duration,
+		ShardGroupDuration: duration,
+		ShardGroups: []meta.ShardGroupInfo{
+			meta.ShardGroupInfo{
+				ID:        nextShardID(),
+				StartTime: time.Unix(0, 0),
+				EndTime:   time.Unix(0, 0).Add(duration).Add(-1),
+				Shards:    shards,
+			},
+		},
+	}
+	return rp
+}
+
+func AttachShardGroupInfo(rp *meta.RetentionPolicyInfo, owners []meta.ShardOwner) {
+	var startTime, endTime time.Time
+	if len(rp.ShardGroups) == 0 {
+		startTime = time.Unix(0, 0)
+	} else {
+		startTime = rp.ShardGroups[len(rp.ShardGroups)-1].StartTime.Add(rp.ShardGroupDuration)
+	}
+	endTime = startTime.Add(rp.ShardGroupDuration).Add(-1)
+
+	sh := meta.ShardGroupInfo{
+		ID:        uint64(len(rp.ShardGroups) + 1),
+		StartTime: startTime,
+		EndTime:   endTime,
+		Shards: []meta.ShardInfo{
+			meta.ShardInfo{
+				ID:     nextShardID(),
+				Owners: owners,
+			},
+		},
+	}
+	rp.ShardGroups = append(rp.ShardGroups, sh)
+}
+
+func nextShardID() uint64 {
+	return atomic.AddUint64(&shardID, 1)
+}

Godeps/_workspace/src/github.com/influxdb/influxdb/cluster/rpc.go

@@ -0,0 +1,205 @@
+package cluster
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/gogo/protobuf/proto"
+	"github.com/influxdb/influxdb/cluster/internal"
+	"github.com/influxdb/influxdb/models"
+)
+
+//go:generate protoc --gogo_out=. internal/data.proto
+
+// MapShardRequest represents the request to map a remote shard for a query.
+type MapShardRequest struct {
+	pb internal.MapShardRequest
+}
+
+// ShardID of the map request
+func (m *MapShardRequest) ShardID() uint64 { return m.pb.GetShardID() }
+
+// Query returns the Shard map request's query
+func (m *MapShardRequest) Query() string { return m.pb.GetQuery() }
+
+// ChunkSize returns Shard map request's chunk size
+func (m *MapShardRequest) ChunkSize() int32 { return m.pb.GetChunkSize() }
+
+// SetShardID sets the map request's shard id
+func (m *MapShardRequest) SetShardID(id uint64) { m.pb.ShardID = &id }
+
+// SetQuery sets the Shard map request's Query
+func (m *MapShardRequest) SetQuery(query string) { m.pb.Query = &query }
+
+// SetChunkSize sets the Shard map request's chunk size
+func (m *MapShardRequest) SetChunkSize(chunkSize int32) { m.pb.ChunkSize = &chunkSize }
+
+// MarshalBinary encodes the object to a binary format.
+func (m *MapShardRequest) MarshalBinary() ([]byte, error) {
+	return proto.Marshal(&m.pb)
+}
+
+// UnmarshalBinary populates MapShardRequest from a binary format.
+func (m *MapShardRequest) UnmarshalBinary(buf []byte) error {
+	if err := proto.Unmarshal(buf, &m.pb); err != nil {
+		return err
+	}
+	return nil
+}
+
+// MapShardResponse represents the response returned from a remote MapShardRequest call
+type MapShardResponse struct {
+	pb internal.MapShardResponse
+}
+
+// NewMapShardResponse returns the response returned from a remote MapShardRequest call
+func NewMapShardResponse(code int, message string) *MapShardResponse {
+	m := &MapShardResponse{}
+	m.SetCode(code)
+	m.SetMessage(message)
+	return m
+}
+
+// Code returns the Shard map response's code
+func (r *MapShardResponse) Code() int { return int(r.pb.GetCode()) }
+
+// Message returns the the Shard map response's Message
+func (r *MapShardResponse) Message() string { return r.pb.GetMessage() }
+
+// TagSets returns Shard map response's tag sets
+func (r *MapShardResponse) TagSets() []string { return r.pb.GetTagSets() }
+
+// Fields returns the Shard map response's Fields
+func (r *MapShardResponse) Fields() []string { return r.pb.GetFields() }
+
+// Data returns the Shard map response's Data
+func (r *MapShardResponse) Data() []byte { return r.pb.GetData() }
+
+// SetCode sets the Shard map response's code
+func (r *MapShardResponse) SetCode(code int) { r.pb.Code = proto.Int32(int32(code)) }
+
+// SetMessage sets Shard map response's message
+func (r *MapShardResponse) SetMessage(message string) { r.pb.Message = &message }
+
+// SetTagSets sets Shard map response's tagsets
+func (r *MapShardResponse) SetTagSets(tagsets []string) { r.pb.TagSets = tagsets }
+
+// SetFields sets the Shard map response's Fields
+func (r *MapShardResponse) SetFields(fields []string) { r.pb.Fields = fields }
+
+// SetData sets the Shard map response's Data
+func (r *MapShardResponse) SetData(data []byte) { r.pb.Data = data }
+
+// MarshalBinary encodes the object to a binary format.
+func (r *MapShardResponse) MarshalBinary() ([]byte, error) {
+	return proto.Marshal(&r.pb)
+}
+
+// UnmarshalBinary populates WritePointRequest from a binary format.
+func (r *MapShardResponse) UnmarshalBinary(buf []byte) error {
+	if err := proto.Unmarshal(buf, &r.pb); err != nil {
+		return err
+	}
+	return nil
+}
+
+// WritePointsRequest represents a request to write point data to the cluster
+type WritePointsRequest struct {
+	Database         string
+	RetentionPolicy  string
+	ConsistencyLevel ConsistencyLevel
+	Points           []models.Point
+}
+
+// AddPoint adds a point to the WritePointRequest with field name 'value'
+func (w *WritePointsRequest) AddPoint(name string, value interface{}, timestamp time.Time, tags map[string]string) {
+	w.Points = append(w.Points, models.NewPoint(
+		name, tags, map[string]interface{}{"value": value}, timestamp,
+	))
+}
+
+// WriteShardRequest represents the a request to write a slice of points to a shard
+type WriteShardRequest struct {
+	pb internal.WriteShardRequest
+}
+
+// WriteShardResponse represents the response returned from a remote WriteShardRequest call
+type WriteShardResponse struct {
+	pb internal.WriteShardResponse
+}
+
+// SetShardID sets the ShardID
+func (w *WriteShardRequest) SetShardID(id uint64) { w.pb.ShardID = &id }
+
+// ShardID gets the ShardID
+func (w *WriteShardRequest) ShardID() uint64 { return w.pb.GetShardID() }
+
+// Points returns the time series Points
+func (w *WriteShardRequest) Points() []models.Point { return w.unmarshalPoints() }
+
+// AddPoint adds a new time series point
+func (w *WriteShardRequest) AddPoint(name string, value interface{}, timestamp time.Time, tags map[string]string) {
+	w.AddPoints([]models.Point{models.NewPoint(
+		name, tags, map[string]interface{}{"value": value}, timestamp,
+	)})
+}
+
+// AddPoints adds a new time series point
+func (w *WriteShardRequest) AddPoints(points []models.Point) {
+	for _, p := range points {
+		w.pb.Points = append(w.pb.Points, []byte(p.String()))
+	}
+}
+
+// MarshalBinary encodes the object to a binary format.
+func (w *WriteShardRequest) MarshalBinary() ([]byte, error) {
+	return proto.Marshal(&w.pb)
+}
+
+// UnmarshalBinary populates WritePointRequest from a binary format.
+func (w *WriteShardRequest) UnmarshalBinary(buf []byte) error {
+	if err := proto.Unmarshal(buf, &w.pb); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (w *WriteShardRequest) unmarshalPoints() []models.Point {
+	points := make([]models.Point, len(w.pb.GetPoints()))
+	for i, p := range w.pb.GetPoints() {
+		pt, err := models.ParsePoints(p)
+		if err != nil {
+			// A error here means that one node parsed the point correctly but sent an
+			// unparseable version to another node.  We could log and drop the point and allow
+			// anti-entropy to resolve the discrepancy but this shouldn't ever happen.
+			panic(fmt.Sprintf("failed to parse point: `%v`: %v", string(p), err))
+		}
+		points[i] = pt[0]
+	}
+	return points
+}
+
+// SetCode sets the Code
+func (w *WriteShardResponse) SetCode(code int) { w.pb.Code = proto.Int32(int32(code)) }
+
+// SetMessage sets the Message
+func (w *WriteShardResponse) SetMessage(message string) { w.pb.Message = &message }
+
+// Code returns the Code
+func (w *WriteShardResponse) Code() int { return int(w.pb.GetCode()) }
+
+// Message returns the Message
+func (w *WriteShardResponse) Message() string { return w.pb.GetMessage() }
+
+// MarshalBinary encodes the object to a binary format.
+func (w *WriteShardResponse) MarshalBinary() ([]byte, error) {
+	return proto.Marshal(&w.pb)
+}
+
+// UnmarshalBinary populates WritePointRequest from a binary format.
+func (w *WriteShardResponse) UnmarshalBinary(buf []byte) error {
+	if err := proto.Unmarshal(buf, &w.pb); err != nil {
+		return err
+	}
+	return nil
+}

Godeps/_workspace/src/github.com/influxdb/influxdb/cluster/rpc_test.go

@@ -0,0 +1,110 @@
+package cluster
+
+import (
+	"testing"
+	"time"
+)
+
+func TestWriteShardRequestBinary(t *testing.T) {
+	sr := &WriteShardRequest{}
+
+	sr.SetShardID(uint64(1))
+	if exp := uint64(1); sr.ShardID() != exp {
+		t.Fatalf("ShardID mismatch: got %v, exp %v", sr.ShardID(), exp)
+	}
+
+	sr.AddPoint("cpu", 1.0, time.Unix(0, 0), map[string]string{"host": "serverA"})
+	sr.AddPoint("cpu", 2.0, time.Unix(0, 0).Add(time.Hour), nil)
+	sr.AddPoint("cpu_load", 3.0, time.Unix(0, 0).Add(time.Hour+time.Second), nil)
+
+	b, err := sr.MarshalBinary()
+	if err != nil {
+		t.Fatalf("WritePointsRequest.MarshalBinary() failed: %v", err)
+	}
+	if len(b) == 0 {
+		t.Fatalf("WritePointsRequest.MarshalBinary() returned 0 bytes")
+	}
+
+	got := &WriteShardRequest{}
+	if err := got.UnmarshalBinary(b); err != nil {
+		t.Fatalf("WritePointsRequest.UnmarshalMarshalBinary() failed: %v", err)
+	}
+
+	if got.ShardID() != sr.ShardID() {
+		t.Errorf("ShardID mismatch: got %v, exp %v", got.ShardID(), sr.ShardID())
+	}
+
+	if len(got.Points()) != len(sr.Points()) {
+		t.Errorf("Points count mismatch: got %v, exp %v", len(got.Points()), len(sr.Points()))
+	}
+
+	srPoints := sr.Points()
+	gotPoints := got.Points()
+	for i, p := range srPoints {
+		g := gotPoints[i]
+
+		if g.Name() != p.Name() {
+			t.Errorf("Point %d name mismatch: got %v, exp %v", i, g.Name(), p.Name())
+		}
+
+		if !g.Time().Equal(p.Time()) {
+			t.Errorf("Point %d time mismatch: got %v, exp %v", i, g.Time(), p.Time())
+		}
+
+		if g.HashID() != p.HashID() {
+			t.Errorf("Point #%d HashID() mismatch: got %v, exp %v", i, g.HashID(), p.HashID())
+		}
+
+		for k, v := range p.Tags() {
+			if g.Tags()[k] != v {
+				t.Errorf("Point #%d tag mismatch: got %v, exp %v", i, k, v)
+			}
+		}
+
+		if len(p.Fields()) != len(g.Fields()) {
+			t.Errorf("Point %d field count mismatch: got %v, exp %v", i, len(g.Fields()), len(p.Fields()))
+		}
+
+		for j, f := range p.Fields() {
+			if g.Fields()[j] != f {
+				t.Errorf("Point %d field mismatch: got %v, exp %v", i, g.Fields()[j], f)
+			}
+		}
+	}
+}
+
+func TestWriteShardResponseBinary(t *testing.T) {
+	sr := &WriteShardResponse{}
+	sr.SetCode(10)
+	sr.SetMessage("foo")
+	b, err := sr.MarshalBinary()
+
+	if exp := 10; sr.Code() != exp {
+		t.Fatalf("Code mismatch: got %v, exp %v", sr.Code(), exp)
+	}
+
+	if exp := "foo"; sr.Message() != exp {
+		t.Fatalf("Message mismatch: got %v, exp %v", sr.Message(), exp)
+	}
+
+	if err != nil {
+		t.Fatalf("WritePointsResponse.MarshalBinary() failed: %v", err)
+	}
+	if len(b) == 0 {
+		t.Fatalf("WritePointsResponse.MarshalBinary() returned 0 bytes")
+	}
+
+	got := &WriteShardResponse{}
+	if err := got.UnmarshalBinary(b); err != nil {
+		t.Fatalf("WritePointsResponse.UnmarshalMarshalBinary() failed: %v", err)
+	}
+
+	if got.Code() != sr.Code() {
+		t.Errorf("Code mismatch: got %v, exp %v", got.Code(), sr.Code())
+	}
+
+	if got.Message() != sr.Message() {
+		t.Errorf("Message mismatch: got %v, exp %v", got.Message(), sr.Message())
+	}
+
+}

Godeps/_workspace/src/github.com/influxdb/influxdb/cluster/service.go

@@ -0,0 +1,371 @@
+package cluster
+
+import (
+	"encoding/binary"
+	"encoding/json"
+	"expvar"
+	"fmt"
+	"io"
+	"log"
+	"net"
+	"os"
+	"strings"
+	"sync"
+
+	"github.com/influxdb/influxdb"
+	"github.com/influxdb/influxdb/influxql"
+	"github.com/influxdb/influxdb/meta"
+	"github.com/influxdb/influxdb/models"
+	"github.com/influxdb/influxdb/tsdb"
+)
+
+// MaxMessageSize defines how large a message can be before we reject it
+const MaxMessageSize = 1024 * 1024 * 1024 // 1GB
+
+// MuxHeader is the header byte used in the TCP mux.
+const MuxHeader = 2
+
+// Statistics maintained by the cluster package
+const (
+	writeShardReq       = "write_shard_req"
+	writeShardPointsReq = "write_shard_points_req"
+	writeShardFail      = "write_shard_fail"
+	mapShardReq         = "map_shard_req"
+	mapShardResp        = "map_shard_resp"
+)
+
+// Service processes data received over raw TCP connections.
+type Service struct {
+	mu sync.RWMutex
+
+	wg      sync.WaitGroup
+	closing chan struct{}
+
+	Listener net.Listener
+
+	MetaStore interface {
+		ShardOwner(shardID uint64) (string, string, *meta.ShardGroupInfo)
+	}
+
+	TSDBStore interface {
+		CreateShard(database, policy string, shardID uint64) error
+		WriteToShard(shardID uint64, points []models.Point) error
+		CreateMapper(shardID uint64, stmt influxql.Statement, chunkSize int) (tsdb.Mapper, error)
+	}
+
+	Logger  *log.Logger
+	statMap *expvar.Map
+}
+
+// NewService returns a new instance of Service.
+func NewService(c Config) *Service {
+	return &Service{
+		closing: make(chan struct{}),
+		Logger:  log.New(os.Stderr, "[tcp] ", log.LstdFlags),
+		statMap: influxdb.NewStatistics("cluster", "cluster", nil),
+	}
+}
+
+// Open opens the network listener and begins serving requests.
+func (s *Service) Open() error {
+
+	s.Logger.Println("Starting cluster service")
+	// Begin serving conections.
+	s.wg.Add(1)
+	go s.serve()
+
+	return nil
+}
+
+// SetLogger sets the internal logger to the logger passed in.
+func (s *Service) SetLogger(l *log.Logger) {
+	s.Logger = l
+}
+
+// serve accepts connections from the listener and handles them.
+func (s *Service) serve() {
+	defer s.wg.Done()
+
+	for {
+		// Check if the service is shutting down.
+		select {
+		case <-s.closing:
+			return
+		default:
+		}
+
+		// Accept the next connection.
+		conn, err := s.Listener.Accept()
+		if err != nil {
+			if strings.Contains(err.Error(), "connection closed") {
+				s.Logger.Printf("cluster service accept error: %s", err)
+				return
+			}
+			s.Logger.Printf("accept error: %s", err)
+			continue
+		}
+
+		// Delegate connection handling to a separate goroutine.
+		s.wg.Add(1)
+		go func() {
+			defer s.wg.Done()
+			s.handleConn(conn)
+		}()
+	}
+}
+
+// Close shuts down the listener and waits for all connections to finish.
+func (s *Service) Close() error {
+	if s.Listener != nil {
+		s.Listener.Close()
+	}
+
+	// Shut down all handlers.
+	close(s.closing)
+	s.wg.Wait()
+
+	return nil
+}
+
+// handleConn services an individual TCP connection.
+func (s *Service) handleConn(conn net.Conn) {
+	// Ensure connection is closed when service is closed.
+	closing := make(chan struct{})
+	defer close(closing)
+	go func() {
+		select {
+		case <-closing:
+		case <-s.closing:
+		}
+		conn.Close()
+	}()
+
+	s.Logger.Printf("accept remote connection from %v\n", conn.RemoteAddr())
+	defer func() {
+		s.Logger.Printf("close remote connection from %v\n", conn.RemoteAddr())
+	}()
+	for {
+		// Read type-length-value.
+		typ, buf, err := ReadTLV(conn)
+		if err != nil {
+			if strings.HasSuffix(err.Error(), "EOF") {
+				return
+			}
+			s.Logger.Printf("unable to read type-length-value %s", err)
+			return
+		}
+
+		// Delegate message processing by type.
+		switch typ {
+		case writeShardRequestMessage:
+			s.statMap.Add(writeShardReq, 1)
+			err := s.processWriteShardRequest(buf)
+			if err != nil {
+				s.Logger.Printf("process write shard error: %s", err)
+			}
+			s.writeShardResponse(conn, err)
+		case mapShardRequestMessage:
+			s.statMap.Add(mapShardReq, 1)
+			err := s.processMapShardRequest(conn, buf)
+			if err != nil {
+				s.Logger.Printf("process map shard error: %s", err)
+				if err := writeMapShardResponseMessage(conn, NewMapShardResponse(1, err.Error())); err != nil {
+					s.Logger.Printf("process map shard error writing response: %s", err.Error())
+				}
+			}
+		default:
+			s.Logger.Printf("cluster service message type not found: %d", typ)
+		}
+	}
+}
+
+func (s *Service) processWriteShardRequest(buf []byte) error {
+	// Build request
+	var req WriteShardRequest
+	if err := req.UnmarshalBinary(buf); err != nil {
+		return err
+	}
+
+	points := req.Points()
+	s.statMap.Add(writeShardPointsReq, int64(len(points)))
+	err := s.TSDBStore.WriteToShard(req.ShardID(), req.Points())
+
+	// We may have received a write for a shard that we don't have locally because the
+	// sending node may have just created the shard (via the metastore) and the write
+	// arrived before the local store could create the shard.  In this case, we need
+	// to check the metastore to determine what database and retention policy this
+	// shard should reside within.
+	if err == tsdb.ErrShardNotFound {
+
+		// Query the metastore for the owner of this shard
+		database, retentionPolicy, sgi := s.MetaStore.ShardOwner(req.ShardID())
+		if sgi == nil {
+			// If we can't find it, then we need to drop this request
+			// as it is no longer valid.  This could happen if writes were queued via
+			// hinted handoff and delivered after a shard group was deleted.
+			s.Logger.Printf("drop write request: shard=%d. shard group does not exist or was deleted", req.ShardID())
+			return nil
+		}
+
+		err = s.TSDBStore.CreateShard(database, retentionPolicy, req.ShardID())
+		if err != nil {
+			return err
+		}
+		return s.TSDBStore.WriteToShard(req.ShardID(), req.Points())
+	}
+
+	if err != nil {
+		s.statMap.Add(writeShardFail, 1)
+		return fmt.Errorf("write shard %d: %s", req.ShardID(), err)
+	}
+
+	return nil
+}
+
+func (s *Service) writeShardResponse(w io.Writer, e error) {
+	// Build response.
+	var resp WriteShardResponse
+	if e != nil {
+		resp.SetCode(1)
+		resp.SetMessage(e.Error())
+	} else {
+		resp.SetCode(0)
+	}
+
+	// Marshal response to binary.
+	buf, err := resp.MarshalBinary()
+	if err != nil {
+		s.Logger.Printf("error marshalling shard response: %s", err)
+		return
+	}
+
+	// Write to connection.
+	if err := WriteTLV(w, writeShardResponseMessage, buf); err != nil {
+		s.Logger.Printf("write shard response error: %s", err)
+	}
+}
+
+func (s *Service) processMapShardRequest(w io.Writer, buf []byte) error {
+	// Decode request
+	var req MapShardRequest
+	if err := req.UnmarshalBinary(buf); err != nil {
+		return err
+	}
+
+	// Parse the statement.
+	q, err := influxql.ParseQuery(req.Query())
+	if err != nil {
+		return fmt.Errorf("processing map shard: %s", err)
+	} else if len(q.Statements) != 1 {
+		return fmt.Errorf("processing map shard: expected 1 statement but got %d", len(q.Statements))
+	}
+
+	m, err := s.TSDBStore.CreateMapper(req.ShardID(), q.Statements[0], int(req.ChunkSize()))
+	if err != nil {
+		return fmt.Errorf("create mapper: %s", err)
+	}
+	if m == nil {
+		return writeMapShardResponseMessage(w, NewMapShardResponse(0, ""))
+	}
+
+	if err := m.Open(); err != nil {
+		return fmt.Errorf("mapper open: %s", err)
+	}
+	defer m.Close()
+
+	var metaSent bool
+	for {
+		var resp MapShardResponse
+
+		if !metaSent {
+			resp.SetTagSets(m.TagSets())
+			resp.SetFields(m.Fields())
+			metaSent = true
+		}
+
+		chunk, err := m.NextChunk()
+		if err != nil {
+			return fmt.Errorf("next chunk: %s", err)
+		}
+
+		// NOTE: Even if the chunk is nil, we still need to send one
+		// empty response to let the other side know we're out of data.
+
+		if chunk != nil {
+			b, err := json.Marshal(chunk)
+			if err != nil {
+				return fmt.Errorf("encoding: %s", err)
+			}
+			resp.SetData(b)
+		}
+
+		// Write to connection.
+		resp.SetCode(0)
+		if err := writeMapShardResponseMessage(w, &resp); err != nil {
+			return err
+		}
+		s.statMap.Add(mapShardResp, 1)
+
+		if chunk == nil {
+			// All mapper data sent.
+			return nil
+		}
+	}
+}
+
+func writeMapShardResponseMessage(w io.Writer, msg *MapShardResponse) error {
+	buf, err := msg.MarshalBinary()
+	if err != nil {
+		return err
+	}
+	return WriteTLV(w, mapShardResponseMessage, buf)
+}
+
+// ReadTLV reads a type-length-value record from r.
+func ReadTLV(r io.Reader) (byte, []byte, error) {
+	var typ [1]byte
+	if _, err := io.ReadFull(r, typ[:]); err != nil {
+		return 0, nil, fmt.Errorf("read message type: %s", err)
+	}
+
+	// Read the size of the message.
+	var sz int64
+	if err := binary.Read(r, binary.BigEndian, &sz); err != nil {
+		return 0, nil, fmt.Errorf("read message size: %s", err)
+	}
+
+	if sz == 0 {
+		return 0, nil, fmt.Errorf("invalid message size: %d", sz)
+	}
+
+	if sz >= MaxMessageSize {
+		return 0, nil, fmt.Errorf("max message size of %d exceeded: %d", MaxMessageSize, sz)
+	}
+
+	// Read the value.
+	buf := make([]byte, sz)
+	if _, err := io.ReadFull(r, buf); err != nil {
+		return 0, nil, fmt.Errorf("read message value: %s", err)
+	}
+
+	return typ[0], buf, nil
+}
+
+// WriteTLV writes a type-length-value record to w.
+func WriteTLV(w io.Writer, typ byte, buf []byte) error {
+	if _, err := w.Write([]byte{typ}); err != nil {
+		return fmt.Errorf("write message type: %s", err)
+	}
+
+	// Write the size of the message.
+	if err := binary.Write(w, binary.BigEndian, int64(len(buf))); err != nil {
+		return fmt.Errorf("write message size: %s", err)
+	}
+
+	// Write the value.
+	if _, err := w.Write(buf); err != nil {
+		return fmt.Errorf("write message value: %s", err)
+	}
+
+	return nil
+}

Godeps/_workspace/src/github.com/influxdb/influxdb/cluster/service_test.go

@@ -0,0 +1,105 @@
+package cluster_test
+
+import (
+	"fmt"
+	"net"
+	"time"
+
+	"github.com/influxdb/influxdb/cluster"
+	"github.com/influxdb/influxdb/influxql"
+	"github.com/influxdb/influxdb/meta"
+	"github.com/influxdb/influxdb/models"
+	"github.com/influxdb/influxdb/tcp"
+	"github.com/influxdb/influxdb/tsdb"
+)
+
+type metaStore struct {
+	host string
+}
+
+func (m *metaStore) Node(nodeID uint64) (*meta.NodeInfo, error) {
+	return &meta.NodeInfo{
+		ID:   nodeID,
+		Host: m.host,
+	}, nil
+}
+
+type testService struct {
+	nodeID           uint64
+	ln               net.Listener
+	muxln            net.Listener
+	writeShardFunc   func(shardID uint64, points []models.Point) error
+	createShardFunc  func(database, policy string, shardID uint64) error
+	createMapperFunc func(shardID uint64, stmt influxql.Statement, chunkSize int) (tsdb.Mapper, error)
+}
+
+func newTestWriteService(f func(shardID uint64, points []models.Point) error) testService {
+	ln, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		panic(err)
+	}
+
+	mux := tcp.NewMux()
+	muxln := mux.Listen(cluster.MuxHeader)
+	go mux.Serve(ln)
+
+	return testService{
+		writeShardFunc: f,
+		ln:             ln,
+		muxln:          muxln,
+	}
+}
+
+func (ts *testService) Close() {
+	if ts.ln != nil {
+		ts.ln.Close()
+	}
+}
+
+type serviceResponses []serviceResponse
+type serviceResponse struct {
+	shardID uint64
+	ownerID uint64
+	points  []models.Point
+}
+
+func (t testService) WriteToShard(shardID uint64, points []models.Point) error {
+	return t.writeShardFunc(shardID, points)
+}
+
+func (t testService) CreateShard(database, policy string, shardID uint64) error {
+	return t.createShardFunc(database, policy, shardID)
+}
+
+func (t testService) CreateMapper(shardID uint64, stmt influxql.Statement, chunkSize int) (tsdb.Mapper, error) {
+	return t.createMapperFunc(shardID, stmt, chunkSize)
+}
+
+func writeShardSuccess(shardID uint64, points []models.Point) error {
+	responses <- &serviceResponse{
+		shardID: shardID,
+		points:  points,
+	}
+	return nil
+}
+
+func writeShardFail(shardID uint64, points []models.Point) error {
+	return fmt.Errorf("failed to write")
+}
+
+var responses = make(chan *serviceResponse, 1024)
+
+func (testService) ResponseN(n int) ([]*serviceResponse, error) {
+	var a []*serviceResponse
+	for {
+		select {
+		case r := <-responses:
+			a = append(a, r)
+			if len(a) == n {
+				return a, nil
+			}
+		case <-time.After(time.Second):
+			return a, fmt.Errorf("unexpected response count: expected: %d, actual: %d", n, len(a))
+		}
+	}
+}

Godeps/_workspace/src/github.com/influxdb/influxdb/cluster/shard_mapper.go

@@ -0,0 +1,259 @@
+package cluster
+
+import (
+	"encoding/json"
+	"fmt"
+	"math/rand"
+	"net"
+	"time"
+
+	"github.com/influxdb/influxdb/influxql"
+	"github.com/influxdb/influxdb/meta"
+	"github.com/influxdb/influxdb/tsdb"
+)
+
+// ShardMapper is responsible for providing mappers for requested shards. It is
+// responsible for creating those mappers from the local store, or reaching
+// out to another node on the cluster.
+type ShardMapper struct {
+	ForceRemoteMapping bool // All shards treated as remote. Useful for testing.
+
+	MetaStore interface {
+		NodeID() uint64
+		Node(id uint64) (ni *meta.NodeInfo, err error)
+	}
+
+	TSDBStore interface {
+		CreateMapper(shardID uint64, stmt influxql.Statement, chunkSize int) (tsdb.Mapper, error)
+	}
+
+	timeout time.Duration
+	pool    *clientPool
+}
+
+// NewShardMapper returns a mapper of local and remote shards.
+func NewShardMapper(timeout time.Duration) *ShardMapper {
+	return &ShardMapper{
+		pool:    newClientPool(),
+		timeout: timeout,
+	}
+}
+
+// CreateMapper returns a Mapper for the given shard ID.
+func (s *ShardMapper) CreateMapper(sh meta.ShardInfo, stmt influxql.Statement, chunkSize int) (tsdb.Mapper, error) {
+	// Create a remote mapper if the local node doesn't own the shard.
+	if !sh.OwnedBy(s.MetaStore.NodeID()) || s.ForceRemoteMapping {
+		// Pick a node in a pseudo-random manner.
+		conn, err := s.dial(sh.Owners[rand.Intn(len(sh.Owners))].NodeID)
+		if err != nil {
+			return nil, err
+		}
+		conn.SetDeadline(time.Now().Add(s.timeout))
+
+		return NewRemoteMapper(conn, sh.ID, stmt, chunkSize), nil
+	}
+
+	// If it is local then return the mapper from the store.
+	m, err := s.TSDBStore.CreateMapper(sh.ID, stmt, chunkSize)
+	if err != nil {
+		return nil, err
+	}
+
+	return m, nil
+}
+
+func (s *ShardMapper) dial(nodeID uint64) (net.Conn, error) {
+	ni, err := s.MetaStore.Node(nodeID)
+	if err != nil {
+		return nil, err
+	}
+	conn, err := net.Dial("tcp", ni.Host)
+	if err != nil {
+		return nil, err
+	}
+
+	// Write the cluster multiplexing header byte
+	conn.Write([]byte{MuxHeader})
+
+	return conn, nil
+}
+
+// RemoteMapper implements the tsdb.Mapper interface. It connects to a remote node,
+// sends a query, and interprets the stream of data that comes back.
+type RemoteMapper struct {
+	shardID   uint64
+	stmt      influxql.Statement
+	chunkSize int
+
+	tagsets []string
+	fields  []string
+
+	conn             net.Conn
+	bufferedResponse *MapShardResponse
+
+	unmarshallers []tsdb.UnmarshalFunc // Mapping-specific unmarshal functions.
+}
+
+// NewRemoteMapper returns a new remote mapper using the given connection.
+func NewRemoteMapper(c net.Conn, shardID uint64, stmt influxql.Statement, chunkSize int) *RemoteMapper {
+	return &RemoteMapper{
+		conn:      c,
+		shardID:   shardID,
+		stmt:      stmt,
+		chunkSize: chunkSize,
+	}
+}
+
+// Open connects to the remote node and starts receiving data.
+func (r *RemoteMapper) Open() (err error) {
+	defer func() {
+		if err != nil {
+			r.conn.Close()
+		}
+	}()
+
+	// Build Map request.
+	var request MapShardRequest
+	request.SetShardID(r.shardID)
+	request.SetQuery(r.stmt.String())
+	request.SetChunkSize(int32(r.chunkSize))
+
+	// Marshal into protocol buffers.
+	buf, err := request.MarshalBinary()
+	if err != nil {
+		return err
+	}
+
+	// Write request.
+	if err := WriteTLV(r.conn, mapShardRequestMessage, buf); err != nil {
+		return err
+	}
+
+	// Read the response.
+	_, buf, err = ReadTLV(r.conn)
+	if err != nil {
+		return err
+	}
+
+	// Unmarshal response.
+	r.bufferedResponse = &MapShardResponse{}
+	if err := r.bufferedResponse.UnmarshalBinary(buf); err != nil {
+		return err
+	}
+
+	if r.bufferedResponse.Code() != 0 {
+		return fmt.Errorf("error code %d: %s", r.bufferedResponse.Code(), r.bufferedResponse.Message())
+	}
+
+	// Decode the first response to get the TagSets.
+	r.tagsets = r.bufferedResponse.TagSets()
+	r.fields = r.bufferedResponse.Fields()
+
+	// Set up each mapping function for this statement.
+	if stmt, ok := r.stmt.(*influxql.SelectStatement); ok {
+		for _, c := range stmt.FunctionCalls() {
+			fn, err := tsdb.InitializeUnmarshaller(c)
+			if err != nil {
+				return err
+			}
+			r.unmarshallers = append(r.unmarshallers, fn)
+		}
+	}
+
+	return nil
+}
+
+// TagSets returns the TagSets
+func (r *RemoteMapper) TagSets() []string {
+	return r.tagsets
+}
+
+// Fields returns RemoteMapper's Fields
+func (r *RemoteMapper) Fields() []string {
+	return r.fields
+}
+
+// NextChunk returns the next chunk read from the remote node to the client.
+func (r *RemoteMapper) NextChunk() (chunk interface{}, err error) {
+	var response *MapShardResponse
+	if r.bufferedResponse != nil {
+		response = r.bufferedResponse
+		r.bufferedResponse = nil
+	} else {
+		response = &MapShardResponse{}
+
+		// Read the response.
+		_, buf, err := ReadTLV(r.conn)
+		if err != nil {
+			return nil, err
+		}
+
+		// Unmarshal response.
+		if err := response.UnmarshalBinary(buf); err != nil {
+			return nil, err
+		}
+
+		if response.Code() != 0 {
+			return nil, fmt.Errorf("error code %d: %s", response.Code(), response.Message())
+		}
+	}
+
+	if response.Data() == nil {
+		return nil, nil
+	}
+
+	moj := &tsdb.MapperOutputJSON{}
+	if err := json.Unmarshal(response.Data(), moj); err != nil {
+		return nil, err
+	}
+	mvj := []*tsdb.MapperValueJSON{}
+	if err := json.Unmarshal(moj.Values, &mvj); err != nil {
+		return nil, err
+	}
+
+	// Prep the non-JSON version of Mapper output.
+	mo := &tsdb.MapperOutput{
+		Name:   moj.Name,
+		Tags:   moj.Tags,
+		Fields: moj.Fields,
+	}
+
+	if len(mvj) == 1 && len(mvj[0].AggData) > 0 {
+		// The MapperValue is carrying aggregate data, so run it through the
+		// custom unmarshallers for the map functions through which the data
+		// was mapped.
+		aggValues := []interface{}{}
+		for i, b := range mvj[0].AggData {
+			v, err := r.unmarshallers[i](b)
+			if err != nil {
+				return nil, err
+			}
+			aggValues = append(aggValues, v)
+		}
+		mo.Values = []*tsdb.MapperValue{&tsdb.MapperValue{
+			Value: aggValues,
+			Tags:  mvj[0].Tags,
+		}}
+	} else {
+		// Must be raw data instead.
+		for _, v := range mvj {
+			var rawValue interface{}
+			if err := json.Unmarshal(v.RawData, &rawValue); err != nil {
+				return nil, err
+			}
+
+			mo.Values = append(mo.Values, &tsdb.MapperValue{
+				Time:  v.Time,
+				Value: rawValue,
+				Tags:  v.Tags,
+			})
+		}
+	}
+
+	return mo, nil
+}
+
+// Close the Mapper
+func (r *RemoteMapper) Close() {
+	r.conn.Close()
+}

Godeps/_workspace/src/github.com/influxdb/influxdb/cluster/shard_mapper_test.go

@@ -0,0 +1,110 @@
+package cluster
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net"
+	"testing"
+
+	"github.com/influxdb/influxdb/influxql"
+	"github.com/influxdb/influxdb/tsdb"
+)
+
+// remoteShardResponder implements the remoteShardConn interface.
+type remoteShardResponder struct {
+	net.Conn
+	t       *testing.T
+	rxBytes []byte
+
+	buffer *bytes.Buffer
+}
+
+func newRemoteShardResponder(outputs []*tsdb.MapperOutput, tagsets []string) *remoteShardResponder {
+	r := &remoteShardResponder{}
+	a := make([]byte, 0, 1024)
+	r.buffer = bytes.NewBuffer(a)
+
+	// Pump the outputs in the buffer for later reading.
+	for _, o := range outputs {
+		resp := &MapShardResponse{}
+		resp.SetCode(0)
+		if o != nil {
+			d, _ := json.Marshal(o)
+			resp.SetData(d)
+			resp.SetTagSets(tagsets)
+		}
+
+		g, _ := resp.MarshalBinary()
+		WriteTLV(r.buffer, mapShardResponseMessage, g)
+	}
+
+	return r
+}
+
+func (r remoteShardResponder) Close() error { return nil }
+func (r remoteShardResponder) Read(p []byte) (n int, err error) {
+	return io.ReadFull(r.buffer, p)
+}
+
+func (r remoteShardResponder) Write(p []byte) (n int, err error) {
+	if r.rxBytes == nil {
+		r.rxBytes = make([]byte, 0)
+	}
+	r.rxBytes = append(r.rxBytes, p...)
+	return len(p), nil
+}
+
+// Ensure a RemoteMapper can process valid responses from a remote shard.
+func TestShardWriter_RemoteMapper_Success(t *testing.T) {
+	expTagSets := []string{"tagsetA"}
+	expOutput := &tsdb.MapperOutput{
+		Name: "cpu",
+		Tags: map[string]string{"host": "serverA"},
+	}
+
+	c := newRemoteShardResponder([]*tsdb.MapperOutput{expOutput, nil}, expTagSets)
+
+	r := NewRemoteMapper(c, 1234, mustParseStmt("SELECT * FROM CPU"), 10)
+	if err := r.Open(); err != nil {
+		t.Fatalf("failed to open remote mapper: %s", err.Error())
+	}
+
+	if r.TagSets()[0] != expTagSets[0] {
+		t.Fatalf("incorrect tagsets received, exp %v, got %v", expTagSets, r.TagSets())
+	}
+
+	// Get first chunk from mapper.
+	chunk, err := r.NextChunk()
+	if err != nil {
+		t.Fatalf("failed to get next chunk from mapper: %s", err.Error())
+	}
+	output, ok := chunk.(*tsdb.MapperOutput)
+	if !ok {
+		t.Fatal("chunk is not of expected type")
+	}
+	if output.Name != "cpu" {
+		t.Fatalf("received output incorrect, exp: %v, got %v", expOutput, output)
+	}
+
+	// Next chunk should be nil, indicating no more data.
+	chunk, err = r.NextChunk()
+	if err != nil {
+		t.Fatalf("failed to get next chunk from mapper: %s", err.Error())
+	}
+	if chunk != nil {
+		t.Fatal("received more chunks when none expected")
+	}
+}
+
+// mustParseStmt parses a single statement or panics.
+func mustParseStmt(stmt string) influxql.Statement {
+	q, err := influxql.ParseQuery(stmt)
+	if err != nil {
+		panic(err)
+	} else if len(q.Statements) != 1 {
+		panic(fmt.Sprintf("expected 1 statement but got %d", len(q.Statements)))
+	}
+	return q.Statements[0]
+}

Godeps/_workspace/src/github.com/influxdb/influxdb/cluster/shard_writer.go

@@ -0,0 +1,165 @@
+package cluster
+
+import (
+	"fmt"
+	"net"
+	"time"
+
+	"github.com/influxdb/influxdb/meta"
+	"github.com/influxdb/influxdb/models"
+	"gopkg.in/fatih/pool.v2"
+)
+
+const (
+	writeShardRequestMessage byte = iota + 1
+	writeShardResponseMessage
+	mapShardRequestMessage
+	mapShardResponseMessage
+)
+
+// ShardWriter writes a set of points to a shard.
+type ShardWriter struct {
+	pool    *clientPool
+	timeout time.Duration
+
+	MetaStore interface {
+		Node(id uint64) (ni *meta.NodeInfo, err error)
+	}
+}
+
+// NewShardWriter returns a new instance of ShardWriter.
+func NewShardWriter(timeout time.Duration) *ShardWriter {
+	return &ShardWriter{
+		pool:    newClientPool(),
+		timeout: timeout,
+	}
+}
+
+// WriteShard writes time series points to a shard
+func (w *ShardWriter) WriteShard(shardID, ownerID uint64, points []models.Point) error {
+	c, err := w.dial(ownerID)
+	if err != nil {
+		return err
+	}
+
+	conn, ok := c.(*pool.PoolConn)
+	if !ok {
+		panic("wrong connection type")
+	}
+	defer func(conn net.Conn) {
+		conn.Close() // return to pool
+	}(conn)
+
+	// Build write request.
+	var request WriteShardRequest
+	request.SetShardID(shardID)
+	request.AddPoints(points)
+
+	// Marshal into protocol buffers.
+	buf, err := request.MarshalBinary()
+	if err != nil {
+		return err
+	}
+
+	// Write request.
+	conn.SetWriteDeadline(time.Now().Add(w.timeout))
+	if err := WriteTLV(conn, writeShardRequestMessage, buf); err != nil {
+		conn.MarkUnusable()
+		return err
+	}
+
+	// Read the response.
+	conn.SetReadDeadline(time.Now().Add(w.timeout))
+	_, buf, err = ReadTLV(conn)
+	if err != nil {
+		conn.MarkUnusable()
+		return err
+	}
+
+	// Unmarshal response.
+	var response WriteShardResponse
+	if err := response.UnmarshalBinary(buf); err != nil {
+		return err
+	}
+
+	if response.Code() != 0 {
+		return fmt.Errorf("error code %d: %s", response.Code(), response.Message())
+	}
+
+	return nil
+}
+
+func (w *ShardWriter) dial(nodeID uint64) (net.Conn, error) {
+	// If we don't have a connection pool for that addr yet, create one
+	_, ok := w.pool.getPool(nodeID)
+	if !ok {
+		factory := &connFactory{nodeID: nodeID, clientPool: w.pool, timeout: w.timeout}
+		factory.metaStore = w.MetaStore
+
+		p, err := pool.NewChannelPool(1, 3, factory.dial)
+		if err != nil {
+			return nil, err
+		}
+		w.pool.setPool(nodeID, p)
+	}
+	return w.pool.conn(nodeID)
+}
+
+// Close closes ShardWriter's pool
+func (w *ShardWriter) Close() error {
+	if w.pool == nil {
+		return fmt.Errorf("client already closed")
+	}
+	w.pool.close()
+	w.pool = nil
+	return nil
+}
+
+const (
+	maxConnections = 500
+	maxRetries     = 3
+)
+
+var errMaxConnectionsExceeded = fmt.Errorf("can not exceed max connections of %d", maxConnections)
+
+type connFactory struct {
+	nodeID  uint64
+	timeout time.Duration
+
+	clientPool interface {
+		size() int
+	}
+
+	metaStore interface {
+		Node(id uint64) (ni *meta.NodeInfo, err error)
+	}
+}
+
+func (c *connFactory) dial() (net.Conn, error) {
+	if c.clientPool.size() > maxConnections {
+		return nil, errMaxConnectionsExceeded
+	}
+
+	ni, err := c.metaStore.Node(c.nodeID)
+	if err != nil {
+		return nil, err
+	}
+
+	if ni == nil {
+		return nil, fmt.Errorf("node %d does not exist", c.nodeID)
+	}
+
+	conn, err := net.DialTimeout("tcp", ni.Host, c.timeout)
+	if err != nil {
+		return nil, err
+	}
+
+	// Write a marker byte for cluster messages.
+	_, err = conn.Write([]byte{MuxHeader})
+	if err != nil {
+		conn.Close()
+		return nil, err
+	}
+
+	return conn, nil
+}

Godeps/_workspace/src/github.com/influxdb/influxdb/cluster/shard_writer_test.go

@@ -0,0 +1,186 @@
+package cluster_test
+
+import (
+	"net"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/influxdb/influxdb/cluster"
+	"github.com/influxdb/influxdb/models"
+)
+
+// Ensure the shard writer can successful write a single request.
+func TestShardWriter_WriteShard_Success(t *testing.T) {
+	ts := newTestWriteService(writeShardSuccess)
+	s := cluster.NewService(cluster.Config{})
+	s.Listener = ts.muxln
+	s.TSDBStore = ts
+	if err := s.Open(); err != nil {
+		t.Fatal(err)
+	}
+	defer s.Close()
+	defer ts.Close()
+
+	w := cluster.NewShardWriter(time.Minute)
+	w.MetaStore = &metaStore{host: ts.ln.Addr().String()}
+
+	// Build a single point.
+	now := time.Now()
+	var points []models.Point
+	points = append(points, models.NewPoint("cpu", models.Tags{"host": "server01"}, map[string]interface{}{"value": int64(100)}, now))
+
+	// Write to shard and close.
+	if err := w.WriteShard(1, 2, points); err != nil {
+		t.Fatal(err)
+	} else if err := w.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	// Validate response.
+	responses, err := ts.ResponseN(1)
+	if err != nil {
+		t.Fatal(err)
+	} else if responses[0].shardID != 1 {
+		t.Fatalf("unexpected shard id: %d", responses[0].shardID)
+	}
+
+	// Validate point.
+	if p := responses[0].points[0]; p.Name() != "cpu" {
+		t.Fatalf("unexpected name: %s", p.Name())
+	} else if p.Fields()["value"] != int64(100) {
+		t.Fatalf("unexpected 'value' field: %d", p.Fields()["value"])
+	} else if p.Tags()["host"] != "server01" {
+		t.Fatalf("unexpected 'host' tag: %s", p.Tags()["host"])
+	} else if p.Time().UnixNano() != now.UnixNano() {
+		t.Fatalf("unexpected time: %s", p.Time())
+	}
+}
+
+// Ensure the shard writer can successful write a multiple requests.
+func TestShardWriter_WriteShard_Multiple(t *testing.T) {
+	ts := newTestWriteService(writeShardSuccess)
+	s := cluster.NewService(cluster.Config{})
+	s.Listener = ts.muxln
+	s.TSDBStore = ts
+	if err := s.Open(); err != nil {
+		t.Fatal(err)
+	}
+	defer s.Close()
+	defer ts.Close()
+
+	w := cluster.NewShardWriter(time.Minute)
+	w.MetaStore = &metaStore{host: ts.ln.Addr().String()}
+
+	// Build a single point.
+	now := time.Now()
+	var points []models.Point
+	points = append(points, models.NewPoint("cpu", models.Tags{"host": "server01"}, map[string]interface{}{"value": int64(100)}, now))
+
+	// Write to shard twice and close.
+	if err := w.WriteShard(1, 2, points); err != nil {
+		t.Fatal(err)
+	} else if err := w.WriteShard(1, 2, points); err != nil {
+		t.Fatal(err)
+	} else if err := w.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	// Validate response.
+	responses, err := ts.ResponseN(1)
+	if err != nil {
+		t.Fatal(err)
+	} else if responses[0].shardID != 1 {
+		t.Fatalf("unexpected shard id: %d", responses[0].shardID)
+	}
+
+	// Validate point.
+	if p := responses[0].points[0]; p.Name() != "cpu" {
+		t.Fatalf("unexpected name: %s", p.Name())
+	} else if p.Fields()["value"] != int64(100) {
+		t.Fatalf("unexpected 'value' field: %d", p.Fields()["value"])
+	} else if p.Tags()["host"] != "server01" {
+		t.Fatalf("unexpected 'host' tag: %s", p.Tags()["host"])
+	} else if p.Time().UnixNano() != now.UnixNano() {
+		t.Fatalf("unexpected time: %s", p.Time())
+	}
+}
+
+// Ensure the shard writer returns an error when the server fails to accept the write.
+func TestShardWriter_WriteShard_Error(t *testing.T) {
+	ts := newTestWriteService(writeShardFail)
+	s := cluster.NewService(cluster.Config{})
+	s.Listener = ts.muxln
+	s.TSDBStore = ts
+	if err := s.Open(); err != nil {
+		t.Fatal(err)
+	}
+	defer s.Close()
+	defer ts.Close()
+
+	w := cluster.NewShardWriter(time.Minute)
+	w.MetaStore = &metaStore{host: ts.ln.Addr().String()}
+	now := time.Now()
+
+	shardID := uint64(1)
+	ownerID := uint64(2)
+	var points []models.Point
+	points = append(points, models.NewPoint(
+		"cpu", models.Tags{"host": "server01"}, map[string]interface{}{"value": int64(100)}, now,
+	))
+
+	if err := w.WriteShard(shardID, ownerID, points); err == nil || err.Error() != "error code 1: write shard 1: failed to write" {
+		t.Fatalf("unexpected error: %v", err)
+	}
+}
+
+// Ensure the shard writer returns an error when dialing times out.
+func TestShardWriter_Write_ErrDialTimeout(t *testing.T) {
+	ts := newTestWriteService(writeShardSuccess)
+	s := cluster.NewService(cluster.Config{})
+	s.Listener = ts.muxln
+	s.TSDBStore = ts
+	if err := s.Open(); err != nil {
+		t.Fatal(err)
+	}
+	defer s.Close()
+	defer ts.Close()
+
+	w := cluster.NewShardWriter(time.Nanosecond)
+	w.MetaStore = &metaStore{host: ts.ln.Addr().String()}
+	now := time.Now()
+
+	shardID := uint64(1)
+	ownerID := uint64(2)
+	var points []models.Point
+	points = append(points, models.NewPoint(
+		"cpu", models.Tags{"host": "server01"}, map[string]interface{}{"value": int64(100)}, now,
+	))
+
+	if err, exp := w.WriteShard(shardID, ownerID, points), "i/o timeout"; err == nil || !strings.Contains(err.Error(), exp) {
+		t.Fatalf("expected error %v, to contain %s", err, exp)
+	}
+}
+
+// Ensure the shard writer returns an error when reading times out.
+func TestShardWriter_Write_ErrReadTimeout(t *testing.T) {
+	ln, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	w := cluster.NewShardWriter(time.Millisecond)
+	w.MetaStore = &metaStore{host: ln.Addr().String()}
+	now := time.Now()
+
+	shardID := uint64(1)
+	ownerID := uint64(2)
+	var points []models.Point
+	points = append(points, models.NewPoint(
+		"cpu", models.Tags{"host": "server01"}, map[string]interface{}{"value": int64(100)}, now,
+	))
+
+	if err := w.WriteShard(shardID, ownerID, points); err == nil || !strings.Contains(err.Error(), "i/o timeout") {
+		t.Fatalf("unexpected error: %s", err)
+	}
+}

Godeps/_workspace/src/github.com/influxdb/influxdb/cmd/influx/main.go

@@ -0,0 +1,779 @@
+package main
+
+import (
+	"encoding/csv"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"io"
+	"net"
+	"net/url"
+	"os"
+	"os/user"
+	"path/filepath"
+	"sort"
+	"strconv"
+	"strings"
+	"text/tabwriter"
+
+	"github.com/influxdb/influxdb/client"
+	"github.com/influxdb/influxdb/cluster"
+	"github.com/influxdb/influxdb/importer/v8"
+	"github.com/peterh/liner"
+)
+
+// These variables are populated via the Go linker.
+var (
+	version string = "0.9"
+)
+
+const (
+	// defaultFormat is the default format of the results when issuing queries
+	defaultFormat = "column"
+
+	// defaultPrecision is the default timestamp format of the results when issuing queries
+	defaultPrecision = "ns"
+
+	// defaultPPS is the default points per second that the import will throttle at
+	// by default it's 0, which means it will not throttle
+	defaultPPS = 0
+)
+
+type CommandLine struct {
+	Client           *client.Client
+	Line             *liner.State
+	Host             string
+	Port             int
+	Username         string
+	Password         string
+	Database         string
+	Ssl              bool
+	RetentionPolicy  string
+	Version          string
+	Pretty           bool   // controls pretty print for json
+	Format           string // controls the output format.  Valid values are json, csv, or column
+	Precision        string
+	WriteConsistency string
+	Execute          string
+	ShowVersion      bool
+	Import           bool
+	PPS              int // Controls how many points per second the import will allow via throttling
+	Path             string
+	Compressed       bool
+}
+
+func main() {
+	c := CommandLine{}
+
+	fs := flag.NewFlagSet("InfluxDB shell version "+version, flag.ExitOnError)
+	fs.StringVar(&c.Host, "host", client.DefaultHost, "Influxdb host to connect to.")
+	fs.IntVar(&c.Port, "port", client.DefaultPort, "Influxdb port to connect to.")
+	fs.StringVar(&c.Username, "username", c.Username, "Username to connect to the server.")
+	fs.StringVar(&c.Password, "password", c.Password, `Password to connect to the server.  Leaving blank will prompt for password (--password="").`)
+	fs.StringVar(&c.Database, "database", c.Database, "Database to connect to the server.")
+	fs.BoolVar(&c.Ssl, "ssl", false, "Use https for connecting to cluster.")
+	fs.StringVar(&c.Format, "format", defaultFormat, "Format specifies the format of the server responses:  json, csv, or column.")
+	fs.StringVar(&c.Precision, "precision", defaultPrecision, "Precision specifies the format of the timestamp:  rfc3339,h,m,s,ms,u or ns.")
+	fs.StringVar(&c.WriteConsistency, "consistency", "any", "Set write consistency level: any, one, quorum, or all.")
+	fs.BoolVar(&c.Pretty, "pretty", false, "Turns on pretty print for the json format.")
+	fs.StringVar(&c.Execute, "execute", c.Execute, "Execute command and quit.")
+	fs.BoolVar(&c.ShowVersion, "version", false, "Displays the InfluxDB version.")
+	fs.BoolVar(&c.Import, "import", false, "Import a previous database.")
+	fs.IntVar(&c.PPS, "pps", defaultPPS, "How many points per second the import will allow.  By default it is zero and will not throttle importing.")
+	fs.StringVar(&c.Path, "path", "", "path to the file to import")
+	fs.BoolVar(&c.Compressed, "compressed", false, "set to true if the import file is compressed")
+
+	// Define our own custom usage to print
+	fs.Usage = func() {
+		fmt.Println(`Usage of influx:
+  -version
+       Display the version and exit.
+  -host 'host name'
+       Host to connect to.
+  -port 'port #'
+       Port to connect to.
+  -database 'database name'
+       Database to connect to the server.
+  -password 'password'
+      Password to connect to the server.  Leaving blank will prompt for password (--password '').
+  -username 'username'
+       Username to connect to the server.
+  -ssl
+        Use https for requests.
+  -execute 'command'
+       Execute command and quit.
+  -format 'json|csv|column'
+       Format specifies the format of the server responses:  json, csv, or column.
+  -precision 'rfc3339|h|m|s|ms|u|ns'
+       Precision specifies the format of the timestamp:  rfc3339, h, m, s, ms, u or ns.
+  -consistency 'any|one|quorum|all'
+       Set write consistency level: any, one, quorum, or all
+  -pretty
+       Turns on pretty print for the json format.
+  -import
+       Import a previous database export from file
+  -pps
+       How many points per second the import will allow.  By default it is zero and will not throttle importing.
+  -path
+       Path to file to import
+  -compressed
+       Set to true if the import file is compressed
+
+Examples:
+
+    # Use influx in a non-interactive mode to query the database "metrics" and pretty print json:
+    $ influx -database 'metrics' -execute 'select * from cpu' -format 'json' -pretty
+
+    # Connect to a specific database on startup and set database context:
+    $ influx -database 'metrics' -host 'localhost' -port '8086'
+`)
+	}
+	fs.Parse(os.Args[1:])
+
+	if c.ShowVersion {
+		showVersion()
+		os.Exit(0)
+	}
+
+	var promptForPassword bool
+	// determine if they set the password flag but provided no value
+	for _, v := range os.Args {
+		v = strings.ToLower(v)
+		if (strings.HasPrefix(v, "-password") || strings.HasPrefix(v, "--password")) && c.Password == "" {
+			promptForPassword = true
+			break
+		}
+	}
+
+	c.Line = liner.NewLiner()
+	defer c.Line.Close()
+
+	if promptForPassword {
+		p, e := c.Line.PasswordPrompt("password: ")
+		if e != nil {
+			fmt.Println("Unable to parse password.")
+		} else {
+			c.Password = p
+		}
+	}
+
+	if err := c.connect(""); err != nil {
+
+	}
+	if c.Execute == "" && !c.Import {
+		fmt.Printf("Connected to %s version %s\n", c.Client.Addr(), c.Version)
+	}
+
+	if c.Execute != "" {
+		// Modify precision before executing query
+		c.SetPrecision(c.Precision)
+		if err := c.ExecuteQuery(c.Execute); err != nil {
+			c.Line.Close()
+			os.Exit(1)
+		}
+		c.Line.Close()
+		os.Exit(0)
+	}
+
+	if c.Import {
+		path := net.JoinHostPort(c.Host, strconv.Itoa(c.Port))
+		u, e := client.ParseConnectionString(path, c.Ssl)
+		if e != nil {
+			fmt.Println(e)
+			return
+		}
+
+		config := v8.NewConfig()
+		config.Username = c.Username
+		config.Password = c.Password
+		config.Precision = "ns"
+		config.WriteConsistency = "any"
+		config.Path = c.Path
+		config.Version = version
+		config.URL = u
+		config.Compressed = c.Compressed
+		config.PPS = c.PPS
+		config.Precision = c.Precision
+
+		i := v8.NewImporter(config)
+		if err := i.Import(); err != nil {
+			fmt.Printf("ERROR: %s\n", err)
+			c.Line.Close()
+			os.Exit(1)
+		}
+		c.Line.Close()
+		os.Exit(0)
+	}
+
+	showVersion()
+
+	var historyFile string
+	usr, err := user.Current()
+	// Only load history if we can get the user
+	if err == nil {
+		historyFile = filepath.Join(usr.HomeDir, ".influx_history")
+
+		if f, err := os.Open(historyFile); err == nil {
+			c.Line.ReadHistory(f)
+			f.Close()
+		}
+	}
+
+	for {
+		l, e := c.Line.Prompt("> ")
+		if e != nil {
+			break
+		}
+		if c.ParseCommand(l) {
+			// write out the history
+			if len(historyFile) > 0 {
+				c.Line.AppendHistory(l)
+				if f, err := os.Create(historyFile); err == nil {
+					c.Line.WriteHistory(f)
+					f.Close()
+				}
+			}
+		} else {
+			break // exit main loop
+		}
+	}
+}
+
+func showVersion() {
+	fmt.Println("InfluxDB shell " + version)
+}
+
+func (c *CommandLine) ParseCommand(cmd string) bool {
+	lcmd := strings.TrimSpace(strings.ToLower(cmd))
+	switch {
+	case strings.HasPrefix(lcmd, "exit"):
+		// signal the program to exit
+		return false
+	case strings.HasPrefix(lcmd, "gopher"):
+		c.gopher()
+	case strings.HasPrefix(lcmd, "connect"):
+		c.connect(cmd)
+	case strings.HasPrefix(lcmd, "auth"):
+		c.SetAuth(cmd)
+	case strings.HasPrefix(lcmd, "help"):
+		c.help()
+	case strings.HasPrefix(lcmd, "format"):
+		c.SetFormat(cmd)
+	case strings.HasPrefix(lcmd, "precision"):
+		c.SetPrecision(cmd)
+	case strings.HasPrefix(lcmd, "consistency"):
+		c.SetWriteConsistency(cmd)
+	case strings.HasPrefix(lcmd, "settings"):
+		c.Settings()
+	case strings.HasPrefix(lcmd, "pretty"):
+		c.Pretty = !c.Pretty
+		if c.Pretty {
+			fmt.Println("Pretty print enabled")
+		} else {
+			fmt.Println("Pretty print disabled")
+		}
+	case strings.HasPrefix(lcmd, "use"):
+		c.use(cmd)
+	case strings.HasPrefix(lcmd, "insert"):
+		c.Insert(cmd)
+	case lcmd == "":
+		break
+	default:
+		c.ExecuteQuery(cmd)
+	}
+	return true
+}
+
+func (c *CommandLine) connect(cmd string) error {
+	var cl *client.Client
+	var u url.URL
+
+	// Remove the "connect" keyword if it exists
+	path := strings.TrimSpace(strings.Replace(cmd, "connect", "", -1))
+
+	// If they didn't provide a connection string, use the current settings
+	if path == "" {
+		path = net.JoinHostPort(c.Host, strconv.Itoa(c.Port))
+	}
+
+	var e error
+	u, e = client.ParseConnectionString(path, c.Ssl)
+	if e != nil {
+		return e
+	}
+
+	config := client.NewConfig()
+	config.URL = u
+	config.Username = c.Username
+	config.Password = c.Password
+	config.UserAgent = "InfluxDBShell/" + version
+	config.Precision = c.Precision
+	cl, err := client.NewClient(config)
+	if err != nil {
+		return fmt.Errorf("Could not create client %s", err)
+	}
+	c.Client = cl
+	if _, v, e := c.Client.Ping(); e != nil {
+		return fmt.Errorf("Failed to connect to %s\n", c.Client.Addr())
+	} else {
+		c.Version = v
+	}
+	return nil
+}
+
+func (c *CommandLine) SetAuth(cmd string) {
+	// If they pass in the entire command, we should parse it
+	// auth <username> <password>
+	args := strings.Fields(cmd)
+	if len(args) == 3 {
+		args = args[1:]
+	} else {
+		args = []string{}
+	}
+
+	if len(args) == 2 {
+		c.Username = args[0]
+		c.Password = args[1]
+	} else {
+		u, e := c.Line.Prompt("username: ")
+		if e != nil {
+			fmt.Printf("Unable to process input: %s", e)
+			return
+		}
+		c.Username = strings.TrimSpace(u)
+		p, e := c.Line.PasswordPrompt("password: ")
+		if e != nil {
+			fmt.Printf("Unable to process input: %s", e)
+			return
+		}
+		c.Password = p
+	}
+
+	// Update the client as well
+	c.Client.SetAuth(c.Username, c.Password)
+}
+
+func (c *CommandLine) use(cmd string) {
+	args := strings.Split(strings.TrimSuffix(strings.TrimSpace(cmd), ";"), " ")
+	if len(args) != 2 {
+		fmt.Printf("Could not parse database name from %q.\n", cmd)
+		return
+	}
+	d := args[1]
+	c.Database = d
+	fmt.Printf("Using database %s\n", d)
+}
+
+func (c *CommandLine) SetPrecision(cmd string) {
+	// Remove the "precision" keyword if it exists
+	cmd = strings.TrimSpace(strings.Replace(cmd, "precision", "", -1))
+	// normalize cmd
+	cmd = strings.ToLower(cmd)
+
+	switch cmd {
+	case "h", "m", "s", "ms", "u", "ns":
+		c.Precision = cmd
+		c.Client.SetPrecision(c.Precision)
+	case "rfc3339":
+		c.Precision = ""
+		c.Client.SetPrecision(c.Precision)
+	default:
+		fmt.Printf("Unknown precision %q. Please use rfc3339, h, m, s, ms, u or ns.\n", cmd)
+	}
+}
+
+func (c *CommandLine) SetFormat(cmd string) {
+	// Remove the "format" keyword if it exists
+	cmd = strings.TrimSpace(strings.Replace(cmd, "format", "", -1))
+	// normalize cmd
+	cmd = strings.ToLower(cmd)
+
+	switch cmd {
+	case "json", "csv", "column":
+		c.Format = cmd
+	default:
+		fmt.Printf("Unknown format %q. Please use json, csv, or column.\n", cmd)
+	}
+}
+
+func (c *CommandLine) SetWriteConsistency(cmd string) {
+	// Remove the "consistency" keyword if it exists
+	cmd = strings.TrimSpace(strings.Replace(cmd, "consistency", "", -1))
+	// normalize cmd
+	cmd = strings.ToLower(cmd)
+
+	_, err := cluster.ParseConsistencyLevel(cmd)
+	if err != nil {
+		fmt.Printf("Unknown consistency level %q. Please use any, one, quorum, or all.\n", cmd)
+		return
+	}
+	c.WriteConsistency = cmd
+}
+
+// isWhitespace returns true if the rune is a space, tab, or newline.
+func isWhitespace(ch rune) bool { return ch == ' ' || ch == '\t' || ch == '\n' }
+
+// isLetter returns true if the rune is a letter.
+func isLetter(ch rune) bool { return (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') }
+
+// isDigit returns true if the rune is a digit.
+func isDigit(ch rune) bool { return (ch >= '0' && ch <= '9') }
+
+// isIdentFirstChar returns true if the rune can be used as the first char in an unquoted identifer.
+func isIdentFirstChar(ch rune) bool { return isLetter(ch) || ch == '_' }
+
+// isIdentChar returns true if the rune can be used in an unquoted identifier.
+func isNotIdentChar(ch rune) bool { return !(isLetter(ch) || isDigit(ch) || ch == '_') }
+
+func parseUnquotedIdentifier(stmt string) (string, string) {
+	if fields := strings.FieldsFunc(stmt, isNotIdentChar); len(fields) > 0 {
+		return fields[0], strings.TrimPrefix(stmt, fields[0])
+	}
+	return "", stmt
+}
+
+func parseDoubleQuotedIdentifier(stmt string) (string, string) {
+	escapeNext := false
+	fields := strings.FieldsFunc(stmt, func(ch rune) bool {
+		if ch == '\\' {
+			escapeNext = true
+		} else if ch == '"' {
+			if !escapeNext {
+				return true
+			}
+			escapeNext = false
+		}
+		return false
+	})
+	if len(fields) > 0 {
+		return fields[0], strings.TrimPrefix(stmt, "\""+fields[0]+"\"")
+	}
+	return "", stmt
+}
+
+func parseNextIdentifier(stmt string) (ident, remainder string) {
+	if len(stmt) > 0 {
+		switch {
+		case isWhitespace(rune(stmt[0])):
+			return parseNextIdentifier(stmt[1:])
+		case isIdentFirstChar(rune(stmt[0])):
+			return parseUnquotedIdentifier(stmt)
+		case stmt[0] == '"':
+			return parseDoubleQuotedIdentifier(stmt)
+		}
+	}
+	return "", stmt
+}
+
+func (c *CommandLine) parseInto(stmt string) string {
+	ident, stmt := parseNextIdentifier(stmt)
+	if strings.HasPrefix(stmt, ".") {
+		c.Database = ident
+		fmt.Printf("Using database %s\n", c.Database)
+		ident, stmt = parseNextIdentifier(stmt[1:])
+	}
+	if strings.HasPrefix(stmt, " ") {
+		c.RetentionPolicy = ident
+		fmt.Printf("Using retention policy %s\n", c.RetentionPolicy)
+		return stmt[1:]
+	}
+	return stmt
+}
+
+func (c *CommandLine) Insert(stmt string) error {
+	i, point := parseNextIdentifier(stmt)
+	if !strings.EqualFold(i, "insert") {
+		fmt.Printf("ERR: found %s, expected INSERT\n", i)
+		return nil
+	}
+	if i, r := parseNextIdentifier(point); strings.EqualFold(i, "into") {
+		point = c.parseInto(r)
+	}
+	_, err := c.Client.Write(client.BatchPoints{
+		Points: []client.Point{
+			client.Point{Raw: point},
+		},
+		Database:         c.Database,
+		RetentionPolicy:  c.RetentionPolicy,
+		Precision:        "n",
+		WriteConsistency: c.WriteConsistency,
+	})
+	if err != nil {
+		fmt.Printf("ERR: %s\n", err)
+		if c.Database == "" {
+			fmt.Println("Note: error may be due to not setting a database or retention policy.")
+			fmt.Println(`Please set a database with the command "use <database>" or`)
+			fmt.Println("INSERT INTO <database>.<retention-policy> <point>")
+		}
+		return err
+	}
+	return nil
+}
+
+func (c *CommandLine) ExecuteQuery(query string) error {
+	response, err := c.Client.Query(client.Query{Command: query, Database: c.Database})
+	if err != nil {
+		fmt.Printf("ERR: %s\n", err)
+		return err
+	}
+	c.FormatResponse(response, os.Stdout)
+	if err := response.Error(); err != nil {
+		fmt.Printf("ERR: %s\n", response.Error())
+		if c.Database == "" {
+			fmt.Println("Warning: It is possible this error is due to not setting a database.")
+			fmt.Println(`Please set a database with the command "use <database>".`)
+		}
+		return err
+	}
+	return nil
+}
+
+func (c *CommandLine) FormatResponse(response *client.Response, w io.Writer) {
+	switch c.Format {
+	case "json":
+		c.writeJSON(response, w)
+	case "csv":
+		c.writeCSV(response, w)
+	case "column":
+		c.writeColumns(response, w)
+	default:
+		fmt.Fprintf(w, "Unknown output format %q.\n", c.Format)
+	}
+}
+
+func (c *CommandLine) writeJSON(response *client.Response, w io.Writer) {
+	var data []byte
+	var err error
+	if c.Pretty {
+		data, err = json.MarshalIndent(response, "", "    ")
+	} else {
+		data, err = json.Marshal(response)
+	}
+	if err != nil {
+		fmt.Fprintf(w, "Unable to parse json: %s\n", err)
+		return
+	}
+	fmt.Fprintln(w, string(data))
+}
+
+func (c *CommandLine) writeCSV(response *client.Response, w io.Writer) {
+	csvw := csv.NewWriter(w)
+	for _, result := range response.Results {
+		// Create a tabbed writer for each result as they won't always line up
+		rows := c.formatResults(result, "\t")
+		for _, r := range rows {
+			csvw.Write(strings.Split(r, "\t"))
+		}
+		csvw.Flush()
+	}
+}
+
+func (c *CommandLine) writeColumns(response *client.Response, w io.Writer) {
+	for _, result := range response.Results {
+		// Create a tabbed writer for each result a they won't always line up
+		w := new(tabwriter.Writer)
+		w.Init(os.Stdout, 0, 8, 1, '\t', 0)
+		csv := c.formatResults(result, "\t")
+		for _, r := range csv {
+			fmt.Fprintln(w, r)
+		}
+		w.Flush()
+	}
+}
+
+// formatResults will behave differently if you are formatting for columns or csv
+func (c *CommandLine) formatResults(result client.Result, separator string) []string {
+	rows := []string{}
+	// Create a tabbed writer for each result a they won't always line up
+	for i, row := range result.Series {
+		// gather tags
+		tags := []string{}
+		for k, v := range row.Tags {
+			tags = append(tags, fmt.Sprintf("%s=%s", k, v))
+			sort.Strings(tags)
+		}
+
+		columnNames := []string{}
+
+		// Only put name/tags in a column if format is csv
+		if c.Format == "csv" {
+			if len(tags) > 0 {
+				columnNames = append([]string{"tags"}, columnNames...)
+			}
+
+			if row.Name != "" {
+				columnNames = append([]string{"name"}, columnNames...)
+			}
+		}
+
+		for _, column := range row.Columns {
+			columnNames = append(columnNames, column)
+		}
+
+		// Output a line separator if we have more than one set or results and format is column
+		if i > 0 && c.Format == "column" {
+			rows = append(rows, "")
+		}
+
+		// If we are column format, we break out the name/tag to seperate lines
+		if c.Format == "column" {
+			if row.Name != "" {
+				n := fmt.Sprintf("name: %s", row.Name)
+				rows = append(rows, n)
+				if len(tags) == 0 {
+					l := strings.Repeat("-", len(n))
+					rows = append(rows, l)
+				}
+			}
+			if len(tags) > 0 {
+				t := fmt.Sprintf("tags: %s", (strings.Join(tags, ", ")))
+				rows = append(rows, t)
+			}
+		}
+
+		rows = append(rows, strings.Join(columnNames, separator))
+
+		// if format is column, break tags to their own line/format
+		if c.Format == "column" && len(tags) > 0 {
+			lines := []string{}
+			for _, columnName := range columnNames {
+				lines = append(lines, strings.Repeat("-", len(columnName)))
+			}
+			rows = append(rows, strings.Join(lines, separator))
+		}
+
+		for _, v := range row.Values {
+			var values []string
+			if c.Format == "csv" {
+				if row.Name != "" {
+					values = append(values, row.Name)
+				}
+				if len(tags) > 0 {
+					values = append(values, strings.Join(tags, ","))
+				}
+			}
+
+			for _, vv := range v {
+				values = append(values, interfaceToString(vv))
+			}
+			rows = append(rows, strings.Join(values, separator))
+		}
+		// Outout a line separator if in column format
+		if c.Format == "column" {
+			rows = append(rows, "")
+		}
+	}
+	return rows
+}
+
+func interfaceToString(v interface{}) string {
+	switch t := v.(type) {
+	case nil:
+		return ""
+	case bool:
+		return fmt.Sprintf("%v", v)
+	case int, int8, int16, int32, int64, uint, uint8, uint16, uint32, uint64, uintptr:
+		return fmt.Sprintf("%d", t)
+	case float32, float64:
+		return fmt.Sprintf("%v", t)
+	default:
+		return fmt.Sprintf("%v", t)
+	}
+}
+
+func (c *CommandLine) Settings() {
+	w := new(tabwriter.Writer)
+	w.Init(os.Stdout, 0, 8, 1, '\t', 0)
+	if c.Port > 0 {
+		fmt.Fprintf(w, "Host\t%s:%d\n", c.Host, c.Port)
+	} else {
+		fmt.Fprintf(w, "Host\t%s\n", c.Host)
+	}
+	fmt.Fprintf(w, "Username\t%s\n", c.Username)
+	fmt.Fprintf(w, "Database\t%s\n", c.Database)
+	fmt.Fprintf(w, "Pretty\t%v\n", c.Pretty)
+	fmt.Fprintf(w, "Format\t%s\n", c.Format)
+	fmt.Fprintf(w, "Write Consistency\t%s\n", c.WriteConsistency)
+	fmt.Fprintln(w)
+	w.Flush()
+}
+
+func (c *CommandLine) help() {
+	fmt.Println(`Usage:
+        connect <host:port>   connect to another node
+        auth                  prompt for username and password
+        pretty                toggle pretty print
+        use <db_name>         set current databases
+        format <format>       set the output format: json, csv, or column
+        precision <format>    set the timestamp format: h,m,s,ms,u,ns
+        consistency <level>   set write consistency level: any, one, quorum, or all
+        settings              output the current settings for the shell
+        exit                  quit the influx shell
+
+        show databases        show database names
+        show series           show series information
+        show measurements     show measurement information
+        show tag keys         show tag key information
+        show tag values       show tag value information
+
+        a full list of influxql commands can be found at:
+        https://influxdb.com/docs/v0.9/query_language/spec.html
+`)
+}
+
+func (c *CommandLine) gopher() {
+	fmt.Println(`
+                                          .-::-::://:-::-    .:/++/'
+                                     '://:-''/oo+//++o+/.://o-    ./+:
+                                  .:-.    '++-         .o/ '+yydhy'  o-
+                               .:/.      .h:         :osoys  .smMN-  :/
+                            -/:.'        s-         /MMMymh.   '/y/  s'
+                         -+s:''''        d          -mMMms//     '-/o:
+                       -/++/++/////:.    o:          '... s-        :s.
+                     :+-+s-'       ':/'  's-             /+          'o:
+                   '+-'o:        /ydhsh.  '//.        '-o-             o-
+                  .y. o:        .MMMdm+y    ':+++:::/+:.'               s:
+                .-h/  y-        'sdmds'h -+ydds:::-.'                   'h.
+             .//-.d'  o:          '.' 'dsNMMMNh:.:++'                    :y
+            +y.  'd   's.            .s:mddds:     ++                     o/
+           'N-  odd    'o/.       './o-s-'   .---+++'                      o-
+           'N'  yNd      .://:/:::::. -s   -+/s/./s'                       'o/'
+            so'  .h         ''''       ////s: '+. .s                         +y'
+             os/-.y'                       's' 'y::+                          +d'
+               '.:o/                        -+:-:.'                            so.---.'
+                   o'                                                          'd-.''/s'
+                   .s'                                                          :y.''.y
+                    -s                                                           mo:::'
+                     ::                                                          yh
+                      //                                      ''''               /M'
+                       o+                                    .s///:/.            'N:
+                        :+                                   /:    -s'            ho
+                         's-                               -/s/:+/.+h'            +h
+                           ys'                            ':'    '-.              -d
+                            oh                                                    .h
+                             /o                                                   .s
+                              s.                                                  .h
+                              -y                                                  .d
+                               m/                                                 -h
+                               +d                                                 /o
+                               'N-                                                y:
+                                h:                                                m.
+                                s-                                               -d
+                                o-                                               s+
+                                +-                                              'm'
+                                s/                                              oo--.
+                                y-                                             /s  ':+'
+                                s'                                           'od--' .d:
+                                -+                                         ':o: ':+-/+
+                                 y-                                      .:+-      '
+                                //o-                                 '.:+/.
+                                .-:+/'                           ''-/+/.
+                                    ./:'                    ''.:o+/-'
+                                      .+o:/:/+-'      ''.-+ooo/-'
+                                         o:   -h///++////-.
+                                        /:   .o/
+                                       //+  'y
+                                       ./sooy.
+
+`)
+}

Godeps/_workspace/src/github.com/influxdb/influxdb/cmd/influx/main_test.go

@@ -0,0 +1,219 @@
+package main_test
+
+import (
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"testing"
+
+	"github.com/influxdb/influxdb/client"
+	main "github.com/influxdb/influxdb/cmd/influx"
+)
+
+func TestParseCommand_CommandsExist(t *testing.T) {
+	t.Parallel()
+	c := main.CommandLine{}
+	tests := []struct {
+		cmd string
+	}{
+		{cmd: "gopher"},
+		{cmd: "connect"},
+		{cmd: "help"},
+		{cmd: "pretty"},
+		{cmd: "use"},
+		{cmd: ""}, // test that a blank command just returns
+	}
+	for _, test := range tests {
+		if !c.ParseCommand(test.cmd) {
+			t.Fatalf(`Command failed for %q.`, test.cmd)
+		}
+	}
+}
+
+func TestParseCommand_TogglePretty(t *testing.T) {
+	t.Parallel()
+	c := main.CommandLine{}
+	if c.Pretty {
+		t.Fatalf(`Pretty should be false.`)
+	}
+	c.ParseCommand("pretty")
+	if !c.Pretty {
+		t.Fatalf(`Pretty should be true.`)
+	}
+	c.ParseCommand("pretty")
+	if c.Pretty {
+		t.Fatalf(`Pretty should be false.`)
+	}
+}
+
+func TestParseCommand_Exit(t *testing.T) {
+	t.Parallel()
+	c := main.CommandLine{}
+	tests := []struct {
+		cmd string
+	}{
+		{cmd: "exit"},
+		{cmd: " exit"},
+		{cmd: "exit "},
+		{cmd: "Exit "},
+	}
+
+	for _, test := range tests {
+		if c.ParseCommand(test.cmd) {
+			t.Fatalf(`Command "exit" failed for %q.`, test.cmd)
+		}
+	}
+}
+
+func TestParseCommand_Use(t *testing.T) {
+	t.Parallel()
+	c := main.CommandLine{}
+	tests := []struct {
+		cmd string
+	}{
+		{cmd: "use db"},
+		{cmd: " use db"},
+		{cmd: "use db "},
+		{cmd: "use db;"},
+		{cmd: "use db; "},
+		{cmd: "Use db"},
+	}
+
+	for _, test := range tests {
+		if !c.ParseCommand(test.cmd) {
+			t.Fatalf(`Command "use" failed for %q.`, test.cmd)
+		}
+
+		if c.Database != "db" {
+			t.Fatalf(`Command "use" changed database to %q. Expected db`, c.Database)
+		}
+	}
+}
+
+func TestParseCommand_Consistency(t *testing.T) {
+	t.Parallel()
+	c := main.CommandLine{}
+	tests := []struct {
+		cmd string
+	}{
+		{cmd: "consistency one"},
+		{cmd: " consistency one"},
+		{cmd: "consistency one "},
+		{cmd: "consistency one;"},
+		{cmd: "consistency one; "},
+		{cmd: "Consistency one"},
+	}
+
+	for _, test := range tests {
+		if !c.ParseCommand(test.cmd) {
+			t.Fatalf(`Command "consistency" failed for %q.`, test.cmd)
+		}
+
+		if c.WriteConsistency != "one" {
+			t.Fatalf(`Command "consistency" changed consistency to %q. Expected one`, c.WriteConsistency)
+		}
+	}
+}
+
+func TestParseCommand_Insert(t *testing.T) {
+	t.Parallel()
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		var data client.Response
+		w.WriteHeader(http.StatusNoContent)
+		_ = json.NewEncoder(w).Encode(data)
+	}))
+	defer ts.Close()
+
+	u, _ := url.Parse(ts.URL)
+	config := client.Config{URL: *u}
+	c, err := client.NewClient(config)
+	if err != nil {
+		t.Fatalf("unexpected error.  expected %v, actual %v", nil, err)
+	}
+	m := main.CommandLine{Client: c}
+
+	tests := []struct {
+		cmd string
+	}{
+		{cmd: "INSERT cpu,host=serverA,region=us-west value=1.0"},
+		{cmd: " INSERT cpu,host=serverA,region=us-west value=1.0"},
+		{cmd: "INSERT   cpu,host=serverA,region=us-west value=1.0"},
+		{cmd: "insert cpu,host=serverA,region=us-west    value=1.0    "},
+		{cmd: "insert"},
+		{cmd: "Insert "},
+		{cmd: "insert c"},
+		{cmd: "insert int"},
+	}
+
+	for _, test := range tests {
+		if !m.ParseCommand(test.cmd) {
+			t.Fatalf(`Command "insert" failed for %q.`, test.cmd)
+		}
+	}
+}
+
+func TestParseCommand_InsertInto(t *testing.T) {
+	t.Parallel()
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		var data client.Response
+		w.WriteHeader(http.StatusNoContent)
+		_ = json.NewEncoder(w).Encode(data)
+	}))
+	defer ts.Close()
+
+	u, _ := url.Parse(ts.URL)
+	config := client.Config{URL: *u}
+	c, err := client.NewClient(config)
+	if err != nil {
+		t.Fatalf("unexpected error.  expected %v, actual %v", nil, err)
+	}
+	m := main.CommandLine{Client: c}
+
+	tests := []struct {
+		cmd, db, rp string
+	}{
+		{
+			cmd: `INSERT INTO test cpu,host=serverA,region=us-west value=1.0`,
+			db:  "",
+			rp:  "test",
+		},
+		{
+			cmd: ` INSERT INTO .test cpu,host=serverA,region=us-west value=1.0`,
+			db:  "",
+			rp:  "test",
+		},
+		{
+			cmd: `INSERT INTO   "test test" cpu,host=serverA,region=us-west value=1.0`,
+			db:  "",
+			rp:  "test test",
+		},
+		{
+			cmd: `Insert iNTO test.test cpu,host=serverA,region=us-west value=1.0`,
+			db:  "test",
+			rp:  "test",
+		},
+		{
+			cmd: `insert into "test test" cpu,host=serverA,region=us-west value=1.0`,
+			db:  "test",
+			rp:  "test test",
+		},
+		{
+			cmd: `insert into "d b"."test test" cpu,host=serverA,region=us-west value=1.0`,
+			db:  "d b",
+			rp:  "test test",
+		},
+	}
+
+	for _, test := range tests {
+		if !m.ParseCommand(test.cmd) {
+			t.Fatalf(`Command "insert into" failed for %q.`, test.cmd)
+		}
+		if m.Database != test.db {
+			t.Fatalf(`Command "insert into" db parsing failed, expected: %q, actual: %q`, test.db, m.Database)
+		}
+		if m.RetentionPolicy != test.rp {
+			t.Fatalf(`Command "insert into" rp parsing failed, expected: %q, actual: %q`, test.rp, m.RetentionPolicy)
+		}
+	}
+}

Godeps/_workspace/src/github.com/influxdb/influxdb/cmd/influx_stress/examples/100B_FLAT.toml

@@ -0,0 +1,30 @@
+channel_buffer_size = 100000
+
+[write]
+  concurrency = 10
+  batch_size = 10000
+  batch_interval = "0s"
+  database = "stress"
+  precision = "n"
+  address = "localhost:8086"
+  reset_database = true
+  start_date = "2006-Jan-02"
+
+[[series]]
+  tick = "10s"
+  jitter = false
+  point_count = 1000000 # number of points that will be written for each of the series
+  measurement = "cpu"
+  series_count = 100000
+
+  [[series.tag]]
+    key = "host"
+    value = "server"
+
+  [[series.tag]]
+    key = "location"
+    value = "loc"
+
+  [[series.field]]
+    key = "value"
+    type = "float64-flat"

Godeps/_workspace/src/github.com/influxdb/influxdb/cmd/influx_stress/examples/100B_STD.toml

@@ -0,0 +1,30 @@
+channel_buffer_size = 100000
+
+[write]
+  concurrency = 10
+  batch_size = 10000
+  batch_interval = "0s"
+  database = "stress"
+  precision = "n"
+  address = "localhost:8086"
+  reset_database = true
+  start_date = "2006-Jan-02"
+
+[[series]]
+  tick = "1ns"
+  jitter = true
+  point_count = 100000 # number of points that will be written for each of the series
+  measurement = "cpu"
+  series_count = 100000
+
+  [[series.tag]]
+    key = "host"
+    value = "server"
+
+  [[series.tag]]
+    key = "location"
+    value = "loc"
+
+  [[series.field]]
+    key = "value"
+    type = "float64"

Godeps/_workspace/src/github.com/influxdb/influxdb/cmd/influx_stress/examples/100M_FLAT.toml

@@ -0,0 +1,47 @@
+channel_buffer_size = 100000
+
+[write]
+  concurrency = 10
+  batch_size = 10000
+  batch_interval = "0s"
+  database = "stress"
+  precision = "s"
+  address = "localhost:8086"
+  reset_database = true
+  start_date = "2006-Jan-02"
+
+[[series]]
+  tick = "5s"
+  jitter = false
+  point_count = 10000 # number of points that will be written for each of the series
+  measurement = "cpu"
+  series_count = 10000
+
+ # tag_count = 20 # number of "generic" tags on a series (e.g. tag-key-1=tag-value, ... ,tag-key-20=tag-value)
+
+  [[series.tag]]
+    key = "host"
+    value = "server"
+
+  [[series.tag]]
+    key = "location"
+    value = "loc"
+
+  [[series.field]]
+    key = "value"
+    type = "float64"
+
+  # Doesnt work as expected
+##  [[series.field]]
+##    key = "value-2"
+##    type = "float64-inc"
+##
+##  [[series.field]]
+##    key = "value-3"
+##    type = "float64-inc+"
+
+ # Has 80% probability of being a constant value
+  [[series.field]]
+    key = "flat_value"
+    type = "float64-flat"
+

Godeps/_workspace/src/github.com/influxdb/influxdb/cmd/influx_stress/examples/100M_STD.toml

@@ -0,0 +1,32 @@
+channel_buffer_size = 100000
+
+[write]
+  concurrency = 10
+  batch_size = 10000
+  batch_interval = "0s"
+  database = "stress"
+  precision = "s"
+  address = "localhost:8086"
+  reset_database = true
+  start_date = "2006-Jan-02"
+
+[[series]]
+  tick = "5s"
+  jitter = false
+  point_count = 10000 # number of points that will be written for each of the series
+  measurement = "cpu"
+  series_count = 10000
+
+ # tag_count = 20 # number of "generic" tags on a series (e.g. tag-key-1=tag-value, ... ,tag-key-20=tag-value)
+
+  [[series.tag]]
+    key = "host"
+    value = "server"
+
+  [[series.tag]]
+    key = "location"
+    value = "loc"
+
+  [[series.field]]
+    key = "value"
+    type = "float64"

Godeps/_workspace/src/github.com/influxdb/influxdb/cmd/influx_stress/examples/5M_STD.toml

@@ -0,0 +1,33 @@
+channel_buffer_size = 100000
+
+[write]
+  concurrency = 10
+  batch_size = 5000
+  batch_interval = "0s"
+  database = "stress"
+  precision = "s"
+  address = "localhost:8086"
+  reset_database = true
+  start_date = "2006-Jan-02"
+
+[[series]]
+  tick = "5s"
+  jitter = false
+  point_count = 50 # number of points that will be written for each of the series
+  measurement = "cpu"
+  series_count = 100000
+
+ # tag_count = 20 # number of "generic" tags on a series (e.g. tag-key-1=tag-value, ... ,tag-key-20=tag-value)
+
+  [[series.tag]]
+    key = "host"
+    value = "idk"
+
+  [[series.tag]]
+    key = "location"
+    value = "lame"
+
+  [[series.field]]
+    key = "value"
+    type = "float64"
+

Godeps/_workspace/src/github.com/influxdb/influxdb/cmd/influx_stress/examples/long_form_date.toml

@@ -0,0 +1,34 @@
+channel_buffer_size = 100000
+
+[write]
+  concurrency = 10
+  batch_size = 10000
+  batch_interval = "0s"
+  database = "stress"
+  precision = "s"
+  address = "localhost:8086"
+  reset_database = true
+  start_date = "Jan 2, 2006 at 3:04pm (MST)"
+
+[[series]]
+  tick = "5s"
+  jitter = true
+  point_count = 10000 # number of points that will be written for each of the series
+  measurement = "cpu"
+  series_count = 10000
+
+ # tag_count = 20 # number of "generic" tags on a series (e.g. tag-key-1=tag-value, ... ,tag-key-20=tag-value)
+
+  [[series.tag]]
+    key = "host"
+    value = "idk"
+
+  [[series.tag]]
+    key = "location"
+    value = "lame"
+
+  [[series.field]]
+    key = "value"
+    type = "float64"
+
+

Godeps/_workspace/src/github.com/influxdb/influxdb/cmd/influx_stress/examples/moderate_burn.toml

@@ -0,0 +1,30 @@
+channel_buffer_size = 100
+
+[write]
+  concurrency = 10
+  batch_size = 10000
+  batch_interval = "1s"
+  database = "stress"
+  precision = "n"
+  address = "localhost:8086"
+  reset_database = true
+
+[[series]]
+  tick = "5s"
+  jitter = true
+  point_count = 10000 # number of points that will be written for each of the series
+  measurement = "cpu"
+  series_count = 100000
+
+  [[series.tag]]
+    key = "host"
+    value = "server"
+
+  [[series.tag]]
+    key = "location"
+    value = "loc"
+
+  [[series.field]]
+    key = "value"
+    type = "float64"
+

Godeps/_workspace/src/github.com/influxdb/influxdb/cmd/influx_stress/examples/template.toml

@@ -0,0 +1,114 @@
+# Set the buffer size for the channel
+# that sends points to the influxdb
+# client
+channel_buffer_size = 100000
+
+# Configuration settings for the 
+# stress test
+[write]
+  # How many concurrent writers to the db
+  concurrency = 10
+  # Size of batches that are sent to db
+  batch_size = 5000
+  # Interval between each batch
+  batch_interval = "0s"
+  # Database that is being written to
+  database = "stress"
+  # Precision of points that are being written
+  precision = "n"
+  # Address of the Influxdb instance
+  address = "localhost:8086"
+  # Drop and Create new DB
+  reset_database = true
+  # The date for the first point that is written into influx
+  start_date = "2006-Jan-02"
+
+# Describes the schema for series that will be
+# written
+[[series]]
+  # How much time between each timestamp
+  tick = "5s"
+  # Randomize timestamp a bit
+  jitter = false
+  # number of points that will be written for each of the series
+  point_count = 1000
+  # name of the measurement that will be written
+  measurement = "cpu"
+  series_count = 10000
+
+  # number of "generic" tags on a series (e.g. tag-key-1=tag-value, ... ,tag-key-20=tag-value)
+  # tag_count = 20
+
+  # Defines a tag for a series
+  [[series.tag]]
+    key = "host"
+    value = "idk"
+
+  [[series.tag]]
+    key = "location"
+    value = "lame"
+
+  # Defines a field for a series
+  [[series.field]]
+    key = "value"
+    type = "float64"
+
+  [[series.field]]
+    key = "percent"
+    type = "int"
+
+  [[series.field]]
+    key = "idk"
+    type = "bool"
+
+  [[series.field]]
+    key = "default"
+
+[[series]]
+  tick = "1ns"
+  point_count = 100 # number of points that will be written for each of the series
+  measurement = "mem"
+  series_count = 100000
+
+  [[series.tag]]
+    key = "host"
+    value = "idk"
+
+  [[series.tag]]
+    key = "location"
+    value = "lame"
+
+  [[series.field]]
+    key = "value"
+    type = "float64"
+
+  [[series.field]]
+    key = "loc"
+    type = "float64"
+
+  [[series.field]]
+    key = "sunny"
+    type = "bool"
+
+  [[series.field]]
+    key = "idk"
+    type = "int"
+
+# Generates queries of the form
+# SELECT aggregates(values) FROM measurements WHERE time > current_timespan - offset
+[measurement_query]
+  enabled = true
+  concurrency = 10
+  aggregates = ["mean", "count"]
+  fields = ["value"]
+  offset = "5h"
+
+# Generates queries of the form
+# SELECT aggregates(values) FROM measurements WHERE tag-key='tag-values-1'
+[series_query]
+  enabled = true
+  concurrency = 1
+  aggregates = ["mean", "count"]
+  fields = ["value"]
+  # Interval between queries
+  interval = "50ms"

Godeps/_workspace/src/github.com/influxdb/influxdb/cmd/influx_stress/influx_stress.go

@@ -0,0 +1,121 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"runtime"
+	"sort"
+	"time"
+
+	"github.com/influxdb/influxdb/stress"
+)
+
+var (
+	batchSize     = flag.Int("batchsize", 5000, "number of points per batch")
+	seriesCount   = flag.Int("series", 100000, "number of unique series to create")
+	pointCount    = flag.Int("points", 100, "number of points per series to create")
+	concurrency   = flag.Int("concurrency", 10, "number of simultaneous writes to run")
+	batchInterval = flag.Duration("batchinterval", 0*time.Second, "duration between batches")
+	database      = flag.String("database", "stress", "name of database")
+	address       = flag.String("addr", "localhost:8086", "IP address and port of database (e.g., localhost:8086)")
+	precision     = flag.String("precision", "n", "The precision that points in the database will be with")
+	test          = flag.String("test", "", "The stress test file")
+)
+
+var ms runner.Measurements
+
+func init() {
+	flag.Var(&ms, "m", "comma-separated list of intervals to use between events")
+}
+
+func main() {
+	var cfg *runner.Config
+	var err error
+
+	runtime.GOMAXPROCS(runtime.NumCPU())
+	flag.Parse()
+
+	cfg = runner.NewConfig()
+
+	if len(ms) == 0 {
+		ms = append(ms, "cpu")
+	}
+
+	for _, m := range ms {
+		cfg.Series = append(cfg.Series, runner.NewSeries(m, 100, 100000))
+	}
+
+	if *test != "" {
+		cfg, err = runner.DecodeFile(*test)
+
+		if err != nil {
+			fmt.Println(err)
+			return
+		}
+
+	}
+
+	d := make(chan struct{})
+	seriesQueryResults := make(chan runner.QueryResults)
+
+	if cfg.SeriesQuery.Enabled {
+		go runner.SeriesQuery(cfg, d, seriesQueryResults)
+	}
+
+	measurementQueryResults := make(chan runner.QueryResults)
+
+	ts := make(chan time.Time)
+	if cfg.MeasurementQuery.Enabled {
+		go runner.MeasurementQuery(cfg, ts, measurementQueryResults)
+	}
+
+	// Get the stress results
+	totalPoints, failedRequests, responseTimes, timer := runner.Run(cfg, d, ts)
+
+	sort.Sort(sort.Reverse(sort.Interface(responseTimes)))
+
+	total := int64(0)
+	for _, t := range responseTimes {
+		total += int64(t.Value)
+	}
+	mean := total / int64(len(responseTimes))
+
+	fmt.Printf("Wrote %d points at average rate of %.0f\n", totalPoints, float64(totalPoints)/timer.Elapsed().Seconds())
+	fmt.Printf("%d requests failed for %d total points that didn't get posted.\n", failedRequests, failedRequests**batchSize)
+	fmt.Println("Average response time: ", time.Duration(mean))
+	fmt.Println("Slowest response times:")
+	for _, r := range responseTimes[:100] {
+		fmt.Println(time.Duration(r.Value))
+	}
+
+	// Get series query results
+	if cfg.SeriesQuery.Enabled {
+		qrs := <-seriesQueryResults
+
+		queryTotal := int64(0)
+		for _, qt := range qrs.ResponseTimes {
+			queryTotal += int64(qt.Value)
+		}
+		seriesQueryMean := queryTotal / int64(len(qrs.ResponseTimes))
+
+		fmt.Printf("Queried Series %d times with a average response time of %v milliseconds\n", qrs.TotalQueries, time.Duration(seriesQueryMean).Seconds()*1000)
+
+	}
+
+	// Get measurement query results
+	if cfg.MeasurementQuery.Enabled {
+		qrs := <-measurementQueryResults
+
+		queryTotal := int64(0)
+		for _, qt := range qrs.ResponseTimes {
+			queryTotal += int64(qt.Value)
+		}
+		seriesQueryMean := queryTotal / int64(len(qrs.ResponseTimes))
+
+		fmt.Printf("Queried Measurement %d times with a average response time of %v milliseconds\n", qrs.TotalQueries, time.Duration(seriesQueryMean).Seconds()*1000)
+
+	}
+
+	return
+
+}

Godeps/_workspace/src/github.com/influxdb/influxdb/cmd/influxd/backup/backup.go

@@ -0,0 +1,170 @@
+package backup
+
+import (
+	"encoding/json"
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"log"
+	"net"
+	"os"
+
+	"github.com/influxdb/influxdb/services/snapshotter"
+	"github.com/influxdb/influxdb/snapshot"
+)
+
+// Suffix is a suffix added to the backup while it's in-process.
+const Suffix = ".pending"
+
+// Command represents the program execution for "influxd backup".
+type Command struct {
+	// The logger passed to the ticker during execution.
+	Logger *log.Logger
+
+	// Standard input/output, overridden for testing.
+	Stderr io.Writer
+}
+
+// NewCommand returns a new instance of Command with default settings.
+func NewCommand() *Command {
+	return &Command{
+		Stderr: os.Stderr,
+	}
+}
+
+// Run executes the program.
+func (cmd *Command) Run(args ...string) error {
+	// Set up logger.
+	cmd.Logger = log.New(cmd.Stderr, "", log.LstdFlags)
+	cmd.Logger.Printf("influxdb backup")
+
+	// Parse command line arguments.
+	host, path, err := cmd.parseFlags(args)
+	if err != nil {
+		return err
+	}
+
+	// Retrieve snapshot from local file.
+	m, err := snapshot.ReadFileManifest(path)
+	if err != nil && !os.IsNotExist(err) {
+		return fmt.Errorf("read file snapshot: %s", err)
+	}
+
+	// Determine temporary path to download to.
+	tmppath := path + Suffix
+
+	// Calculate path of next backup file.
+	// This uses the path if it doesn't exist.
+	// Otherwise it appends an autoincrementing number.
+	path, err = cmd.nextPath(path)
+	if err != nil {
+		return fmt.Errorf("next path: %s", err)
+	}
+
+	// Retrieve snapshot.
+	if err := cmd.download(host, m, tmppath); err != nil {
+		return fmt.Errorf("download: %s", err)
+	}
+
+	// Rename temporary file to final path.
+	if err := os.Rename(tmppath, path); err != nil {
+		return fmt.Errorf("rename: %s", err)
+	}
+
+	// TODO: Check file integrity.
+
+	// Notify user of completion.
+	cmd.Logger.Println("backup complete")
+
+	return nil
+}
+
+// parseFlags parses and validates the command line arguments.
+func (cmd *Command) parseFlags(args []string) (host string, path string, err error) {
+	fs := flag.NewFlagSet("", flag.ContinueOnError)
+	fs.StringVar(&host, "host", "localhost:8088", "")
+	fs.SetOutput(cmd.Stderr)
+	fs.Usage = cmd.printUsage
+	if err := fs.Parse(args); err != nil {
+		return "", "", err
+	}
+
+	// Ensure that only one arg is specified.
+	if fs.NArg() == 0 {
+		return "", "", errors.New("snapshot path required")
+	} else if fs.NArg() != 1 {
+		return "", "", errors.New("only one snapshot path allowed")
+	}
+	path = fs.Arg(0)
+
+	return host, path, nil
+}
+
+// nextPath returns the next file to write to.
+func (cmd *Command) nextPath(path string) (string, error) {
+	// Use base path if it doesn't exist.
+	if _, err := os.Stat(path); os.IsNotExist(err) {
+		return path, nil
+	} else if err != nil {
+		return "", err
+	}
+
+	// Otherwise iterate through incremental files until one is available.
+	for i := 0; ; i++ {
+		s := fmt.Sprintf(path+".%d", i)
+		if _, err := os.Stat(s); os.IsNotExist(err) {
+			return s, nil
+		} else if err != nil {
+			return "", err
+		}
+	}
+}
+
+// download downloads a snapshot from a host to a given path.
+func (cmd *Command) download(host string, m *snapshot.Manifest, path string) error {
+	// Create local file to write to.
+	f, err := os.Create(path)
+	if err != nil {
+		return fmt.Errorf("open temp file: %s", err)
+	}
+	defer f.Close()
+
+	// Connect to snapshotter service.
+	conn, err := net.Dial("tcp", host)
+	if err != nil {
+		return err
+	}
+	defer conn.Close()
+
+	// Send snapshotter marker byte.
+	if _, err := conn.Write([]byte{snapshotter.MuxHeader}); err != nil {
+		return fmt.Errorf("write snapshot header byte: %s", err)
+	}
+
+	// Write the manifest we currently have.
+	if err := json.NewEncoder(conn).Encode(m); err != nil {
+		return fmt.Errorf("encode snapshot manifest: %s", err)
+	}
+
+	// Read snapshot from the connection.
+	if _, err := io.Copy(f, conn); err != nil {
+		return fmt.Errorf("copy snapshot to file: %s", err)
+	}
+
+	// FIXME(benbjohnson): Verify integrity of snapshot.
+
+	return nil
+}
+
+// printUsage prints the usage message to STDERR.
+func (cmd *Command) printUsage() {
+	fmt.Fprintf(cmd.Stderr, `usage: influxd backup [flags] PATH
+
+backup downloads a snapshot of a data node and saves it to disk.
+
+        -host <host:port>
+                          The host to connect to snapshot.
+                          Defaults to 127.0.0.1:8088.
+`)
+}

Godeps/_workspace/src/github.com/influxdb/influxdb/cmd/influxd/backup/backup_test.go

@@ -0,0 +1,125 @@
+package backup_test
+
+/*
+import (
+	"bytes"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"strings"
+	"testing"
+
+	"github.com/influxdb/influxdb"
+	"github.com/influxdb/influxdb/cmd/influxd"
+)
+
+// Ensure the backup can download from the server and save to disk.
+func TestBackupCommand(t *testing.T) {
+	// Mock the backup endpoint.
+	s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path != "/data/snapshot" {
+			t.Fatalf("unexpected url path: %s", r.URL.Path)
+		}
+
+		// Write a simple snapshot to the buffer.
+		sw := influxdb.NewSnapshotWriter()
+		sw.Snapshot = &influxdb.Snapshot{Files: []influxdb.SnapshotFile{
+			{Name: "meta", Size: 5, Index: 10},
+		}}
+		sw.FileWriters["meta"] = influxdb.NopWriteToCloser(bytes.NewBufferString("55555"))
+		if _, err := sw.WriteTo(w); err != nil {
+			t.Fatal(err)
+		}
+	}))
+	defer s.Close()
+
+	// Create a temp path and remove incremental backups at the end.
+	path := tempfile()
+	defer os.Remove(path)
+	defer os.Remove(path + ".0")
+	defer os.Remove(path + ".1")
+
+	// Execute the backup against the mock server.
+	for i := 0; i < 3; i++ {
+		if err := NewBackupCommand().Run("-host", s.URL, path); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	// Verify snapshot and two incremental snapshots were written.
+	if _, err := os.Stat(path); err != nil {
+		t.Fatalf("snapshot not found: %s", err)
+	} else if _, err = os.Stat(path + ".0"); err != nil {
+		t.Fatalf("incremental snapshot(0) not found: %s", err)
+	} else if _, err = os.Stat(path + ".1"); err != nil {
+		t.Fatalf("incremental snapshot(1) not found: %s", err)
+	}
+}
+
+// Ensure the backup command returns an error if flags cannot be parsed.
+func TestBackupCommand_ErrFlagParse(t *testing.T) {
+	cmd := NewBackupCommand()
+	if err := cmd.Run("-bad-flag"); err == nil || err.Error() != `flag provided but not defined: -bad-flag` {
+		t.Fatal(err)
+	} else if !strings.Contains(cmd.Stderr.String(), "usage") {
+		t.Fatal("usage message not displayed")
+	}
+}
+
+// Ensure the backup command returns an error if the host cannot be parsed.
+func TestBackupCommand_ErrInvalidHostURL(t *testing.T) {
+	if err := NewBackupCommand().Run("-host", "http://%f"); err == nil || err.Error() != `parse host url: parse http://%f: hexadecimal escape in host` {
+		t.Fatal(err)
+	}
+}
+
+// Ensure the backup command returns an error if the output path is not specified.
+func TestBackupCommand_ErrPathRequired(t *testing.T) {
+	if err := NewBackupCommand().Run("-host", "//localhost"); err == nil || err.Error() != `snapshot path required` {
+		t.Fatal(err)
+	}
+}
+
+// Ensure the backup returns an error if it cannot connect to the server.
+func TestBackupCommand_ErrConnectionRefused(t *testing.T) {
+	// Start and immediately stop a server so we have a dead port.
+	s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}))
+	s.Close()
+
+	// Execute the backup command.
+	path := tempfile()
+	defer os.Remove(path)
+	if err := NewBackupCommand().Run("-host", s.URL, path); err == nil ||
+		!(strings.Contains(err.Error(), `connection refused`) || strings.Contains(err.Error(), `No connection could be made`)) {
+		t.Fatal(err)
+	}
+}
+
+// Ensure the backup returns any non-200 status codes.
+func TestBackupCommand_ErrServerError(t *testing.T) {
+	s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusInternalServerError)
+	}))
+	defer s.Close()
+
+	// Execute the backup command.
+	path := tempfile()
+	defer os.Remove(path)
+	if err := NewBackupCommand().Run("-host", s.URL, path); err == nil || err.Error() != `download: snapshot error: status=500` {
+		t.Fatal(err)
+	}
+}
+
+// BackupCommand is a test wrapper for main.BackupCommand.
+type BackupCommand struct {
+	*main.BackupCommand
+	Stderr bytes.Buffer
+}
+
+// NewBackupCommand returns a new instance of BackupCommand.
+func NewBackupCommand() *BackupCommand {
+	cmd := &BackupCommand{BackupCommand: main.NewBackupCommand()}
+	cmd.BackupCommand.Stderr = &cmd.Stderr
+	return cmd
+}
+*/

Godeps/_workspace/src/github.com/influxdb/influxdb/cmd/influxd/help/help.go

@@ -0,0 +1,46 @@
+package help
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"strings"
+)
+
+// Command displays help for command-line sub-commands.
+type Command struct {
+	Stdout io.Writer
+}
+
+// NewCommand returns a new instance of Command.
+func NewCommand() *Command {
+	return &Command{
+		Stdout: os.Stdout,
+	}
+}
+
+// Run executes the command.
+func (cmd *Command) Run(args ...string) error {
+	fmt.Fprintln(cmd.Stdout, strings.TrimSpace(usage))
+	return nil
+}
+
+const usage = `
+Configure and start an InfluxDB server.
+
+Usage:
+
+	influxd [[command] [arguments]]
+
+The commands are:
+
+    backup               downloads a snapshot of a data node and saves it to disk
+    config               display the default configuration
+    restore              uses a snapshot of a data node to rebuild a cluster
+    run                  run node with existing configuration
+    version              displays the InfluxDB version
+
+"run" is the default command.
+
+Use "influxd help [command]" for more information about a command.
+`

Godeps/_workspace/src/github.com/influxdb/influxdb/cmd/influxd/main.go

@@ -0,0 +1,205 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"io"
+	"log"
+	"math/rand"
+	"os"
+	"os/signal"
+	"strings"
+	"syscall"
+	"time"
+
+	"github.com/influxdb/influxdb/cmd/influxd/backup"
+	"github.com/influxdb/influxdb/cmd/influxd/help"
+	"github.com/influxdb/influxdb/cmd/influxd/restore"
+	"github.com/influxdb/influxdb/cmd/influxd/run"
+)
+
+// These variables are populated via the Go linker.
+var (
+	version   string = "0.9"
+	commit    string
+	branch    string
+	buildTime string
+)
+
+func init() {
+	// If commit, branch, or build time are not set, make that clear.
+	if commit == "" {
+		commit = "unknown"
+	}
+	if branch == "" {
+		branch = "unknown"
+	}
+	if buildTime == "" {
+		buildTime = "unknown"
+	}
+}
+
+func main() {
+	rand.Seed(time.Now().UnixNano())
+
+	m := NewMain()
+	if err := m.Run(os.Args[1:]...); err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		os.Exit(1)
+	}
+}
+
+// Main represents the program execution.
+type Main struct {
+	Logger *log.Logger
+
+	Stdin  io.Reader
+	Stdout io.Writer
+	Stderr io.Writer
+}
+
+// NewMain return a new instance of Main.
+func NewMain() *Main {
+	return &Main{
+		Logger: log.New(os.Stderr, "[run] ", log.LstdFlags),
+		Stdin:  os.Stdin,
+		Stdout: os.Stdout,
+		Stderr: os.Stderr,
+	}
+}
+
+// Run determines and runs the command specified by the CLI args.
+func (m *Main) Run(args ...string) error {
+	name, args := ParseCommandName(args)
+
+	// Extract name from args.
+	switch name {
+	case "", "run":
+		cmd := run.NewCommand()
+
+		// Tell the server the build details.
+		cmd.Version = version
+		cmd.Commit = commit
+		cmd.Branch = branch
+		cmd.BuildTime = buildTime
+
+		if err := cmd.Run(args...); err != nil {
+			return fmt.Errorf("run: %s", err)
+		}
+
+		signalCh := make(chan os.Signal, 1)
+		signal.Notify(signalCh, os.Interrupt, syscall.SIGTERM)
+		m.Logger.Println("Listening for signals")
+
+		// Block until one of the signals above is received
+		select {
+		case <-signalCh:
+			m.Logger.Println("Signal received, initializing clean shutdown...")
+			go func() {
+				cmd.Close()
+			}()
+		}
+
+		// Block again until another signal is received, a shutdown timeout elapses,
+		// or the Command is gracefully closed
+		m.Logger.Println("Waiting for clean shutdown...")
+		select {
+		case <-signalCh:
+			m.Logger.Println("second signal received, initializing hard shutdown")
+		case <-time.After(time.Second * 30):
+			m.Logger.Println("time limit reached, initializing hard shutdown")
+		case <-cmd.Closed:
+			m.Logger.Println("server shutdown completed")
+		}
+
+		// goodbye.
+
+	case "backup":
+		name := backup.NewCommand()
+		if err := name.Run(args...); err != nil {
+			return fmt.Errorf("backup: %s", err)
+		}
+	case "restore":
+		name := restore.NewCommand()
+		if err := name.Run(args...); err != nil {
+			return fmt.Errorf("restore: %s", err)
+		}
+	case "config":
+		if err := run.NewPrintConfigCommand().Run(args...); err != nil {
+			return fmt.Errorf("config: %s", err)
+		}
+	case "version":
+		if err := NewVersionCommand().Run(args...); err != nil {
+			return fmt.Errorf("version: %s", err)
+		}
+	case "help":
+		if err := help.NewCommand().Run(args...); err != nil {
+			return fmt.Errorf("help: %s", err)
+		}
+	default:
+		return fmt.Errorf(`unknown command "%s"`+"\n"+`Run 'influxd help' for usage`+"\n\n", name)
+	}
+
+	return nil
+}
+
+// ParseCommandName extracts the command name and args from the args list.
+func ParseCommandName(args []string) (string, []string) {
+	// Retrieve command name as first argument.
+	var name string
+	if len(args) > 0 && !strings.HasPrefix(args[0], "-") {
+		name = args[0]
+	}
+
+	// Special case -h immediately following binary name
+	if len(args) > 0 && args[0] == "-h" {
+		name = "help"
+	}
+
+	// If command is "help" and has an argument then rewrite args to use "-h".
+	if name == "help" && len(args) > 1 {
+		args[0], args[1] = args[1], "-h"
+		name = args[0]
+	}
+
+	// If a named command is specified then return it with its arguments.
+	if name != "" {
+		return name, args[1:]
+	}
+	return "", args
+}
+
+// Command represents the command executed by "influxd version".
+type VersionCommand struct {
+	Stdout io.Writer
+	Stderr io.Writer
+}
+
+// NewVersionCommand return a new instance of VersionCommand.
+func NewVersionCommand() *VersionCommand {
+	return &VersionCommand{
+		Stdout: os.Stdout,
+		Stderr: os.Stderr,
+	}
+}
+
+// Run prints the current version and commit info.
+func (cmd *VersionCommand) Run(args ...string) error {
+	// Parse flags in case -h is specified.
+	fs := flag.NewFlagSet("", flag.ContinueOnError)
+	fs.Usage = func() { fmt.Fprintln(cmd.Stderr, strings.TrimSpace(versionUsage)) }
+	if err := fs.Parse(args); err != nil {
+		return err
+	}
+
+	// Print version info.
+	fmt.Fprintf(cmd.Stdout, "InfluxDB v%s (git: %s %s, built %s)\n", version, branch, commit, buildTime)
+
+	return nil
+}
+
+var versionUsage = `
+usage: version
+
+	version displays the InfluxDB version, build branch and git commit hash
+`

Godeps/_workspace/src/github.com/influxdb/influxdb/cmd/influxd/restore/restore.go

@@ -0,0 +1,250 @@
+package restore
+
+import (
+	"bytes"
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"net"
+	"os"
+	"path/filepath"
+
+	"github.com/BurntSushi/toml"
+	"github.com/influxdb/influxdb/meta"
+	"github.com/influxdb/influxdb/snapshot"
+	"github.com/influxdb/influxdb/tsdb"
+)
+
+// Command represents the program execution for "influxd restore".
+type Command struct {
+	Stdout io.Writer
+	Stderr io.Writer
+}
+
+// NewCommand returns a new instance of Command with default settings.
+func NewCommand() *Command {
+	return &Command{
+		Stdout: os.Stdout,
+		Stderr: os.Stderr,
+	}
+}
+
+// Run executes the program.
+func (cmd *Command) Run(args ...string) error {
+	config, path, err := cmd.parseFlags(args)
+	if err != nil {
+		return err
+	}
+
+	return cmd.Restore(config, path)
+}
+
+func (cmd *Command) Restore(config *Config, path string) error {
+	// Remove meta and data directories.
+	if err := os.RemoveAll(config.Meta.Dir); err != nil {
+		return fmt.Errorf("remove meta dir: %s", err)
+	} else if err := os.RemoveAll(config.Data.Dir); err != nil {
+		return fmt.Errorf("remove data dir: %s", err)
+	}
+
+	// Open snapshot file and all incremental backups.
+	mr, files, err := snapshot.OpenFileMultiReader(path)
+	if err != nil {
+		return fmt.Errorf("open multireader: %s", err)
+	}
+	defer closeAll(files)
+
+	// Unpack files from archive.
+	if err := cmd.unpack(mr, config); err != nil {
+		return fmt.Errorf("unpack: %s", err)
+	}
+
+	// Notify user of completion.
+	fmt.Fprintf(os.Stdout, "restore complete using %s", path)
+	return nil
+}
+
+// parseFlags parses and validates the command line arguments.
+func (cmd *Command) parseFlags(args []string) (*Config, string, error) {
+	fs := flag.NewFlagSet("", flag.ContinueOnError)
+	configPath := fs.String("config", "", "")
+	fs.SetOutput(cmd.Stderr)
+	fs.Usage = cmd.printUsage
+	if err := fs.Parse(args); err != nil {
+		return nil, "", err
+	}
+
+	// Parse configuration file from disk.
+	if *configPath == "" {
+		return nil, "", fmt.Errorf("config required")
+	}
+
+	// Parse config.
+	config := Config{
+		Meta: meta.NewConfig(),
+		Data: tsdb.NewConfig(),
+	}
+	if _, err := toml.DecodeFile(*configPath, &config); err != nil {
+		return nil, "", err
+	}
+
+	// Require output path.
+	path := fs.Arg(0)
+	if path == "" {
+		return nil, "", fmt.Errorf("snapshot path required")
+	}
+
+	return &config, path, nil
+}
+
+func closeAll(a []io.Closer) {
+	for _, c := range a {
+		_ = c.Close()
+	}
+}
+
+// unpack expands the files in the snapshot archive into a directory.
+func (cmd *Command) unpack(mr *snapshot.MultiReader, config *Config) error {
+	// Loop over files and extract.
+	for {
+		// Read entry header.
+		sf, err := mr.Next()
+		if err == io.EOF {
+			break
+		} else if err != nil {
+			return fmt.Errorf("next: entry=%s, err=%s", sf.Name, err)
+		}
+
+		// Log progress.
+		fmt.Fprintf(os.Stdout, "unpacking: %s (%d bytes)\n", sf.Name, sf.Size)
+
+		// Handle meta and tsdb files separately.
+		switch sf.Name {
+		case "meta":
+			if err := cmd.unpackMeta(mr, sf, config); err != nil {
+				return fmt.Errorf("meta: %s", err)
+			}
+		default:
+			if err := cmd.unpackData(mr, sf, config); err != nil {
+				return fmt.Errorf("data: %s", err)
+			}
+		}
+	}
+
+	return nil
+}
+
+// unpackMeta reads the metadata from the snapshot and initializes a raft
+// cluster and replaces the root metadata.
+func (cmd *Command) unpackMeta(mr *snapshot.MultiReader, sf snapshot.File, config *Config) error {
+	// Read meta into buffer.
+	var buf bytes.Buffer
+	if _, err := io.CopyN(&buf, mr, sf.Size); err != nil {
+		return fmt.Errorf("copy: %s", err)
+	}
+
+	// Unpack into metadata.
+	var data meta.Data
+	if err := data.UnmarshalBinary(buf.Bytes()); err != nil {
+		return fmt.Errorf("unmarshal: %s", err)
+	}
+
+	// Copy meta config and remove peers so it starts in single mode.
+	c := config.Meta
+	c.Peers = nil
+
+	// Initialize meta store.
+	store := meta.NewStore(config.Meta)
+	store.RaftListener = newNopListener()
+	store.ExecListener = newNopListener()
+
+	// Determine advertised address.
+	_, port, err := net.SplitHostPort(config.Meta.BindAddress)
+	if err != nil {
+		return fmt.Errorf("split bind address: %s", err)
+	}
+	hostport := net.JoinHostPort(config.Meta.Hostname, port)
+
+	// Resolve address.
+	addr, err := net.ResolveTCPAddr("tcp", hostport)
+	if err != nil {
+		return fmt.Errorf("resolve tcp: addr=%s, err=%s", hostport, err)
+	}
+	store.Addr = addr
+
+	// Open the meta store.
+	if err := store.Open(); err != nil {
+		return fmt.Errorf("open store: %s", err)
+	}
+	defer store.Close()
+
+	// Wait for the store to be ready or error.
+	select {
+	case <-store.Ready():
+	case err := <-store.Err():
+		return err
+	}
+
+	// Force set the full metadata.
+	if err := store.SetData(&data); err != nil {
+		return fmt.Errorf("set data: %s", err)
+	}
+
+	return nil
+}
+
+func (cmd *Command) unpackData(mr *snapshot.MultiReader, sf snapshot.File, config *Config) error {
+	path := filepath.Join(config.Data.Dir, sf.Name)
+	// Create parent directory for output file.
+	if err := os.MkdirAll(filepath.Dir(path), 0777); err != nil {
+		return fmt.Errorf("mkdir: entry=%s, err=%s", sf.Name, err)
+	}
+
+	// Create output file.
+	f, err := os.Create(path)
+	if err != nil {
+		return fmt.Errorf("create: entry=%s, err=%s", sf.Name, err)
+	}
+	defer f.Close()
+
+	// Copy contents from reader.
+	if _, err := io.CopyN(f, mr, sf.Size); err != nil {
+		return fmt.Errorf("copy: entry=%s, err=%s", sf.Name, err)
+	}
+
+	return nil
+}
+
+// printUsage prints the usage message to STDERR.
+func (cmd *Command) printUsage() {
+	fmt.Fprintf(cmd.Stderr, `usage: influxd restore [flags] PATH
+
+restore uses a snapshot of a data node to rebuild a cluster.
+
+        -config <path>
+                          Set the path to the configuration file.
+`)
+}
+
+// Config represents a partial config for rebuilding the server.
+type Config struct {
+	Meta *meta.Config `toml:"meta"`
+	Data tsdb.Config  `toml:"data"`
+}
+
+type nopListener struct {
+	closing chan struct{}
+}
+
+func newNopListener() *nopListener {
+	return &nopListener{make(chan struct{})}
+}
+
+func (ln *nopListener) Accept() (net.Conn, error) {
+	<-ln.closing
+	return nil, errors.New("listener closing")
+}
+
+func (ln *nopListener) Close() error   { close(ln.closing); return nil }
+func (ln *nopListener) Addr() net.Addr { return nil }

Godeps/_workspace/src/github.com/influxdb/influxdb/cmd/influxd/restore/restore_test.go

@@ -0,0 +1,155 @@
+package restore_test
+
+/*
+import (
+	"bytes"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"reflect"
+	"testing"
+	"time"
+
+	main "github.com/influxdb/influxdb/cmd/influxd"
+	"github.com/influxdb/influxdb/tsdb"
+)
+
+func newConfig(path string, port int) main.Config {
+	config := main.NewConfig()
+	config.Port = port
+	config.Broker.Enabled = true
+	config.Broker.Dir = filepath.Join(path, "broker")
+
+	config.Data.Enabled = true
+	config.Data.Dir = filepath.Join(path, "data")
+	return *config
+}
+
+// Ensure the restore command can expand a snapshot and bootstrap a broker.
+func TestRestoreCommand(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping TestRestoreCommand")
+	}
+
+	now := time.Now()
+
+	// Create root path to server.
+	path := tempfile()
+	defer os.Remove(path)
+
+	// Parse configuration.
+	config := newConfig(path, 8900)
+
+	// Start server.
+	cmd := main.NewRunCommand()
+	node := cmd.Open(&config, "")
+	if node.Broker == nil {
+		t.Fatal("cannot run broker")
+	} else if node.DataNode == nil {
+		t.Fatal("cannot run server")
+	}
+	b := node.Broker
+	s := node.DataNode
+
+	// Create data.
+	if err := s.CreateDatabase("db"); err != nil {
+		t.Fatalf("cannot create database: %s", err)
+	}
+	if index, err := s.WriteSeries("db", "default", []models.Point{tsdb.NewPoint("cpu", nil, map[string]interface{}{"value": float64(100)}, now)}); err != nil {
+		t.Fatalf("cannot write series: %s", err)
+	} else if err = s.Sync(1, index); err != nil {
+		t.Fatalf("shard sync: %s", err)
+	}
+
+	// Create snapshot writer.
+	sw, err := s.CreateSnapshotWriter()
+	if err != nil {
+		t.Fatalf("create snapshot writer: %s", err)
+	}
+
+	// Snapshot to file.
+	sspath := tempfile()
+	f, err := os.Create(sspath)
+	if err != nil {
+		t.Fatal(err)
+	}
+	sw.WriteTo(f)
+	f.Close()
+
+	// Stop server.
+	node.Close()
+
+	// Remove data & broker directories.
+	if err := os.RemoveAll(path); err != nil {
+		t.Fatalf("remove: %s", err)
+	}
+
+	// Execute the restore.
+	if err := NewRestoreCommand().Restore(&config, sspath); err != nil {
+		t.Fatal(err)
+	}
+
+	// Rewrite config to a new port and re-parse.
+	config = newConfig(path, 8910)
+
+	// Restart server.
+	cmd = main.NewRunCommand()
+	node = cmd.Open(&config, "")
+	if b == nil {
+		t.Fatal("cannot run broker")
+	} else if s == nil {
+		t.Fatal("cannot run server")
+	}
+	b = node.Broker
+	s = node.DataNode
+
+	// Write new data.
+	if err := s.CreateDatabase("newdb"); err != nil {
+		t.Fatalf("cannot create new database: %s", err)
+	}
+	if index, err := s.WriteSeries("newdb", "default", []models.Point{tsdb.NewPoint("mem", nil, map[string]interface{}{"value": float64(1000)}, now)}); err != nil {
+		t.Fatalf("cannot write new series: %s", err)
+	} else if err = s.Sync(2, index); err != nil {
+		t.Fatalf("shard sync: %s", err)
+	}
+
+	// Read series data.
+	if v, err := s.ReadSeries("db", "default", "cpu", nil, now); err != nil {
+		t.Fatal(err)
+	} else if !reflect.DeepEqual(v, map[string]interface{}{"value": float64(100)}) {
+		t.Fatalf("read series(0) mismatch: %#v", v)
+	}
+
+	// Read new series data.
+	if v, err := s.ReadSeries("newdb", "default", "mem", nil, now); err != nil {
+		t.Fatal(err)
+	} else if !reflect.DeepEqual(v, map[string]interface{}{"value": float64(1000)}) {
+		t.Fatalf("read series(1) mismatch: %#v", v)
+	}
+
+	// Stop server.
+	node.Close()
+}
+
+// RestoreCommand is a test wrapper for main.RestoreCommand.
+type RestoreCommand struct {
+	*main.RestoreCommand
+	Stderr bytes.Buffer
+}
+
+// NewRestoreCommand returns a new instance of RestoreCommand.
+func NewRestoreCommand() *RestoreCommand {
+	cmd := &RestoreCommand{RestoreCommand: main.NewRestoreCommand()}
+	cmd.RestoreCommand.Stderr = &cmd.Stderr
+	return cmd
+}
+
+// MustReadFile reads data from a file. Panic on error.
+func MustReadFile(filename string) []byte {
+	b, err := ioutil.ReadFile(filename)
+	if err != nil {
+		panic(err.Error())
+	}
+	return b
+}
+*/

Godeps/_workspace/src/github.com/influxdb/influxdb/cmd/influxd/run/command.go

@@ -0,0 +1,240 @@
+package run
+
+import (
+	"flag"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"log"
+	"os"
+	"path/filepath"
+	"runtime"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/BurntSushi/toml"
+)
+
+const logo = `
+ 8888888           .d888 888                   8888888b.  888888b.
+   888            d88P"  888                   888  "Y88b 888  "88b
+   888            888    888                   888    888 888  .88P
+   888   88888b.  888888 888 888  888 888  888 888    888 8888888K.
+   888   888 "88b 888    888 888  888  Y8bd8P' 888    888 888  "Y88b
+   888   888  888 888    888 888  888   X88K   888    888 888    888
+   888   888  888 888    888 Y88b 888 .d8""8b. 888  .d88P 888   d88P
+ 8888888 888  888 888    888  "Y88888 888  888 8888888P"  8888888P"
+
+`
+
+// Command represents the command executed by "influxd run".
+type Command struct {
+	Version   string
+	Branch    string
+	Commit    string
+	BuildTime string
+
+	closing chan struct{}
+	Closed  chan struct{}
+
+	Stdin  io.Reader
+	Stdout io.Writer
+	Stderr io.Writer
+
+	Server *Server
+}
+
+// NewCommand return a new instance of Command.
+func NewCommand() *Command {
+	return &Command{
+		closing: make(chan struct{}),
+		Closed:  make(chan struct{}),
+		Stdin:   os.Stdin,
+		Stdout:  os.Stdout,
+		Stderr:  os.Stderr,
+	}
+}
+
+// Run parses the config from args and runs the server.
+func (cmd *Command) Run(args ...string) error {
+	// Parse the command line flags.
+	options, err := cmd.ParseFlags(args...)
+	if err != nil {
+		return err
+	}
+
+	// Print sweet InfluxDB logo.
+	fmt.Print(logo)
+
+	// Mark start-up in log.
+	log.Printf("InfluxDB starting, version %s, branch %s, commit %s, built %s",
+		cmd.Version, cmd.Branch, cmd.Commit, cmd.BuildTime)
+	log.Printf("Go version %s, GOMAXPROCS set to %d", runtime.Version(), runtime.GOMAXPROCS(0))
+
+	// Write the PID file.
+	if err := cmd.writePIDFile(options.PIDFile); err != nil {
+		return fmt.Errorf("write pid file: %s", err)
+	}
+
+	// Turn on block profiling to debug stuck databases
+	runtime.SetBlockProfileRate(int(1 * time.Second))
+
+	// Parse config
+	config, err := cmd.ParseConfig(options.ConfigPath)
+	if err != nil {
+		return fmt.Errorf("parse config: %s", err)
+	}
+
+	// Apply any environment variables on top of the parsed config
+	if err := config.ApplyEnvOverrides(); err != nil {
+		return fmt.Errorf("apply env config: %v", err)
+	}
+
+	// Override config hostname if specified in the command line args.
+	if options.Hostname != "" {
+		config.Meta.Hostname = options.Hostname
+	}
+
+	if options.Join != "" {
+		config.Meta.Peers = strings.Split(options.Join, ",")
+	}
+
+	// Validate the configuration.
+	if err := config.Validate(); err != nil {
+		return fmt.Errorf("%s. To generate a valid configuration file run `influxd config > influxdb.generated.conf`.", err)
+	}
+
+	// Create server from config and start it.
+	buildInfo := &BuildInfo{
+		Version: cmd.Version,
+		Commit:  cmd.Commit,
+		Branch:  cmd.Branch,
+		Time:    cmd.BuildTime,
+	}
+	s, err := NewServer(config, buildInfo)
+	if err != nil {
+		return fmt.Errorf("create server: %s", err)
+	}
+	s.CPUProfile = options.CPUProfile
+	s.MemProfile = options.MemProfile
+	if err := s.Open(); err != nil {
+		return fmt.Errorf("open server: %s", err)
+	}
+	cmd.Server = s
+
+	// Begin monitoring the server's error channel.
+	go cmd.monitorServerErrors()
+
+	return nil
+}
+
+// Close shuts down the server.
+func (cmd *Command) Close() error {
+	defer close(cmd.Closed)
+	close(cmd.closing)
+	if cmd.Server != nil {
+		return cmd.Server.Close()
+	}
+	return nil
+}
+
+func (cmd *Command) monitorServerErrors() {
+	logger := log.New(cmd.Stderr, "", log.LstdFlags)
+	for {
+		select {
+		case err := <-cmd.Server.Err():
+			logger.Println(err)
+		case <-cmd.closing:
+			return
+		}
+	}
+}
+
+// ParseFlags parses the command line flags from args and returns an options set.
+func (cmd *Command) ParseFlags(args ...string) (Options, error) {
+	var options Options
+	fs := flag.NewFlagSet("", flag.ContinueOnError)
+	fs.StringVar(&options.ConfigPath, "config", "", "")
+	fs.StringVar(&options.PIDFile, "pidfile", "", "")
+	fs.StringVar(&options.Hostname, "hostname", "", "")
+	fs.StringVar(&options.Join, "join", "", "")
+	fs.StringVar(&options.CPUProfile, "cpuprofile", "", "")
+	fs.StringVar(&options.MemProfile, "memprofile", "", "")
+	fs.Usage = func() { fmt.Fprintln(cmd.Stderr, usage) }
+	if err := fs.Parse(args); err != nil {
+		return Options{}, err
+	}
+	return options, nil
+}
+
+// writePIDFile writes the process ID to path.
+func (cmd *Command) writePIDFile(path string) error {
+	// Ignore if path is not set.
+	if path == "" {
+		return nil
+	}
+
+	// Ensure the required directory structure exists.
+	err := os.MkdirAll(filepath.Dir(path), 0777)
+	if err != nil {
+		return fmt.Errorf("mkdir: %s", err)
+	}
+
+	// Retrieve the PID and write it.
+	pid := strconv.Itoa(os.Getpid())
+	if err := ioutil.WriteFile(path, []byte(pid), 0666); err != nil {
+		return fmt.Errorf("write file: %s", err)
+	}
+
+	return nil
+}
+
+// ParseConfig parses the config at path.
+// Returns a demo configuration if path is blank.
+func (cmd *Command) ParseConfig(path string) (*Config, error) {
+	// Use demo configuration if no config path is specified.
+	if path == "" {
+		log.Println("no configuration provided, using default settings")
+		return NewDemoConfig()
+	}
+
+	log.Printf("Using configuration at: %s\n", path)
+
+	config := NewConfig()
+	if _, err := toml.DecodeFile(path, &config); err != nil {
+		return nil, err
+	}
+
+	return config, nil
+}
+
+var usage = `usage: run [flags]
+
+run starts the broker and data node server. If this is the first time running
+the command then a new cluster will be initialized unless the -join argument
+is used.
+
+        -config <path>
+                          Set the path to the configuration file.
+
+        -hostname <name>
+                          Override the hostname, the 'hostname' configuration
+                          option will be overridden.
+
+        -join <url>
+                          Joins the server to an existing cluster.
+
+        -pidfile <path>
+                          Write process ID to a file.
+`
+
+// Options represents the command line options that can be parsed.
+type Options struct {
+	ConfigPath string
+	PIDFile    string
+	Hostname   string
+	Join       string
+	CPUProfile string
+	MemProfile string
+}

Godeps/_workspace/src/github.com/influxdb/influxdb/cmd/influxd/run/config.go

@@ -0,0 +1,225 @@
+package run
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"os/user"
+	"path/filepath"
+	"reflect"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/influxdb/influxdb/cluster"
+	"github.com/influxdb/influxdb/meta"
+	"github.com/influxdb/influxdb/monitor"
+	"github.com/influxdb/influxdb/services/admin"
+	"github.com/influxdb/influxdb/services/collectd"
+	"github.com/influxdb/influxdb/services/continuous_querier"
+	"github.com/influxdb/influxdb/services/graphite"
+	"github.com/influxdb/influxdb/services/hh"
+	"github.com/influxdb/influxdb/services/httpd"
+	"github.com/influxdb/influxdb/services/opentsdb"
+	"github.com/influxdb/influxdb/services/precreator"
+	"github.com/influxdb/influxdb/services/retention"
+	"github.com/influxdb/influxdb/services/udp"
+	"github.com/influxdb/influxdb/tsdb"
+)
+
+// Config represents the configuration format for the influxd binary.
+type Config struct {
+	Meta       *meta.Config      `toml:"meta"`
+	Data       tsdb.Config       `toml:"data"`
+	Cluster    cluster.Config    `toml:"cluster"`
+	Retention  retention.Config  `toml:"retention"`
+	Precreator precreator.Config `toml:"shard-precreation"`
+
+	Admin     admin.Config      `toml:"admin"`
+	Monitor   monitor.Config    `toml:"monitor"`
+	HTTPD     httpd.Config      `toml:"http"`
+	Graphites []graphite.Config `toml:"graphite"`
+	Collectd  collectd.Config   `toml:"collectd"`
+	OpenTSDB  opentsdb.Config   `toml:"opentsdb"`
+	UDPs      []udp.Config      `toml:"udp"`
+
+	// Snapshot SnapshotConfig `toml:"snapshot"`
+	ContinuousQuery continuous_querier.Config `toml:"continuous_queries"`
+
+	HintedHandoff hh.Config `toml:"hinted-handoff"`
+
+	// Server reporting
+	ReportingDisabled bool `toml:"reporting-disabled"`
+}
+
+// NewConfig returns an instance of Config with reasonable defaults.
+func NewConfig() *Config {
+	c := &Config{}
+	c.Meta = meta.NewConfig()
+	c.Data = tsdb.NewConfig()
+	c.Cluster = cluster.NewConfig()
+	c.Precreator = precreator.NewConfig()
+
+	c.Admin = admin.NewConfig()
+	c.Monitor = monitor.NewConfig()
+	c.HTTPD = httpd.NewConfig()
+	c.Collectd = collectd.NewConfig()
+	c.OpenTSDB = opentsdb.NewConfig()
+
+	c.ContinuousQuery = continuous_querier.NewConfig()
+	c.Retention = retention.NewConfig()
+	c.HintedHandoff = hh.NewConfig()
+
+	return c
+}
+
+// NewDemoConfig returns the config that runs when no config is specified.
+func NewDemoConfig() (*Config, error) {
+	c := NewConfig()
+
+	var homeDir string
+	// By default, store meta and data files in current users home directory
+	u, err := user.Current()
+	if err == nil {
+		homeDir = u.HomeDir
+	} else if os.Getenv("HOME") != "" {
+		homeDir = os.Getenv("HOME")
+	} else {
+		return nil, fmt.Errorf("failed to determine current user for storage")
+	}
+
+	c.Meta.Dir = filepath.Join(homeDir, ".influxdb/meta")
+	c.Data.Dir = filepath.Join(homeDir, ".influxdb/data")
+	c.HintedHandoff.Dir = filepath.Join(homeDir, ".influxdb/hh")
+	c.Data.WALDir = filepath.Join(homeDir, ".influxdb/wal")
+
+	c.Admin.Enabled = true
+
+	return c, nil
+}
+
+// Validate returns an error if the config is invalid.
+func (c *Config) Validate() error {
+	if c.Meta.Dir == "" {
+		return errors.New("Meta.Dir must be specified")
+	} else if c.Data.Dir == "" {
+		return errors.New("Data.Dir must be specified")
+	} else if c.HintedHandoff.Dir == "" {
+		return errors.New("HintedHandoff.Dir must be specified")
+	} else if c.Data.WALDir == "" {
+		return errors.New("Data.WALDir must be specified")
+	}
+
+	for _, g := range c.Graphites {
+		if err := g.Validate(); err != nil {
+			return fmt.Errorf("invalid graphite config: %v", err)
+		}
+	}
+	return nil
+}
+
+func (c *Config) ApplyEnvOverrides() error {
+	return c.applyEnvOverrides("INFLUXDB", reflect.ValueOf(c))
+}
+
+func (c *Config) applyEnvOverrides(prefix string, spec reflect.Value) error {
+	// If we have a pointer, dereference it
+	s := spec
+	if spec.Kind() == reflect.Ptr {
+		s = spec.Elem()
+	}
+
+	// Make sure we have struct
+	if s.Kind() != reflect.Struct {
+		return nil
+	}
+
+	typeOfSpec := s.Type()
+	for i := 0; i < s.NumField(); i++ {
+		f := s.Field(i)
+		// Get the toml tag to determine what env var name to use
+		configName := typeOfSpec.Field(i).Tag.Get("toml")
+		// Replace hyphens with underscores to avoid issues with shells
+		configName = strings.Replace(configName, "-", "_", -1)
+		fieldName := typeOfSpec.Field(i).Name
+
+		// Skip any fields that we cannot set
+		if f.CanSet() || f.Kind() == reflect.Slice {
+
+			// Use the upper-case prefix and toml name for the env var
+			key := strings.ToUpper(configName)
+			if prefix != "" {
+				key = strings.ToUpper(fmt.Sprintf("%s_%s", prefix, configName))
+			}
+			value := os.Getenv(key)
+
+			// If the type is s slice, apply to each using the index as a suffix
+			// e.g. GRAPHITE_0
+			if f.Kind() == reflect.Slice || f.Kind() == reflect.Array {
+				for i := 0; i < f.Len(); i++ {
+					if err := c.applyEnvOverrides(fmt.Sprintf("%s_%d", key, i), f.Index(i)); err != nil {
+						return err
+					}
+				}
+				continue
+			}
+
+			// If it's a sub-config, recursively apply
+			if f.Kind() == reflect.Struct || f.Kind() == reflect.Ptr {
+				if err := c.applyEnvOverrides(key, f); err != nil {
+					return err
+				}
+				continue
+			}
+
+			// Skip any fields we don't have a value to set
+			if value == "" {
+				continue
+			}
+
+			switch f.Kind() {
+			case reflect.String:
+				f.SetString(value)
+			case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
+
+				var intValue int64
+
+				// Handle toml.Duration
+				if f.Type().Name() == "Duration" {
+					dur, err := time.ParseDuration(value)
+					if err != nil {
+						return fmt.Errorf("failed to apply %v to %v using type %v and value '%v'", key, fieldName, f.Type().String(), value)
+					}
+					intValue = dur.Nanoseconds()
+				} else {
+					var err error
+					intValue, err = strconv.ParseInt(value, 0, f.Type().Bits())
+					if err != nil {
+						return fmt.Errorf("failed to apply %v to %v using type %v and value '%v'", key, fieldName, f.Type().String(), value)
+					}
+				}
+
+				f.SetInt(intValue)
+			case reflect.Bool:
+				boolValue, err := strconv.ParseBool(value)
+				if err != nil {
+					return fmt.Errorf("failed to apply %v to %v using type %v and value '%v'", key, fieldName, f.Type().String(), value)
+
+				}
+				f.SetBool(boolValue)
+			case reflect.Float32, reflect.Float64:
+				floatValue, err := strconv.ParseFloat(value, f.Type().Bits())
+				if err != nil {
+					return fmt.Errorf("failed to apply %v to %v using type %v and value '%v'", key, fieldName, f.Type().String(), value)
+
+				}
+				f.SetFloat(floatValue)
+			default:
+				if err := c.applyEnvOverrides(key, f); err != nil {
+					return err
+				}
+			}
+		}
+	}
+	return nil
+}

Godeps/_workspace/src/github.com/influxdb/influxdb/cmd/influxd/run/config_command.go

@@ -0,0 +1,83 @@
+package run
+
+import (
+	"flag"
+	"fmt"
+	"io"
+	"os"
+
+	"github.com/BurntSushi/toml"
+)
+
+// PrintConfigCommand represents the command executed by "influxd config".
+type PrintConfigCommand struct {
+	Stdin  io.Reader
+	Stdout io.Writer
+	Stderr io.Writer
+}
+
+// NewPrintConfigCommand return a new instance of PrintConfigCommand.
+func NewPrintConfigCommand() *PrintConfigCommand {
+	return &PrintConfigCommand{
+		Stdin:  os.Stdin,
+		Stdout: os.Stdout,
+		Stderr: os.Stderr,
+	}
+}
+
+// Run parses and prints the current config loaded.
+func (cmd *PrintConfigCommand) Run(args ...string) error {
+	// Parse command flags.
+	fs := flag.NewFlagSet("", flag.ContinueOnError)
+	configPath := fs.String("config", "", "")
+	hostname := fs.String("hostname", "", "")
+	fs.Usage = func() { fmt.Fprintln(cmd.Stderr, printConfigUsage) }
+	if err := fs.Parse(args); err != nil {
+		return err
+	}
+
+	// Parse config from path.
+	config, err := cmd.parseConfig(*configPath)
+	if err != nil {
+		return fmt.Errorf("parse config: %s", err)
+	}
+
+	// Apply any environment variables on top of the parsed config
+	if err := config.ApplyEnvOverrides(); err != nil {
+		return fmt.Errorf("apply env config: %v", err)
+	}
+
+	// Override config properties.
+	if *hostname != "" {
+		config.Meta.Hostname = *hostname
+	}
+
+	// Validate the configuration.
+	if err := config.Validate(); err != nil {
+		return fmt.Errorf("%s. To generate a valid configuration file run `influxd config > influxdb.generated.conf`.", err)
+	}
+
+	toml.NewEncoder(cmd.Stdout).Encode(config)
+	fmt.Fprint(cmd.Stdout, "\n")
+
+	return nil
+}
+
+// ParseConfig parses the config at path.
+// Returns a demo configuration if path is blank.
+func (cmd *PrintConfigCommand) parseConfig(path string) (*Config, error) {
+	if path == "" {
+		return NewDemoConfig()
+	}
+
+	config := NewConfig()
+	if _, err := toml.DecodeFile(path, &config); err != nil {
+		return nil, err
+	}
+	return config, nil
+}
+
+var printConfigUsage = `usage: config
+
+	config displays the default configuration
+`

Godeps/_workspace/src/github.com/influxdb/influxdb/cmd/influxd/run/config_test.go

@@ -0,0 +1,142 @@
+package run_test
+
+import (
+	"os"
+	"testing"
+
+	"github.com/BurntSushi/toml"
+	"github.com/influxdb/influxdb/cmd/influxd/run"
+)
+
+// Ensure the configuration can be parsed.
+func TestConfig_Parse(t *testing.T) {
+	// Parse configuration.
+	var c run.Config
+	if _, err := toml.Decode(`
+[meta]
+dir = "/tmp/meta"
+
+[data]
+dir = "/tmp/data"
+
+[cluster]
+
+[admin]
+bind-address = ":8083"
+
+[http]
+bind-address = ":8087"
+
+[[graphite]]
+protocol = "udp"
+
+[[graphite]]
+protocol = "tcp"
+
+[collectd]
+bind-address = ":1000"
+
+[opentsdb]
+bind-address = ":2000"
+
+[[udp]]
+bind-address = ":4444"
+
+[monitoring]
+enabled = true
+
+[continuous_queries]
+enabled = true
+`, &c); err != nil {
+		t.Fatal(err)
+	}
+
+	// Validate configuration.
+	if c.Meta.Dir != "/tmp/meta" {
+		t.Fatalf("unexpected meta dir: %s", c.Meta.Dir)
+	} else if c.Data.Dir != "/tmp/data" {
+		t.Fatalf("unexpected data dir: %s", c.Data.Dir)
+	} else if c.Admin.BindAddress != ":8083" {
+		t.Fatalf("unexpected admin bind address: %s", c.Admin.BindAddress)
+	} else if c.HTTPD.BindAddress != ":8087" {
+		t.Fatalf("unexpected api bind address: %s", c.HTTPD.BindAddress)
+	} else if len(c.Graphites) != 2 {
+		t.Fatalf("unexpected graphites count: %d", len(c.Graphites))
+	} else if c.Graphites[0].Protocol != "udp" {
+		t.Fatalf("unexpected graphite protocol(0): %s", c.Graphites[0].Protocol)
+	} else if c.Graphites[1].Protocol != "tcp" {
+		t.Fatalf("unexpected graphite protocol(1): %s", c.Graphites[1].Protocol)
+	} else if c.Collectd.BindAddress != ":1000" {
+		t.Fatalf("unexpected collectd bind address: %s", c.Collectd.BindAddress)
+	} else if c.OpenTSDB.BindAddress != ":2000" {
+		t.Fatalf("unexpected opentsdb bind address: %s", c.OpenTSDB.BindAddress)
+	} else if c.UDPs[0].BindAddress != ":4444" {
+		t.Fatalf("unexpected udp bind address: %s", c.UDPs[0].BindAddress)
+	} else if c.ContinuousQuery.Enabled != true {
+		t.Fatalf("unexpected continuous query enabled: %v", c.ContinuousQuery.Enabled)
+	}
+}
+
+// Ensure the configuration can be parsed.
+func TestConfig_Parse_EnvOverride(t *testing.T) {
+	// Parse configuration.
+	var c run.Config
+	if _, err := toml.Decode(`
+[meta]
+dir = "/tmp/meta"
+
+[data]
+dir = "/tmp/data"
+
+[cluster]
+
+[admin]
+bind-address = ":8083"
+
+[http]
+bind-address = ":8087"
+
+[[graphite]]
+protocol = "udp"
+
+[[graphite]]
+protocol = "tcp"
+
+[collectd]
+bind-address = ":1000"
+
+[opentsdb]
+bind-address = ":2000"
+
+[[udp]]
+bind-address = ":4444"
+
+[monitoring]
+enabled = true
+
+[continuous_queries]
+enabled = true
+`, &c); err != nil {
+		t.Fatal(err)
+	}
+
+	if err := os.Setenv("INFLUXDB_UDP_BIND_ADDRESS", ":1234"); err != nil {
+		t.Fatalf("failed to set env var: %v", err)
+	}
+
+	if err := os.Setenv("INFLUXDB_GRAPHITE_1_PROTOCOL", "udp"); err != nil {
+		t.Fatalf("failed to set env var: %v", err)
+	}
+
+	if err := c.ApplyEnvOverrides(); err != nil {
+		t.Fatalf("failed to apply env overrides: %v", err)
+	}
+
+	if c.UDPs[0].BindAddress != ":4444" {
+		t.Fatalf("unexpected udp bind address: %s", c.UDPs[0].BindAddress)
+	}
+
+	if c.Graphites[1].Protocol != "udp" {
+		t.Fatalf("unexpected graphite protocol(0): %s", c.Graphites[0].Protocol)
+	}
+}

Godeps/_workspace/src/github.com/influxdb/influxdb/cmd/influxd/run/server.go

@@ -0,0 +1,589 @@
+package run
+
+import (
+	"bytes"
+	"fmt"
+	"log"
+	"net"
+	"net/http"
+	"os"
+	"runtime"
+	"runtime/pprof"
+	"strings"
+	"time"
+
+	"github.com/influxdb/influxdb/cluster"
+	"github.com/influxdb/influxdb/meta"
+	"github.com/influxdb/influxdb/monitor"
+	"github.com/influxdb/influxdb/services/admin"
+	"github.com/influxdb/influxdb/services/collectd"
+	"github.com/influxdb/influxdb/services/continuous_querier"
+	"github.com/influxdb/influxdb/services/copier"
+	"github.com/influxdb/influxdb/services/graphite"
+	"github.com/influxdb/influxdb/services/hh"
+	"github.com/influxdb/influxdb/services/httpd"
+	"github.com/influxdb/influxdb/services/opentsdb"
+	"github.com/influxdb/influxdb/services/precreator"
+	"github.com/influxdb/influxdb/services/retention"
+	"github.com/influxdb/influxdb/services/snapshotter"
+	"github.com/influxdb/influxdb/services/udp"
+	"github.com/influxdb/influxdb/tcp"
+	"github.com/influxdb/influxdb/tsdb"
+	_ "github.com/influxdb/influxdb/tsdb/engine"
+)
+
+// BuildInfo represents the build details for the server code.
+type BuildInfo struct {
+	Version string
+	Commit  string
+	Branch  string
+	Time    string
+}
+
+// Server represents a container for the metadata and storage data and services.
+// It is built using a Config and it manages the startup and shutdown of all
+// services in the proper order.
+type Server struct {
+	buildInfo BuildInfo
+
+	err     chan error
+	closing chan struct{}
+
+	Hostname    string
+	BindAddress string
+	Listener    net.Listener
+
+	MetaStore     *meta.Store
+	TSDBStore     *tsdb.Store
+	QueryExecutor *tsdb.QueryExecutor
+	PointsWriter  *cluster.PointsWriter
+	ShardWriter   *cluster.ShardWriter
+	ShardMapper   *cluster.ShardMapper
+	HintedHandoff *hh.Service
+
+	Services []Service
+
+	// These references are required for the tcp muxer.
+	ClusterService     *cluster.Service
+	SnapshotterService *snapshotter.Service
+	CopierService      *copier.Service
+
+	Monitor *monitor.Monitor
+
+	// Server reporting
+	reportingDisabled bool
+
+	// Profiling
+	CPUProfile string
+	MemProfile string
+}
+
+// NewServer returns a new instance of Server built from a config.
+func NewServer(c *Config, buildInfo *BuildInfo) (*Server, error) {
+	// Construct base meta store and data store.
+	tsdbStore := tsdb.NewStore(c.Data.Dir)
+	tsdbStore.EngineOptions.Config = c.Data
+
+	s := &Server{
+		buildInfo: *buildInfo,
+		err:       make(chan error),
+		closing:   make(chan struct{}),
+
+		Hostname:    c.Meta.Hostname,
+		BindAddress: c.Meta.BindAddress,
+
+		MetaStore: meta.NewStore(c.Meta),
+		TSDBStore: tsdbStore,
+
+		Monitor: monitor.New(c.Monitor),
+
+		reportingDisabled: c.ReportingDisabled,
+	}
+
+	// Copy TSDB configuration.
+	s.TSDBStore.EngineOptions.EngineVersion = c.Data.Engine
+	s.TSDBStore.EngineOptions.MaxWALSize = c.Data.MaxWALSize
+	s.TSDBStore.EngineOptions.WALFlushInterval = time.Duration(c.Data.WALFlushInterval)
+	s.TSDBStore.EngineOptions.WALPartitionFlushDelay = time.Duration(c.Data.WALPartitionFlushDelay)
+
+	// Set the shard mapper
+	s.ShardMapper = cluster.NewShardMapper(time.Duration(c.Cluster.ShardMapperTimeout))
+	s.ShardMapper.ForceRemoteMapping = c.Cluster.ForceRemoteShardMapping
+	s.ShardMapper.MetaStore = s.MetaStore
+	s.ShardMapper.TSDBStore = s.TSDBStore
+
+	// Initialize query executor.
+	s.QueryExecutor = tsdb.NewQueryExecutor(s.TSDBStore)
+	s.QueryExecutor.MetaStore = s.MetaStore
+	s.QueryExecutor.MetaStatementExecutor = &meta.StatementExecutor{Store: s.MetaStore}
+	s.QueryExecutor.MonitorStatementExecutor = &monitor.StatementExecutor{Monitor: s.Monitor}
+	s.QueryExecutor.ShardMapper = s.ShardMapper
+	s.QueryExecutor.QueryLogEnabled = c.Data.QueryLogEnabled
+
+	// Set the shard writer
+	s.ShardWriter = cluster.NewShardWriter(time.Duration(c.Cluster.ShardWriterTimeout))
+	s.ShardWriter.MetaStore = s.MetaStore
+
+	// Create the hinted handoff service
+	s.HintedHandoff = hh.NewService(c.HintedHandoff, s.ShardWriter)
+
+	// Initialize points writer.
+	s.PointsWriter = cluster.NewPointsWriter()
+	s.PointsWriter.WriteTimeout = time.Duration(c.Cluster.WriteTimeout)
+	s.PointsWriter.MetaStore = s.MetaStore
+	s.PointsWriter.TSDBStore = s.TSDBStore
+	s.PointsWriter.ShardWriter = s.ShardWriter
+	s.PointsWriter.HintedHandoff = s.HintedHandoff
+
+	// Initialize the monitor
+	s.Monitor.Version = s.buildInfo.Version
+	s.Monitor.Commit = s.buildInfo.Commit
+	s.Monitor.Branch = s.buildInfo.Branch
+	s.Monitor.BuildTime = s.buildInfo.Time
+	s.Monitor.MetaStore = s.MetaStore
+	s.Monitor.PointsWriter = s.PointsWriter
+
+	// Append services.
+	s.appendClusterService(c.Cluster)
+	s.appendPrecreatorService(c.Precreator)
+	s.appendSnapshotterService()
+	s.appendCopierService()
+	s.appendAdminService(c.Admin)
+	s.appendContinuousQueryService(c.ContinuousQuery)
+	s.appendHTTPDService(c.HTTPD)
+	s.appendCollectdService(c.Collectd)
+	if err := s.appendOpenTSDBService(c.OpenTSDB); err != nil {
+		return nil, err
+	}
+	for _, g := range c.UDPs {
+		s.appendUDPService(g)
+	}
+	s.appendRetentionPolicyService(c.Retention)
+	for _, g := range c.Graphites {
+		if err := s.appendGraphiteService(g); err != nil {
+			return nil, err
+		}
+	}
+
+	return s, nil
+}
+
+func (s *Server) appendClusterService(c cluster.Config) {
+	srv := cluster.NewService(c)
+	srv.TSDBStore = s.TSDBStore
+	srv.MetaStore = s.MetaStore
+	s.Services = append(s.Services, srv)
+	s.ClusterService = srv
+}
+
+func (s *Server) appendSnapshotterService() {
+	srv := snapshotter.NewService()
+	srv.TSDBStore = s.TSDBStore
+	srv.MetaStore = s.MetaStore
+	s.Services = append(s.Services, srv)
+	s.SnapshotterService = srv
+}
+
+func (s *Server) appendCopierService() {
+	srv := copier.NewService()
+	srv.TSDBStore = s.TSDBStore
+	s.Services = append(s.Services, srv)
+	s.CopierService = srv
+}
+
+func (s *Server) appendRetentionPolicyService(c retention.Config) {
+	if !c.Enabled {
+		return
+	}
+	srv := retention.NewService(c)
+	srv.MetaStore = s.MetaStore
+	srv.TSDBStore = s.TSDBStore
+	s.Services = append(s.Services, srv)
+}
+
+func (s *Server) appendAdminService(c admin.Config) {
+	if !c.Enabled {
+		return
+	}
+	srv := admin.NewService(c)
+	s.Services = append(s.Services, srv)
+}
+
+func (s *Server) appendHTTPDService(c httpd.Config) {
+	if !c.Enabled {
+		return
+	}
+	srv := httpd.NewService(c)
+	srv.Handler.MetaStore = s.MetaStore
+	srv.Handler.QueryExecutor = s.QueryExecutor
+	srv.Handler.PointsWriter = s.PointsWriter
+	srv.Handler.Version = s.buildInfo.Version
+
+	// If a ContinuousQuerier service has been started, attach it.
+	for _, srvc := range s.Services {
+		if cqsrvc, ok := srvc.(continuous_querier.ContinuousQuerier); ok {
+			srv.Handler.ContinuousQuerier = cqsrvc
+		}
+	}
+
+	s.Services = append(s.Services, srv)
+}
+
+func (s *Server) appendCollectdService(c collectd.Config) {
+	if !c.Enabled {
+		return
+	}
+	srv := collectd.NewService(c)
+	srv.MetaStore = s.MetaStore
+	srv.PointsWriter = s.PointsWriter
+	s.Services = append(s.Services, srv)
+}
+
+func (s *Server) appendOpenTSDBService(c opentsdb.Config) error {
+	if !c.Enabled {
+		return nil
+	}
+	srv, err := opentsdb.NewService(c)
+	if err != nil {
+		return err
+	}
+	srv.PointsWriter = s.PointsWriter
+	srv.MetaStore = s.MetaStore
+	s.Services = append(s.Services, srv)
+	return nil
+}
+
+func (s *Server) appendGraphiteService(c graphite.Config) error {
+	if !c.Enabled {
+		return nil
+	}
+	srv, err := graphite.NewService(c)
+	if err != nil {
+		return err
+	}
+
+	srv.PointsWriter = s.PointsWriter
+	srv.MetaStore = s.MetaStore
+	srv.Monitor = s.Monitor
+	s.Services = append(s.Services, srv)
+	return nil
+}
+
+func (s *Server) appendPrecreatorService(c precreator.Config) error {
+	if !c.Enabled {
+		return nil
+	}
+	srv, err := precreator.NewService(c)
+	if err != nil {
+		return err
+	}
+
+	srv.MetaStore = s.MetaStore
+	s.Services = append(s.Services, srv)
+	return nil
+}
+
+func (s *Server) appendUDPService(c udp.Config) {
+	if !c.Enabled {
+		return
+	}
+	srv := udp.NewService(c)
+	srv.PointsWriter = s.PointsWriter
+	s.Services = append(s.Services, srv)
+}
+
+func (s *Server) appendContinuousQueryService(c continuous_querier.Config) {
+	if !c.Enabled {
+		return
+	}
+	srv := continuous_querier.NewService(c)
+	srv.MetaStore = s.MetaStore
+	srv.QueryExecutor = s.QueryExecutor
+	srv.PointsWriter = s.PointsWriter
+	s.Services = append(s.Services, srv)
+}
+
+// Err returns an error channel that multiplexes all out of band errors received from all services.
+func (s *Server) Err() <-chan error { return s.err }
+
+// Open opens the meta and data store and all services.
+func (s *Server) Open() error {
+	if err := func() error {
+		// Start profiling, if set.
+		startProfile(s.CPUProfile, s.MemProfile)
+
+		host, port, err := s.hostAddr()
+		if err != nil {
+			return err
+		}
+
+		hostport := net.JoinHostPort(host, port)
+		addr, err := net.ResolveTCPAddr("tcp", hostport)
+		if err != nil {
+			return fmt.Errorf("resolve tcp: addr=%s, err=%s", hostport, err)
+		}
+		s.MetaStore.Addr = addr
+		s.MetaStore.RemoteAddr = &tcpaddr{hostport}
+
+		// Open shared TCP connection.
+		ln, err := net.Listen("tcp", s.BindAddress)
+		if err != nil {
+			return fmt.Errorf("listen: %s", err)
+		}
+		s.Listener = ln
+
+		// The port 0 is used, we need to retrieve the port assigned by the kernel
+		if strings.HasSuffix(s.BindAddress, ":0") {
+			s.MetaStore.Addr = ln.Addr()
+		}
+
+		// Multiplex listener.
+		mux := tcp.NewMux()
+		s.MetaStore.RaftListener = mux.Listen(meta.MuxRaftHeader)
+		s.MetaStore.ExecListener = mux.Listen(meta.MuxExecHeader)
+		s.MetaStore.RPCListener = mux.Listen(meta.MuxRPCHeader)
+
+		s.ClusterService.Listener = mux.Listen(cluster.MuxHeader)
+		s.SnapshotterService.Listener = mux.Listen(snapshotter.MuxHeader)
+		s.CopierService.Listener = mux.Listen(copier.MuxHeader)
+		go mux.Serve(ln)
+
+		// Open meta store.
+		if err := s.MetaStore.Open(); err != nil {
+			return fmt.Errorf("open meta store: %s", err)
+		}
+		go s.monitorErrorChan(s.MetaStore.Err())
+
+		// Wait for the store to initialize.
+		<-s.MetaStore.Ready()
+
+		if err := s.Monitor.Open(); err != nil {
+			return fmt.Errorf("open monitor: %v", err)
+		}
+
+		// Open TSDB store.
+		if err := s.TSDBStore.Open(); err != nil {
+			return fmt.Errorf("open tsdb store: %s", err)
+		}
+
+		// Open the hinted handoff service
+		if err := s.HintedHandoff.Open(); err != nil {
+			return fmt.Errorf("open hinted handoff: %s", err)
+		}
+
+		for _, service := range s.Services {
+			if err := service.Open(); err != nil {
+				return fmt.Errorf("open service: %s", err)
+			}
+		}
+
+		// Start the reporting service, if not disabled.
+		if !s.reportingDisabled {
+			go s.startServerReporting()
+		}
+
+		return nil
+
+	}(); err != nil {
+		s.Close()
+		return err
+	}
+
+	return nil
+}
+
+// Close shuts down the meta and data stores and all services.
+func (s *Server) Close() error {
+	stopProfile()
+
+	// Close the listener first to stop any new connections
+	if s.Listener != nil {
+		s.Listener.Close()
+	}
+
+	// Close services to allow any inflight requests to complete
+	// and prevent new requests from being accepted.
+	for _, service := range s.Services {
+		service.Close()
+	}
+
+	if s.Monitor != nil {
+		s.Monitor.Close()
+	}
+
+	if s.HintedHandoff != nil {
+		s.HintedHandoff.Close()
+	}
+
+	// Close the TSDBStore, no more reads or writes at this point
+	if s.TSDBStore != nil {
+		s.TSDBStore.Close()
+	}
+
+	// Finally close the meta-store since everything else depends on it
+	if s.MetaStore != nil {
+		s.MetaStore.Close()
+	}
+
+	close(s.closing)
+	return nil
+}
+
+// startServerReporting starts periodic server reporting.
+func (s *Server) startServerReporting() {
+	for {
+		select {
+		case <-s.closing:
+			return
+		default:
+		}
+		if err := s.MetaStore.WaitForLeader(30 * time.Second); err != nil {
+			log.Printf("no leader available for reporting: %s", err.Error())
+			time.Sleep(time.Second)
+			continue
+		}
+		s.reportServer()
+		<-time.After(24 * time.Hour)
+	}
+}
+
+// reportServer reports anonymous statistics about the system.
+func (s *Server) reportServer() {
+	dis, err := s.MetaStore.Databases()
+	if err != nil {
+		log.Printf("failed to retrieve databases for reporting: %s", err.Error())
+		return
+	}
+	numDatabases := len(dis)
+
+	numMeasurements := 0
+	numSeries := 0
+	for _, di := range dis {
+		d := s.TSDBStore.DatabaseIndex(di.Name)
+		if d == nil {
+			// No data in this store for this database.
+			continue
+		}
+		m, s := d.MeasurementSeriesCounts()
+		numMeasurements += m
+		numSeries += s
+	}
+
+	clusterID, err := s.MetaStore.ClusterID()
+	if err != nil {
+		log.Printf("failed to retrieve cluster ID for reporting: %s", err.Error())
+		return
+	}
+
+	json := fmt.Sprintf(`[{
+    "name":"reports",
+    "columns":["os", "arch", "version", "server_id", "cluster_id", "num_series", "num_measurements", "num_databases"],
+    "points":[["%s", "%s", "%s", "%x", "%x", "%d", "%d", "%d"]]
+  }]`, runtime.GOOS, runtime.GOARCH, s.buildInfo.Version, s.MetaStore.NodeID(), clusterID, numSeries, numMeasurements, numDatabases)
+
+	data := bytes.NewBufferString(json)
+
+	log.Printf("Sending anonymous usage statistics to m.influxdb.com")
+
+	client := http.Client{Timeout: time.Duration(5 * time.Second)}
+	go client.Post("http://m.influxdb.com:8086/db/reporting/series?u=reporter&p=influxdb", "application/json", data)
+}
+
+// monitorErrorChan reads an error channel and resends it through the server.
+func (s *Server) monitorErrorChan(ch <-chan error) {
+	for {
+		select {
+		case err, ok := <-ch:
+			if !ok {
+				return
+			}
+			s.err <- err
+		case <-s.closing:
+			return
+		}
+	}
+}
+
+// hostAddr returns the host and port that remote nodes will use to reach this
+// node.
+func (s *Server) hostAddr() (string, string, error) {
+	// Resolve host to address.
+	_, port, err := net.SplitHostPort(s.BindAddress)
+	if err != nil {
+		return "", "", fmt.Errorf("split bind address: %s", err)
+	}
+
+	host := s.Hostname
+
+	// See if we might have a port that will override the BindAddress port
+	if host != "" && host[len(host)-1] >= '0' && host[len(host)-1] <= '9' && strings.Contains(host, ":") {
+		hostArg, portArg, err := net.SplitHostPort(s.Hostname)
+		if err != nil {
+			return "", "", err
+		}
+
+		if hostArg != "" {
+			host = hostArg
+		}
+
+		if portArg != "" {
+			port = portArg
+		}
+	}
+	return host, port, nil
+}
+
+// Service represents a service attached to the server.
+type Service interface {
+	Open() error
+	Close() error
+}
+
+// prof stores the file locations of active profiles.
+var prof struct {
+	cpu *os.File
+	mem *os.File
+}
+
+// StartProfile initializes the cpu and memory profile, if specified.
+func startProfile(cpuprofile, memprofile string) {
+	if cpuprofile != "" {
+		f, err := os.Create(cpuprofile)
+		if err != nil {
+			log.Fatalf("cpuprofile: %v", err)
+		}
+		log.Printf("writing CPU profile to: %s\n", cpuprofile)
+		prof.cpu = f
+		pprof.StartCPUProfile(prof.cpu)
+	}
+
+	if memprofile != "" {
+		f, err := os.Create(memprofile)
+		if err != nil {
+			log.Fatalf("memprofile: %v", err)
+		}
+		log.Printf("writing mem profile to: %s\n", memprofile)
+		prof.mem = f
+		runtime.MemProfileRate = 4096
+	}
+
+}
+
+// StopProfile closes the cpu and memory profiles if they are running.
+func stopProfile() {
+	if prof.cpu != nil {
+		pprof.StopCPUProfile()
+		prof.cpu.Close()
+		log.Println("CPU profile stopped")
+	}
+	if prof.mem != nil {
+		pprof.Lookup("heap").WriteTo(prof.mem, 0)
+		prof.mem.Close()
+		log.Println("mem profile stopped")
+	}
+}
+
+type tcpaddr struct{ host string }
+
+func (a *tcpaddr) Network() string { return "tcp" }
+func (a *tcpaddr) String() string  { return a.host }

Godeps/_workspace/src/github.com/influxdb/influxdb/cmd/influxd/run/server_helpers_test.go

@@ -0,0 +1,356 @@
+// This package is a set of convenience helpers and structs to make integration testing easier
+package run_test
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"log"
+	"math"
+	"net/http"
+	"net/url"
+	"os"
+	"regexp"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/influxdb/influxdb/cmd/influxd/run"
+	"github.com/influxdb/influxdb/meta"
+	"github.com/influxdb/influxdb/services/httpd"
+	"github.com/influxdb/influxdb/toml"
+)
+
+// Server represents a test wrapper for run.Server.
+type Server struct {
+	*run.Server
+	Config *run.Config
+}
+
+// NewServer returns a new instance of Server.
+func NewServer(c *run.Config) *Server {
+	buildInfo := &run.BuildInfo{
+		Version: "testServer",
+		Commit:  "testCommit",
+		Branch:  "testBranch",
+	}
+	srv, _ := run.NewServer(c, buildInfo)
+	s := Server{
+		Server: srv,
+		Config: c,
+	}
+	s.TSDBStore.EngineOptions.Config = c.Data
+	configureLogging(&s)
+	return &s
+}
+
+// OpenServer opens a test server.
+func OpenServer(c *run.Config, joinURLs string) *Server {
+	s := NewServer(c)
+	configureLogging(s)
+	if err := s.Open(); err != nil {
+		panic(err.Error())
+	}
+
+	return s
+}
+
+// OpenServerWithVersion opens a test server with a specific version.
+func OpenServerWithVersion(c *run.Config, version string) *Server {
+	buildInfo := &run.BuildInfo{
+		Version: version,
+		Commit:  "",
+		Branch:  "",
+	}
+	srv, _ := run.NewServer(c, buildInfo)
+	s := Server{
+		Server: srv,
+		Config: c,
+	}
+	configureLogging(&s)
+	if err := s.Open(); err != nil {
+		panic(err.Error())
+	}
+
+	return &s
+}
+
+// Close shuts down the server and removes all temporary paths.
+func (s *Server) Close() {
+	os.RemoveAll(s.Config.Meta.Dir)
+	os.RemoveAll(s.Config.Data.Dir)
+	os.RemoveAll(s.Config.HintedHandoff.Dir)
+	s.Server.Close()
+}
+
+// URL returns the base URL for the httpd endpoint.
+func (s *Server) URL() string {
+	for _, service := range s.Services {
+		if service, ok := service.(*httpd.Service); ok {
+			return "http://" + service.Addr().String()
+		}
+	}
+	panic("httpd server not found in services")
+}
+
+// CreateDatabaseAndRetentionPolicy will create the database and retention policy.
+func (s *Server) CreateDatabaseAndRetentionPolicy(db string, rp *meta.RetentionPolicyInfo) error {
+	if _, err := s.MetaStore.CreateDatabase(db); err != nil {
+		return err
+	} else if _, err := s.MetaStore.CreateRetentionPolicy(db, rp); err != nil {
+		return err
+	}
+	return nil
+}
+
+// Query executes a query against the server and returns the results.
+func (s *Server) Query(query string) (results string, err error) {
+	return s.QueryWithParams(query, nil)
+}
+
+// Query executes a query against the server and returns the results.
+func (s *Server) QueryWithParams(query string, values url.Values) (results string, err error) {
+	if values == nil {
+		values = url.Values{}
+	}
+	values.Set("q", query)
+	return s.HTTPGet(s.URL() + "/query?" + values.Encode())
+}
+
+// HTTPGet makes an HTTP GET request to the server and returns the response.
+func (s *Server) HTTPGet(url string) (results string, err error) {
+	resp, err := http.Get(url)
+	if err != nil {
+		return "", err
+	}
+	body := string(MustReadAll(resp.Body))
+	switch resp.StatusCode {
+	case http.StatusBadRequest:
+		if !expectPattern(".*error parsing query*.", body) {
+			return "", fmt.Errorf("unexpected status code: code=%d, body=%s", resp.StatusCode, body)
+		}
+		return body, nil
+	case http.StatusOK:
+		return body, nil
+	default:
+		return "", fmt.Errorf("unexpected status code: code=%d, body=%s", resp.StatusCode, body)
+	}
+}
+
+// HTTPPost makes an HTTP POST request to the server and returns the response.
+func (s *Server) HTTPPost(url string, content []byte) (results string, err error) {
+	buf := bytes.NewBuffer(content)
+	resp, err := http.Post(url, "application/json", buf)
+	if err != nil {
+		return "", err
+	}
+	body := string(MustReadAll(resp.Body))
+	switch resp.StatusCode {
+	case http.StatusBadRequest:
+		if !expectPattern(".*error parsing query*.", body) {
+			return "", fmt.Errorf("unexpected status code: code=%d, body=%s", resp.StatusCode, body)
+		}
+		return body, nil
+	case http.StatusOK, http.StatusNoContent:
+		return body, nil
+	default:
+		return "", fmt.Errorf("unexpected status code: code=%d, body=%s", resp.StatusCode, body)
+	}
+}
+
+// Write executes a write against the server and returns the results.
+func (s *Server) Write(db, rp, body string, params url.Values) (results string, err error) {
+	if params == nil {
+		params = url.Values{}
+	}
+	if params.Get("db") == "" {
+		params.Set("db", db)
+	}
+	if params.Get("rp") == "" {
+		params.Set("rp", rp)
+	}
+	resp, err := http.Post(s.URL()+"/write?"+params.Encode(), "", strings.NewReader(body))
+	if err != nil {
+		return "", err
+	} else if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusNoContent {
+		return "", fmt.Errorf("invalid status code: code=%d, body=%s", resp.StatusCode, MustReadAll(resp.Body))
+	}
+	return string(MustReadAll(resp.Body)), nil
+}
+
+// NewConfig returns the default config with temporary paths.
+func NewConfig() *run.Config {
+	c := run.NewConfig()
+	c.ReportingDisabled = true
+	c.Cluster.ShardWriterTimeout = toml.Duration(30 * time.Second)
+	c.Cluster.WriteTimeout = toml.Duration(30 * time.Second)
+	c.Meta.Dir = MustTempFile()
+	c.Meta.BindAddress = "127.0.0.1:0"
+	c.Meta.HeartbeatTimeout = toml.Duration(50 * time.Millisecond)
+	c.Meta.ElectionTimeout = toml.Duration(50 * time.Millisecond)
+	c.Meta.LeaderLeaseTimeout = toml.Duration(50 * time.Millisecond)
+	c.Meta.CommitTimeout = toml.Duration(5 * time.Millisecond)
+
+	c.Data.Dir = MustTempFile()
+	c.Data.WALDir = MustTempFile()
+	c.Data.WALLoggingEnabled = false
+
+	c.HintedHandoff.Dir = MustTempFile()
+
+	c.HTTPD.Enabled = true
+	c.HTTPD.BindAddress = "127.0.0.1:0"
+	c.HTTPD.LogEnabled = testing.Verbose()
+
+	c.Monitor.StoreEnabled = false
+
+	return c
+}
+
+func newRetentionPolicyInfo(name string, rf int, duration time.Duration) *meta.RetentionPolicyInfo {
+	return &meta.RetentionPolicyInfo{Name: name, ReplicaN: rf, Duration: duration}
+}
+
+func maxFloat64() string {
+	maxFloat64, _ := json.Marshal(math.MaxFloat64)
+	return string(maxFloat64)
+}
+
+func maxInt64() string {
+	maxInt64, _ := json.Marshal(^int64(0))
+	return string(maxInt64)
+}
+
+func now() time.Time {
+	return time.Now().UTC()
+}
+
+func yesterday() time.Time {
+	return now().Add(-1 * time.Hour * 24)
+}
+
+func mustParseTime(layout, value string) time.Time {
+	tm, err := time.Parse(layout, value)
+	if err != nil {
+		panic(err)
+	}
+	return tm
+}
+
+// MustReadAll reads r. Panic on error.
+func MustReadAll(r io.Reader) []byte {
+	b, err := ioutil.ReadAll(r)
+	if err != nil {
+		panic(err)
+	}
+	return b
+}
+
+// MustTempFile returns a path to a temporary file.
+func MustTempFile() string {
+	f, err := ioutil.TempFile("", "influxd-")
+	if err != nil {
+		panic(err)
+	}
+	f.Close()
+	os.Remove(f.Name())
+	return f.Name()
+}
+
+func expectPattern(exp, act string) bool {
+	re := regexp.MustCompile(exp)
+	if !re.MatchString(act) {
+		return false
+	}
+	return true
+}
+
+type Query struct {
+	name     string
+	command  string
+	params   url.Values
+	exp, act string
+	pattern  bool
+	skip     bool
+	repeat   int
+}
+
+// Execute runs the command and returns an err if it fails
+func (q *Query) Execute(s *Server) (err error) {
+	if q.params == nil {
+		q.act, err = s.Query(q.command)
+		return
+	}
+	q.act, err = s.QueryWithParams(q.command, q.params)
+	return
+}
+
+func (q *Query) success() bool {
+	if q.pattern {
+		return expectPattern(q.exp, q.act)
+	}
+	return q.exp == q.act
+}
+
+func (q *Query) Error(err error) string {
+	return fmt.Sprintf("%s: %v", q.name, err)
+}
+
+func (q *Query) failureMessage() string {
+	return fmt.Sprintf("%s: unexpected results\nquery:  %s\nexp:    %s\nactual: %s\n", q.name, q.command, q.exp, q.act)
+}
+
+type Test struct {
+	initialized bool
+	write       string
+	params      url.Values
+	db          string
+	rp          string
+	exp         string
+	queries     []*Query
+}
+
+func NewTest(db, rp string) Test {
+	return Test{
+		db: db,
+		rp: rp,
+	}
+}
+
+func (t *Test) addQueries(q ...*Query) {
+	t.queries = append(t.queries, q...)
+}
+
+func (t *Test) init(s *Server) error {
+	if t.write == "" || t.initialized {
+		return nil
+	}
+	t.initialized = true
+	if res, err := s.Write(t.db, t.rp, t.write, t.params); err != nil {
+		return err
+	} else if t.exp != res {
+		return fmt.Errorf("unexpected results\nexp: %s\ngot: %s\n", t.exp, res)
+	}
+	return nil
+}
+
+func configureLogging(s *Server) {
+	// Set the logger to discard unless verbose is on
+	if !testing.Verbose() {
+		type logSetter interface {
+			SetLogger(*log.Logger)
+		}
+		nullLogger := log.New(ioutil.Discard, "", 0)
+		s.MetaStore.Logger = nullLogger
+		s.TSDBStore.Logger = nullLogger
+		s.HintedHandoff.SetLogger(nullLogger)
+		s.Monitor.SetLogger(nullLogger)
+		s.QueryExecutor.SetLogger(nullLogger)
+		for _, service := range s.Services {
+			if service, ok := service.(logSetter); ok {
+				service.SetLogger(nullLogger)
+			}
+		}
+	}
+}

Godeps/_workspace/src/github.com/influxdb/influxdb/cmd/influxd/run/server_test.md

@@ -0,0 +1,150 @@
+# Server Integration Tests
+
+Currently, the file `server_test.go` has integration tests for single node scenarios.
+At some point we'll need to add cluster tests, and may add them in a different file, or 
+rename `server_test.go` to `server_single_node_test.go` or something like that.
+
+## What is in a test?
+
+Each test is broken apart effectively into the following areas:
+
+- Write sample data
+- Use cases for table driven test, that include a command (typically a query) and an expected result.
+
+When each test runs it does the following:
+
+- init: determines if there are any writes and if so, writes them to the in-memory database
+- queries: iterate through each query, executing the command, and comparing the results to the expected result.
+
+## Idempotent - Allows for parallel tests
+
+Each test should be `idempotent`, meaining that its data will not be affected by other tests, or use cases within the table tests themselves.
+This allows for parallel testing, keeping the test suite total execution time very low.
+
+### Basic sample test
+
+```go
+// Ensure the server can have a database with multiple measurements.
+func TestServer_Query_Multiple_Measurements(t *testing.T) {
+	t.Parallel()
+	s := OpenServer(NewConfig(), "")
+	defer s.Close()
+
+	if err := s.CreateDatabaseAndRetentionPolicy("db0", newRetentionPolicyInfo("rp0", 1, 1*time.Hour)); err != nil {
+		t.Fatal(err)
+	}
+
+	// Make sure we do writes for measurements that will span across shards
+	writes := []string{
+		fmt.Sprintf("cpu,host=server01 value=100,core=4 %d", mustParseTime(time.RFC3339Nano, "2000-01-01T00:00:00Z").UnixNano()),
+		fmt.Sprintf("cpu1,host=server02 value=50,core=2 %d", mustParseTime(time.RFC3339Nano, "2015-01-01T00:00:00Z").UnixNano()),
+	}
+	test := NewTest("db0", "rp0")
+	test.write = strings.Join(writes, "\n")
+
+	test.addQueries([]*Query{
+		&Query{
+			name:    "measurement in one shard but not another shouldn't panic server",
+			command: `SELECT host,value  FROM db0.rp0.cpu`,
+			exp:     `{"results":[{"series":[{"name":"cpu","tags":{"host":"server01"},"columns":["time","value"],"values":[["2000-01-01T00:00:00Z",100]]}]}]}`,
+		},
+	}...)
+
+	if err := test.init(s); err != nil {
+		t.Fatalf("test init failed: %s", err)
+	}
+
+	for _, query := range test.queries {
+		if query.skip {
+			t.Logf("SKIP:: %s", query.name)
+			continue
+		}
+		if err := query.Execute(s); err != nil {
+			t.Error(query.Error(err))
+		} else if !query.success() {
+			t.Error(query.failureMessage())
+		}
+	}
+}
+```
+
+Let's break this down:
+
+In this test, we first tell it to run in parallel with the `t.Parallel()` call.
+
+We then open a new server with:
+
+```go
+s := OpenServer(NewConfig(), "")
+defer s.Close()
+```
+
+If needed, we create a database and default retention policy.  This is usually needed
+when inserting and querying data.  This is not needed if you are testing commands like `CREATE DATABASE`, `SHOW DIAGNOSTICS`, etc.
+
+```go
+if err := s.CreateDatabaseAndRetentionPolicy("db0", newRetentionPolicyInfo("rp0", 1, 1*time.Hour)); err != nil {
+	t.Fatal(err)
+}
+```
+
+Next, set up the write data you need:
+
+```go
+writes := []string{
+	fmt.Sprintf("cpu,host=server01 value=100,core=4 %d", mustParseTime(time.RFC3339Nano, "2000-01-01T00:00:00Z").UnixNano()),
+	fmt.Sprintf("cpu1,host=server02 value=50,core=2 %d", mustParseTime(time.RFC3339Nano, "2015-01-01T00:00:00Z").UnixNano()),
+}
+```
+Create a new test with the database and retention policy:
+
+```go
+test := NewTest("db0", "rp0")
+```
+
+Send in the writes:
+```go
+test.write = strings.Join(writes, "\n")
+```
+
+Add some queries (the second one is mocked out to show how to add more than one):
+
+```go
+test.addQueries([]*Query{
+	&Query{
+		name:    "measurement in one shard but not another shouldn't panic server",
+		command: `SELECT host,value  FROM db0.rp0.cpu`,
+		exp:     `{"results":[{"series":[{"name":"cpu","tags":{"host":"server01"},"columns":["time","value"],"values":[["2000-01-01T00:00:00Z",100]]}]}]}`,
+	},
+	&Query{
+		name:    "another test here...",
+		command: `Some query command`,
+		exp:     `the expected results`,
+	},
+}...)
+```
+
+The rest of the code is boilerplate execution code.  It is purposefully not refactored out to a helper
+to make sure the test failure reports the proper lines for debugging purposes.
+
+#### Running the tests
+
+To run the tests:
+
+```sh
+go test ./cmd/influxd/run -parallel 500 -timeout 10s
+```
+
+#### Running a specific test
+
+```sh
+go test ./cmd/influxd/run -parallel 500 -timeout 10s -run TestServer_Query_Fill
+```
+
+#### Verbose feedback
+
+By default, all logs are silenced when testing.  If you pass in the `-v` flag, the test suite becomes verbose, and enables all logging in the system
+
+```sh
+go test ./cmd/influxd/run -parallel 500 -timeout 10s -run TestServer_Query_Fill -v
+```

Godeps/_workspace/src/github.com/influxdb/influxdb/cmd/inspect/main.go

@@ -0,0 +1,136 @@
+package main
+
+import (
+	"encoding/binary"
+	"flag"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+	"text/tabwriter"
+
+	"github.com/influxdb/influxdb/tsdb"
+	_ "github.com/influxdb/influxdb/tsdb/engine"
+)
+
+func main() {
+
+	var path string
+	flag.StringVar(&path, "p", os.Getenv("HOME")+"/.influxdb", "Root storage path. [$HOME/.influxdb]")
+	flag.Parse()
+
+	tstore := tsdb.NewStore(filepath.Join(path, "data"))
+	tstore.Logger = log.New(ioutil.Discard, "", log.LstdFlags)
+	tstore.EngineOptions.Config.Dir = filepath.Join(path, "data")
+	tstore.EngineOptions.Config.WALLoggingEnabled = false
+	tstore.EngineOptions.Config.WALDir = filepath.Join(path, "wal")
+	if err := tstore.Open(); err != nil {
+		fmt.Printf("Failed to open dir: %v\n", err)
+		os.Exit(1)
+	}
+
+	size, err := tstore.DiskSize()
+	if err != nil {
+		fmt.Printf("Failed to determine disk usage: %v\n", err)
+	}
+
+	// Summary stats
+	fmt.Printf("Shards: %d, Indexes: %d, Databases: %d, Disk Size: %d, Series: %d\n",
+		tstore.ShardN(), tstore.DatabaseIndexN(), len(tstore.Databases()), size, countSeries(tstore))
+	fmt.Println()
+
+	tw := tabwriter.NewWriter(os.Stdout, 16, 8, 0, '\t', 0)
+
+	fmt.Fprintln(tw, strings.Join([]string{"Shard", "DB", "Measurement", "Tags [#K/#V]", "Fields [Name:Type]", "Series"}, "\t"))
+
+	shardIDs := tstore.ShardIDs()
+
+	databases := tstore.Databases()
+	sort.Strings(databases)
+
+	for _, db := range databases {
+		index := tstore.DatabaseIndex(db)
+		measurements := index.Measurements()
+		sort.Sort(measurements)
+		for _, m := range measurements {
+			tags := m.TagKeys()
+			tagValues := 0
+			for _, tag := range tags {
+				tagValues += len(m.TagValues(tag))
+			}
+			fields := m.FieldNames()
+			sort.Strings(fields)
+			series := m.SeriesKeys()
+			sort.Strings(series)
+			sort.Sort(ShardIDs(shardIDs))
+
+			// Sample a point from each measurement to determine the field types
+			for _, shardID := range shardIDs {
+				shard := tstore.Shard(shardID)
+				tx, err := shard.ReadOnlyTx()
+				if err != nil {
+					fmt.Printf("Failed to get transaction: %v", err)
+				}
+
+				for _, key := range series {
+					fieldSummary := []string{}
+					cursor := tx.Cursor(key, m.FieldNames(), shard.FieldCodec(m.Name), true)
+
+					// Series doesn't exist in this shard
+					if cursor == nil {
+						continue
+					}
+
+					// Seek to the beginning
+					_, fields := cursor.SeekTo(0)
+					if fields, ok := fields.(map[string]interface{}); ok {
+						for field, value := range fields {
+							fieldSummary = append(fieldSummary, fmt.Sprintf("%s:%T", field, value))
+						}
+						sort.Strings(fieldSummary)
+
+						fmt.Fprintf(tw, "%d\t%s\t%s\t%d/%d\t%d [%s]\t%d\n", shardID, db, m.Name, len(tags), tagValues,
+							len(fields), strings.Join(fieldSummary, ","), len(series))
+					}
+					break
+				}
+				tx.Rollback()
+			}
+		}
+	}
+	tw.Flush()
+}
+
+func countSeries(tstore *tsdb.Store) int {
+	var count int
+	for _, shardID := range tstore.ShardIDs() {
+		shard := tstore.Shard(shardID)
+		cnt, err := shard.SeriesCount()
+		if err != nil {
+			fmt.Printf("series count failed: %v\n", err)
+			continue
+		}
+		count += cnt
+	}
+	return count
+}
+
+func btou64(b []byte) uint64 {
+	return binary.BigEndian.Uint64(b)
+}
+
+// u64tob converts a uint64 into an 8-byte slice.
+func u64tob(v uint64) []byte {
+	b := make([]byte, 8)
+	binary.BigEndian.PutUint64(b, v)
+	return b
+}
+
+type ShardIDs []uint64
+
+func (a ShardIDs) Len() int           { return len(a) }
+func (a ShardIDs) Less(i, j int) bool { return a[i] < a[j] }
+func (a ShardIDs) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }

Godeps/_workspace/src/github.com/influxdb/influxdb/errors.go

@@ -0,0 +1,82 @@
+package influxdb
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"runtime"
+	"strings"
+)
+
+var (
+	// ErrFieldsRequired is returned when a point does not any fields.
+	ErrFieldsRequired = errors.New("fields required")
+
+	// ErrFieldTypeConflict is returned when a new field already exists with a different type.
+	ErrFieldTypeConflict = errors.New("field type conflict")
+)
+
+func ErrDatabaseNotFound(name string) error { return fmt.Errorf("database not found: %s", name) }
+
+func ErrRetentionPolicyNotFound(name string) error {
+	return fmt.Errorf("retention policy not found: %s", name)
+}
+
+func ErrMeasurementNotFound(name string) error { return fmt.Errorf("measurement not found: %s", name) }
+
+func Errorf(format string, a ...interface{}) (err error) {
+	if _, file, line, ok := runtime.Caller(2); ok {
+		a = append(a, file, line)
+		err = fmt.Errorf(format+" (%s:%d)", a...)
+	} else {
+		err = fmt.Errorf(format, a...)
+	}
+	return
+}
+
+// IsClientError indicates whether an error is a known client error.
+func IsClientError(err error) bool {
+	if err == nil {
+		return false
+	}
+
+	if err == ErrFieldsRequired {
+		return true
+	}
+	if err == ErrFieldTypeConflict {
+		return true
+	}
+
+	if strings.Contains(err.Error(), ErrFieldTypeConflict.Error()) {
+		return true
+	}
+
+	return false
+}
+
+// mustMarshal encodes a value to JSON.
+// This will panic if an error occurs. This should only be used internally when
+// an invalid marshal will cause corruption and a panic is appropriate.
+func mustMarshalJSON(v interface{}) []byte {
+	b, err := json.Marshal(v)
+	if err != nil {
+		panic("marshal: " + err.Error())
+	}
+	return b
+}
+
+// mustUnmarshalJSON decodes a value from JSON.
+// This will panic if an error occurs. This should only be used internally when
+// an invalid unmarshal will cause corruption and a panic is appropriate.
+func mustUnmarshalJSON(b []byte, v interface{}) {
+	if err := json.Unmarshal(b, v); err != nil {
+		panic("unmarshal: " + err.Error())
+	}
+}
+
+// assert will panic with a given formatted message if the given condition is false.
+func assert(condition bool, msg string, v ...interface{}) {
+	if !condition {
+		panic(fmt.Sprintf("assert failed: "+msg, v...))
+	}
+}

Godeps/_workspace/src/github.com/influxdb/influxdb/etc/burn-in/.rvmrc

@@ -0,0 +1 @@
+rvm use ruby-2.1.0@burn-in --create

Godeps/_workspace/src/github.com/influxdb/influxdb/etc/burn-in/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+gem "colorize"
+gem "influxdb"

Godeps/_workspace/src/github.com/influxdb/influxdb/etc/burn-in/Gemfile.lock

@@ -0,0 +1,14 @@
+GEM
+  remote: https://rubygems.org/
+  specs:
+    colorize (0.6.0)
+    influxdb (0.0.16)
+      json
+    json (1.8.1)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  colorize
+  influxdb

Godeps/_workspace/src/github.com/influxdb/influxdb/etc/burn-in/burn-in.rb

@@ -0,0 +1,79 @@
+require "influxdb"
+require "colorize"
+require "benchmark"
+
+require_relative "log"
+require_relative "random_gaussian"
+
+BATCH_SIZE = 10_000
+
+Log.info "Starting burn-in suite"
+master = InfluxDB::Client.new
+master.delete_database("burn-in") rescue nil
+master.create_database("burn-in")
+master.create_database_user("burn-in", "user", "pass")
+
+master.database = "burn-in"
+# master.query "select * from test1 into test2;"
+# master.query "select count(value) from test1 group by time(1m) into test2;"
+
+influxdb = InfluxDB::Client.new "burn-in", username: "user", password: "pass"
+
+Log.success "Connected to server #{influxdb.host}:#{influxdb.port}"
+
+Log.log "Creating RandomGaussian(500, 25)"
+gaussian = RandomGaussian.new(500, 25)
+point_count = 0
+
+while true
+  Log.log "Generating 10,000 points.."
+  points = []
+  BATCH_SIZE.times do |n|
+    points << {value: gaussian.rand.to_i.abs}
+  end
+  point_count += points.length
+
+  Log.info "Sending points to server.."
+  begin
+    st = Time.now
+    foo = influxdb.write_point("test1", points)
+    et = Time.now
+    Log.log foo.inspect
+    Log.log "#{et-st} seconds elapsed"
+    Log.success "Write successful."
+  rescue => e
+    Log.failure "Write failed:"
+    Log.log e
+  end
+  sleep 0.5
+
+  Log.info "Checking regular points"
+  st = Time.now
+  response = influxdb.query("select count(value) from test1;")
+  et = Time.now
+
+  Log.log "#{et-st} seconds elapsed"
+
+  response_count = response["test1"].first["count"]
+  if point_count == response_count
+    Log.success "Point counts match: #{point_count} == #{response_count}"
+  else
+    Log.failure "Point counts don't match: #{point_count} != #{response_count}"
+  end
+
+  # Log.info "Checking continuous query points for test2"
+  # st = Time.now
+  # response = influxdb.query("select count(value) from test2;")
+  # et = Time.now
+
+  # Log.log "#{et-st} seconds elapsed"
+
+  # response_count = response["test2"].first["count"]
+  # if point_count == response_count
+    # Log.success "Point counts match: #{point_count} == #{response_count}"
+  # else
+    # Log.failure "Point counts don't match: #{point_count} != #{response_count}"
+  # end
+end
+
+

Godeps/_workspace/src/github.com/influxdb/influxdb/etc/burn-in/log.rb

@@ -0,0 +1,23 @@
+module Log
+  def self.info(msg)
+    print Time.now.strftime("%r") + " | "
+    puts msg.to_s.colorize(:yellow)
+  end
+
+  def self.success(msg)
+    print Time.now.strftime("%r") + " | "
+    puts msg.to_s.colorize(:green)
+  end
+
+  def self.failure(msg)
+    print Time.now.strftime("%r") + " | "
+    puts msg.to_s.colorize(:red)
+  end
+
+  def self.log(msg)
+    print Time.now.strftime("%r") + " | "
+    puts msg.to_s
+  end
+end
+
+

Godeps/_workspace/src/github.com/influxdb/influxdb/etc/burn-in/random_gaussian.rb

@@ -0,0 +1,31 @@
+class RandomGaussian
+  def initialize(mean, stddev, rand_helper = lambda { Kernel.rand })
+    @rand_helper = rand_helper
+    @mean = mean
+    @stddev = stddev
+    @valid = false
+    @next = 0
+  end
+
+  def rand
+    if @valid then
+      @valid = false
+      return @next
+    else
+      @valid = true
+      x, y = self.class.gaussian(@mean, @stddev, @rand_helper)
+      @next = y
+      return x
+    end
+  end
+
+  private
+  def self.gaussian(mean, stddev, rand)
+    theta = 2 * Math::PI * rand.call
+    rho = Math.sqrt(-2 * Math.log(1 - rand.call))
+    scale = stddev * rho
+    x = mean + scale * Math.cos(theta)
+    y = mean + scale * Math.sin(theta)
+    return x, y
+  end
+end

Godeps/_workspace/src/github.com/influxdb/influxdb/etc/burn-in/random_points.rb

@@ -0,0 +1,29 @@
+require "influxdb"
+
+ONE_WEEK_IN_SECONDS = 7*24*60*60
+NUM_POINTS = 10_000
+BATCHES = 100
+
+master = InfluxDB::Client.new
+master.delete_database("ctx") rescue nil
+master.create_database("ctx")
+
+influxdb = InfluxDB::Client.new "ctx"
+influxdb.time_precision = "s"
+
+names = ["foo", "bar", "baz", "quu", "qux"]
+
+st = Time.now
+BATCHES.times do |m|
+  points = []
+
+  puts "Writing #{NUM_POINTS} points, time ##{m}.."
+  NUM_POINTS.times do |n|
+    timestamp = Time.now.to_i - rand(ONE_WEEK_IN_SECONDS)
+    points << {value: names.sample, time: timestamp}
+  end
+
+  influxdb.write_point("ct1", points)
+end
+puts st
+puts Time.now

Godeps/_workspace/src/github.com/influxdb/influxdb/etc/config.sample.toml

@@ -0,0 +1,283 @@
+### Welcome to the InfluxDB configuration file.
+
+# Once every 24 hours InfluxDB will report anonymous data to m.influxdb.com
+# The data includes raft id (random 8 bytes), os, arch, version, and metadata.
+# We don't track ip addresses of servers reporting. This is only used
+# to track the number of instances running and the versions, which
+# is very helpful for us.
+# Change this option to true to disable reporting.
+reporting-disabled = false
+
+###
+### [meta]
+###
+### Controls the parameters for the Raft consensus group that stores metadata
+### about the InfluxDB cluster.
+###
+
+[meta]
+  dir = "/var/opt/influxdb/meta"
+  hostname = "localhost"
+  bind-address = ":8088"
+  retention-autocreate = true
+  election-timeout = "1s"
+  heartbeat-timeout = "1s"
+  leader-lease-timeout = "500ms"
+  commit-timeout = "50ms"
+
+###
+### [data]
+###
+### Controls where the actual shard data for InfluxDB lives and how it is
+### flushed from the WAL. "dir" may need to be changed to a suitable place
+### for your system, but the WAL settings are an advanced configuration. The
+### defaults should work for most systems.
+###
+
+[data]
+  dir = "/var/opt/influxdb/data"
+
+  # Controls the engine type for new shards. Options are b1, bz1, or tsm1.
+  # b1 is the 0.9.2 storage engine, bz1 is the 0.9.3 and 0.9.4 engine.
+  # tsm1 is the 0.9.5 engine
+  # engine ="bz1"
+
+  # The following WAL settings are for the b1 storage engine used in 0.9.2. They won't
+  # apply to any new shards created after upgrading to a version > 0.9.3.
+  max-wal-size = 104857600 # Maximum size the WAL can reach before a flush. Defaults to 100MB.
+  wal-flush-interval = "10m" # Maximum time data can sit in WAL before a flush.
+  wal-partition-flush-delay = "2s" # The delay time between each WAL partition being flushed.
+
+  # These are the WAL settings for the storage engine >= 0.9.3
+  wal-dir = "/var/opt/influxdb/wal"
+  wal-enable-logging = true
+
+  # When a series in the WAL in-memory cache reaches this size in bytes it is marked as ready to
+  # flush to the index
+  # wal-ready-series-size = 25600
+
+  # Flush and compact a partition once this ratio of series are over the ready size
+  # wal-compaction-threshold = 0.6
+
+  # Force a flush and compaction if any series in a partition gets above this size in bytes
+  # wal-max-series-size = 2097152
+
+  # Force a flush of all series and full compaction if there have been no writes in this
+  # amount of time. This is useful for ensuring that shards that are cold for writes don't
+  # keep a bunch of data cached in memory and in the WAL.
+  # wal-flush-cold-interval = "10m"
+
+  # Force a partition to flush its largest series if it reaches this approximate size in
+  # bytes. Remember there are 5 partitions so you'll need at least 5x this amount of memory.
+  # The more memory you have, the bigger this can be.
+  # wal-partition-size-threshold = 20971520
+
+  # Whether queries should be logged before execution. Very useful for troubleshooting, but will
+  # log any sensitive data contained within a query.
+  # query-log-enabled = true
+
+###
+### [cluster]
+###
+### Controls non-Raft cluster behavior, which generally includes how data is
+### shared across shards.
+###
+
+[cluster]
+  shard-writer-timeout = "10s" # The time within which a shard must respond to write.
+  write-timeout = "5s" # The time within which a write operation must complete on the cluster.
+
+###
+### [retention]
+###
+### Controls the enforcement of retention policies for evicting old data.
+###
+
+[retention]
+  enabled = true
+  check-interval = "30m"
+
+###
+### Controls the system self-monitoring, statistics and diagnostics.
+###
+### The internal database for monitoring data is created automatically if
+### if it does not already exist. The target retention within this database
+### is called 'monitor' and is also created with a retention period of 7 days
+### and a replication factor of 1, if it does not exist. In all cases the
+### this retention policy is configured as the default for the database.
+
+[monitor]
+  store-enabled = true # Whether to record statistics internally.
+  store-database = "_internal" # The destination database for recorded statistics
+  store-interval = "10s" # The interval at which to record statistics
+
+###
+### [admin]
+###
+### Controls the availability of the built-in, web-based admin interface. If HTTPS is
+### enabled for the admin interface, HTTPS must also be enabled on the [http] service.
+###
+
+[admin]
+  enabled = true
+  bind-address = ":8083"
+  https-enabled = false
+  https-certificate = "/etc/ssl/influxdb.pem"
+
+###
+### [http]
+###
+### Controls how the HTTP endpoints are configured. These are the primary
+### mechanism for getting data into and out of InfluxDB.
+###
+
+[http]
+  enabled = true
+  bind-address = ":8086"
+  auth-enabled = false
+  log-enabled = true
+  write-tracing = false
+  pprof-enabled = false
+  https-enabled = false
+  https-certificate = "/etc/ssl/influxdb.pem"
+
+###
+### [[graphite]]
+###
+### Controls one or many listeners for Graphite data.
+###
+
+[[graphite]]
+  enabled = false
+  # database = "graphite"
+  # bind-address = ":2003"
+  # protocol = "tcp"
+  # consistency-level = "one"
+  # name-separator = "."
+
+  # These next lines control how batching works. You should have this enabled
+  # otherwise you could get dropped metrics or poor performance. Batching
+  # will buffer points in memory if you have many coming in.
+
+  # batch-size = 1000 # will flush if this many points get buffered
+  # batch-pending = 5 # number of batches that may be pending in memory
+  # batch-timeout = "1s" # will flush at least this often even if we haven't hit buffer limit
+
+  ## "name-schema" configures tag names for parsing the metric name from graphite protocol;
+  ## separated by `name-separator`.
+  ## The "measurement" tag is special and the corresponding field will become
+  ## the name of the metric.
+  ## e.g. "type.host.measurement.device" will parse "server.localhost.cpu.cpu0" as
+  ## {
+  ##     measurement: "cpu",
+  ##     tags: {
+  ##         "type": "server",
+  ##         "host": "localhost,
+  ##         "device": "cpu0"
+  ##     }
+  ## }
+  # name-schema = "type.host.measurement.device"
+
+  ## If set to true, when the input metric name has more fields than `name-schema` specified,
+  ## the extra fields will be ignored.
+  ## Otherwise an error will be logged and the metric rejected.
+  # ignore-unnamed = true
+
+###
+### [collectd]
+###
+### Controls the listener for collectd data.
+###
+
+[collectd]
+  enabled = false
+  # bind-address = ""
+  # database = ""
+  # typesdb = ""
+
+  # These next lines control how batching works. You should have this enabled
+  # otherwise you could get dropped metrics or poor performance. Batching
+  # will buffer points in memory if you have many coming in.
+
+  # batch-size = 1000 # will flush if this many points get buffered
+  # batch-pending = 5 # number of batches that may be pending in memory
+  # batch-timeout = "1s" # will flush at least this often even if we haven't hit buffer limit
+
+###
+### [opentsdb]
+###
+### Controls the listener for OpenTSDB data.
+###
+
+[opentsdb]
+  enabled = false
+  # bind-address = ":4242"
+  # database = "opentsdb"
+  # retention-policy = ""
+  # consistency-level = "one"
+  # tls-enabled = false
+  # certificate= ""
+
+  # These next lines control how batching works. You should have this enabled
+  # otherwise you could get dropped metrics or poor performance. Only points
+  # metrics received over the telnet protocol undergo batching.
+
+  # batch-size = 1000 # will flush if this many points get buffered
+  # batch-pending = 5 # number of batches that may be pending in memory
+  # batch-timeout = "1s" # will flush at least this often even if we haven't hit buffer limit
+
+###
+### [[udp]]
+###
+### Controls the listeners for InfluxDB line protocol data via UDP.
+###
+
+[[udp]]
+  enabled = false
+  # bind-address = ""
+  # database = ""
+  # retention-policy = ""
+
+  # These next lines control how batching works. You should have this enabled
+  # otherwise you could get dropped metrics or poor performance. Batching
+  # will buffer points in memory if you have many coming in.
+
+  # batch-size = 1000 # will flush if this many points get buffered
+  # batch-pending = 5 # number of batches that may be pending in memory
+  # batch-timeout = "1s" # will flush at least this often even if we haven't hit buffer limit
+
+###
+### [continuous_queries]
+###
+### Controls how continuous queries are run within InfluxDB.
+###
+
+[continuous_queries]
+  log-enabled = true
+  enabled = true
+  recompute-previous-n = 2
+  recompute-no-older-than = "10m"
+  compute-runs-per-interval = 10
+  compute-no-more-than = "2m"
+
+###
+### [hinted-handoff]
+###
+### Controls the hinted handoff feature, which allows nodes to temporarily
+### store queued data when one node of a cluster is down for a short period
+### of time.
+###
+
+[hinted-handoff]
+  enabled = true
+  dir = "/var/opt/influxdb/hh"
+  max-size = 1073741824
+  max-age = "168h"
+  retry-rate-limit = 0
+
+  # Hinted handoff will start retrying writes to down nodes at a rate of once per second.
+  # If any error occurs, it will backoff in an exponential manner, until the interval
+  # reaches retry-max-interval. Once writes to all nodes are successfully completed the
+  # interval will reset to retry-interval.
+  retry-interval = "1s"
+  retry-max-interval = "1m"

Godeps/_workspace/src/github.com/influxdb/influxdb/importer/README.md

@@ -0,0 +1,193 @@
+# Import/Export
+
+## Exporting from 0.8.9
+
+Version `0.8.9` of InfluxDB adds support to export your data to a format that can be imported into `0.9.3` and later.
+
+Note that `0.8.9` can be found here:
+
+```
+http://get.influxdb.org.s3.amazonaws.com/influxdb_0.8.9_amd64.deb
+http://get.influxdb.org.s3.amazonaws.com/influxdb-0.8.9-1.x86_64.rpm
+```
+
+### Design
+
+`0.8.9` exports raw data to a flat file that includes two sections, `DDL` and `DML`.  You can choose to export them independently (see below).
+
+The `DDL` section contains the sql commands to create databases and retention policies.  the `DML` section is [line protocol](https://github.com/influxdb/influxdb/blob/master/tsdb/README.md) and can be directly posted to the [http endpoint](https://influxdb.com/docs/v0.9/guides/writing_data.html) in `0.9`.  Remember that batching is important and we don't recommend batch sizes over 5k.
+
+You need to specify a database and shard group when you export.
+
+To list out your shards, use the following http endpoint:
+
+`/cluster/shard_spaces`
+
+example:
+```sh
+http://username:password@localhost:8086/cluster/shard_spaces
+```
+
+Then, to export a database with then name "metrics" and a shard space with the name "default", issue the following curl command:
+
+```sh
+curl -o export http://username:password@http://localhost:8086/export/metrics/default
+```
+
+Compression is supported, and will result in a significantly smaller file size.
+
+Use the following command for compression:
+```sh
+curl -o export.gz --compressed http://username:password@http://localhost:8086/export/metrics/default
+```
+
+You can also export just the `DDL` with this option:
+
+```sh
+curl -o export.ddl http://username:password@http://localhost:8086/export/metrics/default?l=ddl
+```
+
+Or just the `DML` with this option:
+
+```sh
+curl -o export.dml.gz --compressed http://username:password@http://localhost:8086/export/metrics/default?l=dml
+```
+
+### Assumptions
+
+- Series name mapping follows these [guidelines](https://influxdb.com/docs/v0.8/advanced_topics/schema_design.html)
+- Database name will map directly from `0.8` to `0.9`
+- Shard Spaces map to Retention Policies
+- Shard Space Duration is ignored, as in `0.9` we determine shard size automatically
+- Regex is used to match the correct series names and only exports that data for the database
+- Duration becomes the new Retention Policy duration
+
+- Users are not migrated due to inability to get passwords.  Anyone using users will need to manually set these back up in `0.9`
+
+### Upgrade Recommendations
+
+It's recommended that you upgrade to `0.9.3` first and have all your writes going there.  Then, on the `0.8.X` instances, upgrade to `0.8.9`.
+
+It is important that when exporting you change your config to allow for the http endpoints not timing out.  To do so, make this change in your config:
+
+```toml
+# Configure the http api
+[api]
+read-timeout = "0s"
+```
+
+### Exceptions
+
+If a series can't be exported to tags based on the guidelines mentioned above,
+we will insert the entire series name as the measurement name.  You can either 
+allow that to import into the new InfluxDB instance, or you can do your own 
+data massage on it prior to importing it.
+
+For example, if you have the following series name:
+
+```
+metric.disk.c.host.server01.single
+```
+
+It will export as exactly thta as the measurement name and no tags:
+
+```
+metric.disk.c.host.server01.single
+```
+
+### Export Metrics
+
+When you export, you will now get comments inline in the `DML`:
+
+`# Found 999 Series for export`
+
+As well as count totals for each series exported:
+
+`# Series FOO - Points Exported: 999`
+
+With a total at the bottom:
+
+`# Points Exported: 999`
+
+You can grep the file that was exported at the end to get all the export metrics:
+
+`cat myexport | grep Exported`
+
+## Importing
+
+Version `0.9.3` of InfluxDB adds support to import your data from version `0.8.9`.
+
+## Caveats
+
+For the export/import to work, all requisites have to be met.  For export, all series names in `0.8` should be in the following format:
+
+```
+<tagName>.<tagValue>.<tagName>.<tagValue>.<measurement>
+```
+for example:
+```
+az.us-west-1.host.serverA.cpu
+```
+or any number of tags 
+```
+building.2.temperature
+```
+
+Additionally, the fields need to have a consistent type (all float64, int64, etc) for every write in `0.8`.  Otherwise they have the potential to fail writes in the import.
+See below for more information.
+
+## Running the import command
+ 
+ To import via the cli, you can specify the following command:
+
+ ```sh
+ influx -import -path=metrics-default.gz -compressed
+ ```
+
+ If the file is not compressed you can issue it without the `-compressed` flag:
+
+ ```sh
+ influx -import -path=metrics-default
+ ```
+
+ To redirect failed import lines to another file, run this command:
+
+ ```sh
+ influx -import -path=metrics-default.gz -compressed > failures
+ ```
+
+ The import will use the line protocol in batches of 5,000 lines per batch when sending data to the server.
+ 
+### Throttiling the import
+ 
+ If you need to throttle the import so the database has time to ingest, you can use the `-pps` flag.  This will limit the points per second that will be sent to the server.
+ 
+  ```sh
+ influx -import -path=metrics-default.gz -compressed -pps 50000 > failures
+ ```
+ 
+ Which is stating that you don't want MORE than 50,000 points per second to write to the database. Due to the processing that is taking place however, you will likely never get exactly 50,000 pps, more like 35,000 pps, etc. 
+
+## Understanding the results of the import
+
+During the import, a status message will write out for every 100,000 points imported and report stats on the progress of the import:
+
+```
+2015/08/21 14:48:01 Processed 3100000 lines.  Time elapsed: 56.740578415s.  Points per second (PPS): 54634
+```
+
+ The batch will give some basic stats when finished:
+
+ ```sh
+ 2015/07/29 23:15:20 Processed 2 commands
+ 2015/07/29 23:15:20 Processed 70207923 inserts
+ 2015/07/29 23:15:20 Failed 29785000 inserts
+ ```
+
+ Most inserts fail due to the following types of error:
+
+ ```sh
+ 2015/07/29 22:18:28 error writing batch:  write failed: field type conflict: input field "value" on measurement "metric" is type float64, already exists as type integer
+ ```
+
+ This is due to the fact that in `0.8` a field could get created and saved as int or float types for independent writes.  In `0.9` the field has to have a consistent type.

Godeps/_workspace/src/github.com/influxdb/influxdb/importer/v8/importer.go

@@ -0,0 +1,236 @@
+package v8
+
+import (
+	"bufio"
+	"compress/gzip"
+	"fmt"
+	"io"
+	"log"
+	"net/url"
+	"os"
+	"strings"
+	"time"
+
+	"github.com/influxdb/influxdb/client"
+)
+
+const batchSize = 5000
+
+// Config is the config used to initialize a Importer importer
+type Config struct {
+	Username         string
+	Password         string
+	URL              url.URL
+	Precision        string
+	WriteConsistency string
+	Path             string
+	Version          string
+	Compressed       bool
+	PPS              int
+}
+
+// NewConfig returns an initialized *Config
+func NewConfig() *Config {
+	return &Config{}
+}
+
+// Importer is the importer used for importing 0.8 data
+type Importer struct {
+	client                *client.Client
+	database              string
+	retentionPolicy       string
+	config                *Config
+	batch                 []string
+	totalInserts          int
+	failedInserts         int
+	totalCommands         int
+	throttlePointsWritten int
+	lastWrite             time.Time
+	throttle              *time.Ticker
+}
+
+// NewImporter will return an intialized Importer struct
+func NewImporter(config *Config) *Importer {
+	return &Importer{
+		config: config,
+		batch:  make([]string, 0, batchSize),
+	}
+}
+
+// Import processes the specified file in the Config and writes the data to the databases in chunks specified by batchSize
+func (i *Importer) Import() error {
+	// Create a client and try to connect
+	config := client.NewConfig()
+	config.URL = i.config.URL
+	config.Username = i.config.Username
+	config.Password = i.config.Password
+	config.UserAgent = fmt.Sprintf("influxDB importer/%s", i.config.Version)
+	cl, err := client.NewClient(config)
+	if err != nil {
+		return fmt.Errorf("could not create client %s", err)
+	}
+	i.client = cl
+	if _, _, e := i.client.Ping(); e != nil {
+		return fmt.Errorf("failed to connect to %s\n", i.client.Addr())
+	}
+
+	// Validate args
+	if i.config.Path == "" {
+		return fmt.Errorf("file argument required")
+	}
+
+	defer func() {
+		if i.totalInserts > 0 {
+			log.Printf("Processed %d commands\n", i.totalCommands)
+			log.Printf("Processed %d inserts\n", i.totalInserts)
+			log.Printf("Failed %d inserts\n", i.failedInserts)
+		}
+	}()
+
+	// Open the file
+	f, err := os.Open(i.config.Path)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	var r io.Reader
+
+	// If gzipped, wrap in a gzip reader
+	if i.config.Compressed {
+		gr, err := gzip.NewReader(f)
+		if err != nil {
+			return err
+		}
+		defer gr.Close()
+		// Set the reader to the gzip reader
+		r = gr
+	} else {
+		// Standard text file so our reader can just be the file
+		r = f
+	}
+
+	// Get our reader
+	scanner := bufio.NewScanner(r)
+
+	// Process the DDL
+	i.processDDL(scanner)
+
+	// Set up our throttle channel.  Since there is effectively no other activity at this point
+	// the smaller resolution gets us much closer to the requested PPS
+	i.throttle = time.NewTicker(time.Microsecond)
+	defer i.throttle.Stop()
+
+	// Prime the last write
+	i.lastWrite = time.Now()
+
+	// Process the DML
+	i.processDML(scanner)
+
+	// Check if we had any errors scanning the file
+	if err := scanner.Err(); err != nil {
+		return fmt.Errorf("reading standard input: %s", err)
+	}
+
+	return nil
+}
+
+func (i *Importer) processDDL(scanner *bufio.Scanner) {
+	for scanner.Scan() {
+		line := scanner.Text()
+		// If we find the DML token, we are done with DDL
+		if strings.HasPrefix(line, "# DML") {
+			return
+		}
+		if strings.HasPrefix(line, "#") {
+			continue
+		}
+		i.queryExecutor(line)
+	}
+}
+
+func (i *Importer) processDML(scanner *bufio.Scanner) {
+	start := time.Now()
+	for scanner.Scan() {
+		line := scanner.Text()
+		if strings.HasPrefix(line, "# CONTEXT-DATABASE:") {
+			i.database = strings.TrimSpace(strings.Split(line, ":")[1])
+		}
+		if strings.HasPrefix(line, "# CONTEXT-RETENTION-POLICY:") {
+			i.retentionPolicy = strings.TrimSpace(strings.Split(line, ":")[1])
+		}
+		if strings.HasPrefix(line, "#") {
+			continue
+		}
+		i.batchAccumulator(line, start)
+	}
+}
+
+func (i *Importer) execute(command string) {
+	response, err := i.client.Query(client.Query{Command: command, Database: i.database})
+	if err != nil {
+		log.Printf("error: %s\n", err)
+		return
+	}
+	if err := response.Error(); err != nil {
+		log.Printf("error: %s\n", response.Error())
+	}
+}
+
+func (i *Importer) queryExecutor(command string) {
+	i.totalCommands++
+	i.execute(command)
+}
+
+func (i *Importer) batchAccumulator(line string, start time.Time) {
+	i.batch = append(i.batch, line)
+	if len(i.batch) == batchSize {
+		if e := i.batchWrite(); e != nil {
+			log.Println("error writing batch: ", e)
+			// Output failed lines to STDOUT so users can capture lines that failed to import
+			fmt.Println(strings.Join(i.batch, "\n"))
+			i.failedInserts += len(i.batch)
+		} else {
+			i.totalInserts += len(i.batch)
+		}
+		i.batch = i.batch[:0]
+		// Give some status feedback every 100000 lines processed
+		processed := i.totalInserts + i.failedInserts
+		if processed%100000 == 0 {
+			since := time.Since(start)
+			pps := float64(processed) / since.Seconds()
+			log.Printf("Processed %d lines.  Time elapsed: %s.  Points per second (PPS): %d", processed, since.String(), int64(pps))
+		}
+	}
+}
+
+func (i *Importer) batchWrite() error {
+	// Accumulate the batch size to see how many points we have written this second
+	i.throttlePointsWritten += len(i.batch)
+
+	// Find out when we last wrote data
+	since := time.Since(i.lastWrite)
+
+	// Check to see if we've exceeded our points per second for the current timeframe
+	var currentPPS int
+	if since.Seconds() > 0 {
+		currentPPS = int(float64(i.throttlePointsWritten) / since.Seconds())
+	} else {
+		currentPPS = i.throttlePointsWritten
+	}
+
+	// If our currentPPS is greater than the PPS specified, then we wait and retry
+	if int(currentPPS) > i.config.PPS && i.config.PPS != 0 {
+		// Wait for the next tick
+		<-i.throttle.C
+
+		// Decrement the batch size back out as it is going to get called again
+		i.throttlePointsWritten -= len(i.batch)
+		return i.batchWrite()
+	}
+
+	_, e := i.client.WriteLineProtocol(strings.Join(i.batch, "\n"), i.database, i.retentionPolicy, i.config.Precision, i.config.WriteConsistency)
+	i.throttlePointsWritten = 0
+	i.lastWrite = time.Now()
+	return e
+}

Godeps/_workspace/src/github.com/influxdb/influxdb/influxql/INFLUXQL.md

@@ -87,11 +87,11 @@ CREATE       CONTINUOUS   DATABASE     DATABASES    DEFAULT      DELETE
 DESC         DROP         DURATION     END          EXISTS       EXPLAIN
 FIELD        FROM         GRANT        GROUP        IF           IN
 INNER        INSERT       INTO         KEY          KEYS         LIMIT
-SHOW         MEASUREMENT  MEASUREMENTS OFFSET       ON           ORDER
+SHOW         MEASUREMENT  MEASUREMENTS NOT          OFFSET       ON
-PASSWORD     POLICY       POLICIES     PRIVILEGES   QUERIES      QUERY
+ORDER        PASSWORD     POLICY       POLICIES     PRIVILEGES   QUERIES
-READ         REPLICATION  RETENTION    REVOKE       SELECT       SERIES
+QUERY        READ         REPLICATION  RETENTION    REVOKE       SELECT
-SLIMIT       SOFFSET      TAG          TO           USER         USERS
+SERIES       SLIMIT       SOFFSET      TAG          TO           USER
-VALUES       WHERE        WITH         WRITE
+USERS        VALUES       WHERE        WITH         WRITE
 ```
 
 ## Literals
@@ -124,9 +124,7 @@ string_lit          = `'` { unicode_char } `'`' .
 
 Duration literals specify a length of time.  An integer literal followed immediately (with no spaces) by a duration unit listed below is interpreted as a duration literal.
 
-```
+### Duration units
-Duration unit definitions
--------------------------
 | Units  | Meaning                                 |
 |--------|-----------------------------------------|
 | u or µ | microseconds (1 millionth of a second)  |
@@ -136,7 +134,6 @@ Duration unit definitions
 | h      | hour                                    |
 | d      | day                                     |
 | w      | week                                    |
-```
 
 ```
 duration_lit        = int_lit duration_unit .
@@ -191,6 +188,7 @@ statement           = alter_retention_policy_stmt |
                       show_measurements_stmt |
                       show_retention_policies |
                       show_series_stmt |
+                      show_shards_stmt |
                       show_tag_keys_stmt |
                       show_tag_values_stmt |
                       show_users_stmt |
@@ -455,7 +453,7 @@ SHOW FIELD KEYS FROM cpu;
 
 ### SHOW MEASUREMENTS
 
-show_measurements_stmt = [ where_clause ] [ group_by_clause ] [ limit_clause ]
+show_measurements_stmt = "SHOW MEASUREMENTS" [ where_clause ] [ group_by_clause ] [ limit_clause ]
                          [ offset_clause ] .
 
 ```sql
@@ -482,7 +480,7 @@ SHOW RETENTION POLICIES ON mydb;
 ### SHOW SERIES
 
 ```
-show_series_stmt = [ from_clause ] [ where_clause ] [ group_by_clause ]
+show_series_stmt = "SHOW SERIES" [ from_clause ] [ where_clause ] [ group_by_clause ]
                    [ limit_clause ] [ offset_clause ] .
 ```
 
@@ -492,10 +490,22 @@ show_series_stmt = [ from_clause ] [ where_clause ] [ group_by_clause ]
 
 ```
 
+### SHOW SHARDS
+
+```
+show_shards_stmt = "SHOW SHARDS" .
+```
+
+#### Example:
+
+```sql
+SHOW SHARDS;
+```
+
 ### SHOW TAG KEYS
 
 ```
-show_tag_keys_stmt = [ from_clause ] [ where_clause ] [ group_by_clause ]
+show_tag_keys_stmt = "SHOW TAG KEYS" [ from_clause ] [ where_clause ] [ group_by_clause ]
                      [ limit_clause ] [ offset_clause ] .
 ```
 
@@ -518,7 +528,7 @@ SHOW TAG KEYS WHERE host = 'serverA';
 ### SHOW TAG VALUES
 
 ```
-show_tag_values_stmt = [ from_clause ] with_tag_clause [ where_clause ]
+show_tag_values_stmt = "SHOW TAG VALUES" [ from_clause ] with_tag_clause [ where_clause ]
                        [ group_by_clause ] [ limit_clause ] [ offset_clause ] .
 ```
 
@@ -551,7 +561,7 @@ SHOW USERS;
 ### REVOKE
 
 ```
-revoke_stmt = privilege [ "ON" db_name ] "FROM" user_name
+revoke_stmt = "REVOKE" privilege [ "ON" db_name ] "FROM" user_name
 ```
 
 #### Examples:
@@ -567,7 +577,7 @@ REVOKE READ ON mydb FROM jdoe;
 ### SELECT
 
 ```
-select_stmt = fields from_clause [ into_clause ] [ where_clause ]
+select_stmt = "SELECT" fields from_clause [ into_clause ] [ where_clause ]
               [ group_by_clause ] [ order_by_clause ] [ limit_clause ]
               [ offset_clause ] [ slimit_clause ] [ soffset_clause ].
 ```

Godeps/_workspace/src/github.com/influxdb/influxdb/influxql/ast.go

@@ -9,6 +9,8 @@ import (
 	"strconv"
 	"strings"
 	"time"
+
+	"github.com/influxdb/influxdb/pkg/slices"
 )
 
 // DataType represents the primitive data types available in InfluxQL.
@@ -90,6 +92,7 @@ func (*DropDatabaseStatement) node()          {}
 func (*DropMeasurementStatement) node()       {}
 func (*DropRetentionPolicyStatement) node()   {}
 func (*DropSeriesStatement) node()            {}
+func (*DropServerStatement) node()            {}
 func (*DropUserStatement) node()              {}
 func (*GrantStatement) node()                 {}
 func (*GrantAdminStatement) node()            {}
@@ -105,6 +108,7 @@ func (*ShowFieldKeysStatement) node()         {}
 func (*ShowRetentionPoliciesStatement) node() {}
 func (*ShowMeasurementsStatement) node()      {}
 func (*ShowSeriesStatement) node()            {}
+func (*ShowShardsStatement) node()            {}
 func (*ShowStatsStatement) node()             {}
 func (*ShowDiagnosticsStatement) node()       {}
 func (*ShowTagKeysStatement) node()           {}
@@ -195,6 +199,7 @@ func (*DropDatabaseStatement) stmt()          {}
 func (*DropMeasurementStatement) stmt()       {}
 func (*DropRetentionPolicyStatement) stmt()   {}
 func (*DropSeriesStatement) stmt()            {}
+func (*DropServerStatement) stmt()            {}
 func (*DropUserStatement) stmt()              {}
 func (*GrantStatement) stmt()                 {}
 func (*GrantAdminStatement) stmt()            {}
@@ -206,6 +211,7 @@ func (*ShowFieldKeysStatement) stmt()         {}
 func (*ShowMeasurementsStatement) stmt()      {}
 func (*ShowRetentionPoliciesStatement) stmt() {}
 func (*ShowSeriesStatement) stmt()            {}
+func (*ShowShardsStatement) stmt()            {}
 func (*ShowStatsStatement) stmt()             {}
 func (*ShowDiagnosticsStatement) stmt()       {}
 func (*ShowTagKeysStatement) stmt()           {}
@@ -274,7 +280,7 @@ type SortField struct {
 // String returns a string representation of a sort field
 func (field *SortField) String() string {
 	var buf bytes.Buffer
-	if field.Name == "" {
+	if field.Name != "" {
 		_, _ = buf.WriteString(field.Name)
 		_, _ = buf.WriteString(" ")
 	}
@@ -302,12 +308,19 @@ func (a SortFields) String() string {
 type CreateDatabaseStatement struct {
 	// Name of the database to be created.
 	Name string
+
+	// IfNotExists indicates whether to return without error if the database
+	// already exists.
+	IfNotExists bool
 }
 
 // String returns a string representation of the create database statement.
 func (s *CreateDatabaseStatement) String() string {
 	var buf bytes.Buffer
 	_, _ = buf.WriteString("CREATE DATABASE ")
+	if s.IfNotExists {
+		_, _ = buf.WriteString("IF NOT EXISTS ")
+	}
 	_, _ = buf.WriteString(s.Name)
 	return buf.String()
 }
@@ -707,6 +720,18 @@ type SelectStatement struct {
 	FillValue interface{}
 }
 
+// SourceNames returns a list of source names.
+func (s *SelectStatement) SourceNames() []string {
+	a := make([]string, 0, len(s.Sources))
+	for _, src := range s.Sources {
+		switch src := src.(type) {
+		case *Measurement:
+			a = append(a, src.Name)
+		}
+	}
+	return a
+}
+
 // HasDerivative returns true if one of the function calls in the statement is a
 // derivative aggregate
 func (s *SelectStatement) HasDerivative() bool {
@@ -732,6 +757,11 @@ func (s *SelectStatement) IsSimpleDerivative() bool {
 	return false
 }
 
+// TimeAscending returns true if the time field is sorted in chronological order.
+func (s *SelectStatement) TimeAscending() bool {
+	return len(s.SortFields) == 0 || s.SortFields[0].Ascending
+}
+
 // Clone returns a deep copy of the statement.
 func (s *SelectStatement) Clone() *SelectStatement {
 	clone := &SelectStatement{
@@ -848,6 +878,48 @@ func (s *SelectStatement) RewriteDistinct() {
 	}
 }
 
+// ColumnNames will walk all fields and functions and return the appropriate field names for the select statement
+// while maintaining order of the field names
+func (s *SelectStatement) ColumnNames() []string {
+	// Always set the first column to be time, even if they didn't specify it
+	columnNames := []string{"time"}
+
+	// First walk each field
+	for _, field := range s.Fields {
+		switch f := field.Expr.(type) {
+		case *Call:
+			if f.Name == "top" || f.Name == "bottom" {
+				if len(f.Args) == 2 {
+					columnNames = append(columnNames, f.Name)
+					continue
+				}
+				// We have a special case now where we have to add the column names for the fields TOP or BOTTOM asked for as well
+				columnNames = slices.Union(columnNames, f.Fields(), true)
+				continue
+			}
+			columnNames = append(columnNames, field.Name())
+		default:
+			// time is always first, and we already added it, so ignore it if they asked for it anywhere else.
+			if field.Name() != "time" {
+				columnNames = append(columnNames, field.Name())
+			}
+		}
+	}
+
+	return columnNames
+}
+
+// HasTimeFieldSpecified will walk all fields and determine if the user explicitly asked for time
+// This is needed to determine re-write behaviors for functions like TOP and BOTTOM
+func (s *SelectStatement) HasTimeFieldSpecified() bool {
+	for _, f := range s.Fields {
+		if f.Name() == "time" {
+			return true
+		}
+	}
+	return false
+}
+
 // String returns a string representation of the select statement.
 func (s *SelectStatement) String() string {
 	var buf bytes.Buffer
@@ -989,6 +1061,10 @@ func (s *SelectStatement) validate(tr targetRequirement) error {
 		return err
 	}
 
+	if err := s.validateDimensions(); err != nil {
+		return err
+	}
+
 	if err := s.validateDistinct(); err != nil {
 		return err
 	}
@@ -1005,10 +1081,6 @@ func (s *SelectStatement) validate(tr targetRequirement) error {
 		return err
 	}
 
-	if err := s.validateWildcard(); err != nil {
-		return err
-	}
-
 	return nil
 }
 
@@ -1020,40 +1092,155 @@ func (s *SelectStatement) validateFields() error {
 	return nil
 }
 
-func (s *SelectStatement) validateAggregates(tr targetRequirement) error {
+func (s *SelectStatement) validateDimensions() error {
-	// First, if 1 field is an aggregate, then all fields must be an aggregate. This is
+	var dur time.Duration
-	// a explicit limitation of the current system.
+	for _, dim := range s.Dimensions {
+		switch expr := dim.Expr.(type) {
+		case *Call:
+			// Ensure the call is time() and it only has one duration argument.
+			// If we already have a duration
+			if expr.Name != "time" {
+				return errors.New("only time() calls allowed in dimensions")
+			} else if len(expr.Args) != 1 {
+				return errors.New("time dimension expected one argument")
+			} else if lit, ok := expr.Args[0].(*DurationLiteral); !ok {
+				return errors.New("time dimension must have one duration argument")
+			} else if dur != 0 {
+				return errors.New("multiple time dimensions not allowed")
+			} else {
+				dur = lit.Val
+			}
+		case *VarRef:
+			if strings.ToLower(expr.Val) == "time" {
+				return errors.New("time() is a function and expects at least one argument")
+			}
+		case *Wildcard:
+		default:
+			return errors.New("only time and tag dimensions allowed")
+		}
+	}
+	return nil
+}
+
+// validSelectWithAggregate determines if a SELECT statement has the correct
+// combination of aggregate functions combined with selected fields and tags
+// Currently we don't have support for all aggregates, but aggregates that
+// can be combined with fields/tags are:
+//  TOP, BOTTOM, MAX, MIN, FIRST, LAST
+func (s *SelectStatement) validSelectWithAggregate() error {
+	calls := map[string]struct{}{}
 	numAggregates := 0
 	for _, f := range s.Fields {
-		if _, ok := f.Expr.(*Call); ok {
+		if c, ok := f.Expr.(*Call); ok {
+			calls[c.Name] = struct{}{}
 			numAggregates++
 		}
 	}
+	// For TOP, BOTTOM, MAX, MIN, FIRST, LAST (selector functions) it is ok to ask for fields and tags
+	// but only if one function is specified.  Combining multiple functions and fields and tags is not currently supported
+	onlySelectors := true
+	for k := range calls {
+		switch k {
+		case "top", "bottom", "max", "min", "first", "last":
+		default:
+			onlySelectors = false
+			break
+		}
+	}
+	if onlySelectors {
+		// If they only have one selector, they can have as many fields or tags as they want
+		if numAggregates == 1 {
+			return nil
+		}
+		// If they have multiple selectors, they are not allowed to have any other fields or tags specified
+		if numAggregates > 1 && len(s.Fields) != numAggregates {
+			return fmt.Errorf("mixing multiple selector functions with tags or fields is not supported")
+		}
+	}
+
 	if numAggregates != 0 && numAggregates != len(s.Fields) {
 		return fmt.Errorf("mixing aggregate and non-aggregate queries is not supported")
 	}
+	return nil
+}
 
-	// Secondly, determine if specific calls have at least one and only one argument
+func (s *SelectStatement) validateAggregates(tr targetRequirement) error {
 	for _, f := range s.Fields {
-		if c, ok := f.Expr.(*Call); ok {
+		switch expr := f.Expr.(type) {
-			switch c.Name {
+		case *Call:
+			switch expr.Name {
 			case "derivative", "non_negative_derivative":
-				if min, max, got := 1, 2, len(c.Args); got > max || got < min {
+				if err := s.validSelectWithAggregate(); err != nil {
-					return fmt.Errorf("invalid number of arguments for %s, expected at least %d but no more than %d, got %d", c.Name, min, max, got)
+					return err
 				}
+				if min, max, got := 1, 2, len(expr.Args); got > max || got < min {
+					return fmt.Errorf("invalid number of arguments for %s, expected at least %d but no more than %d, got %d", expr.Name, min, max, got)
+				}
+				// Validate that if they have grouping by time, they need a sub-call like min/max, etc.
+				groupByInterval, _ := s.GroupByInterval()
+				if groupByInterval > 0 {
+					if _, ok := expr.Args[0].(*Call); !ok {
+						return fmt.Errorf("aggregate function required inside the call to %s", expr.Name)
+					}
+				}
+
 			case "percentile":
-				if exp, got := 2, len(c.Args); got != exp {
+				if err := s.validSelectWithAggregate(); err != nil {
-					return fmt.Errorf("invalid number of arguments for %s, expected %d, got %d", c.Name, exp, got)
+					return err
+				}
+				if exp, got := 2, len(expr.Args); got != exp {
+					return fmt.Errorf("invalid number of arguments for %s, expected %d, got %d", expr.Name, exp, got)
+				}
+				_, ok := expr.Args[1].(*NumberLiteral)
+				if !ok {
+					return fmt.Errorf("expected float argument in percentile()")
+				}
+			case "top", "bottom":
+				if exp, got := 2, len(expr.Args); got < exp {
+					return fmt.Errorf("invalid number of arguments for %s, expected at least %d, got %d", expr.Name, exp, got)
+				}
+				if len(expr.Args) > 1 {
+					callLimit, ok := expr.Args[len(expr.Args)-1].(*NumberLiteral)
+					if !ok {
+						return fmt.Errorf("expected integer as last argument in %s(), found %s", expr.Name, expr.Args[len(expr.Args)-1])
+					}
+					// Check if they asked for a limit smaller than what they passed into the call
+					if int64(callLimit.Val) > int64(s.Limit) && s.Limit != 0 {
+						return fmt.Errorf("limit (%d) in %s function can not be larger than the LIMIT (%d) in the select statement", int64(callLimit.Val), expr.Name, int64(s.Limit))
+					}
+
+					for _, v := range expr.Args[:len(expr.Args)-1] {
+						if _, ok := v.(*VarRef); !ok {
+							return fmt.Errorf("only fields or tags are allowed in %s(), found %s", expr.Name, v)
+						}
+					}
 				}
 			default:
-				if exp, got := 1, len(c.Args); got != exp {
+				if err := s.validSelectWithAggregate(); err != nil {
-					return fmt.Errorf("invalid number of arguments for %s, expected %d, got %d", c.Name, exp, got)
+					return err
+				}
+				if exp, got := 1, len(expr.Args); got != exp {
+					return fmt.Errorf("invalid number of arguments for %s, expected %d, got %d", expr.Name, exp, got)
+				}
+				switch fc := expr.Args[0].(type) {
+				case *VarRef:
+					// do nothing
+				case *Call:
+					if fc.Name != "distinct" {
+						return fmt.Errorf("expected field argument in %s()", expr.Name)
+					}
+				case *Distinct:
+					if expr.Name != "count" {
+						return fmt.Errorf("expected field argument in %s()", expr.Name)
+					}
+				default:
+					return fmt.Errorf("expected field argument in %s()", expr.Name)
 				}
 			}
 		}
 	}
 
-	// Now, check that we have valid duration and where clauses for aggregates
+	// Check that we have valid duration and where clauses for aggregates
 
 	// fetch the group by duration
 	groupByDuration, _ := s.GroupByInterval()
@@ -1072,13 +1259,6 @@ func (s *SelectStatement) validateAggregates(tr targetRequirement) error {
 	return nil
 }
 
-func (s *SelectStatement) validateWildcard() error {
-	if s.HasWildcard() && len(s.Fields) > 1 {
-		return fmt.Errorf("wildcards can not be combined with other fields")
-	}
-	return nil
-}
-
 func (s *SelectStatement) HasDistinct() bool {
 	// determine if we have a call named distinct
 	for _, f := range s.Fields {
@@ -1378,6 +1558,25 @@ func (s *SelectStatement) NamesInDimension() []string {
 	return a
 }
 
+// LimitTagSets returns a tag set list with SLIMIT and SOFFSET applied.
+func (s *SelectStatement) LimitTagSets(a []*TagSet) []*TagSet {
+	// Ignore if no limit or offset is specified.
+	if s.SLimit == 0 && s.SOffset == 0 {
+		return a
+	}
+
+	// If offset is beyond the number of tag sets then return nil.
+	if s.SOffset > len(a) {
+		return nil
+	}
+
+	// Clamp limit to the max number of tag sets.
+	if s.SOffset+s.SLimit > len(a) {
+		s.SLimit = len(a) - s.SOffset
+	}
+	return a[s.SOffset : s.SOffset+s.SLimit]
+}
+
 // walkNames will walk the Expr and return the database fields
 func walkNames(exp Expr) []string {
 	switch expr := exp.(type) {
@@ -1414,6 +1613,15 @@ func (s *SelectStatement) FunctionCalls() []*Call {
 	return a
 }
 
+// FunctionCallsByPosition returns the Call objects from the query in the order they appear in the select statement
+func (s *SelectStatement) FunctionCallsByPosition() [][]*Call {
+	var a [][]*Call
+	for _, f := range s.Fields {
+		a = append(a, walkFunctionCalls(f.Expr))
+	}
+	return a
+}
+
 // walkFunctionCalls walks the Field of a query for any function calls made
 func walkFunctionCalls(exp Expr) []*Call {
 	switch expr := exp.(type) {
@@ -1501,6 +1709,9 @@ func (t *Target) String() string {
 	var buf bytes.Buffer
 	_, _ = buf.WriteString("INTO ")
 	_, _ = buf.WriteString(t.Measurement.String())
+	if t.Measurement.Name == "" {
+		_, _ = buf.WriteString(":MEASUREMENT")
+	}
 
 	return buf.String()
 }
@@ -1615,6 +1826,30 @@ func (s DropSeriesStatement) RequiredPrivileges() ExecutionPrivileges {
 	return ExecutionPrivileges{{Admin: false, Name: "", Privilege: WritePrivilege}}
 }
 
+// DropServerStatement represents a command for removing a server from the cluster.
+type DropServerStatement struct {
+	// ID of the node to be dropped.
+	NodeID uint64
+	// Force will force the server to drop even it it means losing data
+	Force bool
+}
+
+// String returns a string representation of the drop series statement.
+func (s *DropServerStatement) String() string {
+	var buf bytes.Buffer
+	_, _ = buf.WriteString("DROP SERVER ")
+	_, _ = buf.WriteString(strconv.FormatUint(s.NodeID, 10))
+	if s.Force {
+		_, _ = buf.WriteString(" FORCE")
+	}
+	return buf.String()
+}
+
+// RequiredPrivileges returns the privilege required to execute a DropServerStatement.
+func (s *DropServerStatement) RequiredPrivileges() ExecutionPrivileges {
+	return ExecutionPrivileges{{Name: "", Privilege: AllPrivileges}}
+}
+
 // ShowContinuousQueriesStatement represents a command for listing continuous queries.
 type ShowContinuousQueriesStatement struct{}
 
@@ -1810,18 +2045,19 @@ func (s *ShowRetentionPoliciesStatement) RequiredPrivileges() ExecutionPrivilege
 	return ExecutionPrivileges{{Admin: false, Name: "", Privilege: ReadPrivilege}}
 }
 
-// ShowRetentionPoliciesStatement represents a command for displaying stats for a given server.
+// ShowStats statement displays statistics for a given module.
 type ShowStatsStatement struct {
-	// Hostname or IP of the server for stats.
+	// Module
-	Host string
+	Module string
 }
 
 // String returns a string representation of a ShowStatsStatement.
 func (s *ShowStatsStatement) String() string {
 	var buf bytes.Buffer
 	_, _ = buf.WriteString("SHOW STATS ")
-	if s.Host != "" {
+	if s.Module != "" {
-		_, _ = buf.WriteString(s.Host)
+		_, _ = buf.WriteString("FOR ")
+		_, _ = buf.WriteString(s.Module)
 	}
 	return buf.String()
 }
@@ -1831,11 +2067,33 @@ func (s *ShowStatsStatement) RequiredPrivileges() ExecutionPrivileges {
 	return ExecutionPrivileges{{Admin: true, Name: "", Privilege: AllPrivileges}}
 }
 
+// ShowShardsStatement represents a command for displaying shards in the cluster.
+type ShowShardsStatement struct{}
+
+// String returns a string representation.
+func (s *ShowShardsStatement) String() string { return "SHOW SHARDS" }
+
+// RequiredPrivileges returns the privileges required to execute the statement.
+func (s *ShowShardsStatement) RequiredPrivileges() ExecutionPrivileges {
+	return ExecutionPrivileges{{Admin: true, Name: "", Privilege: AllPrivileges}}
+}
+
 // ShowDiagnosticsStatement represents a command for show node diagnostics.
-type ShowDiagnosticsStatement struct{}
+type ShowDiagnosticsStatement struct {
+	// Module
+	Module string
+}
 
 // String returns a string representation of the ShowDiagnosticsStatement.
-func (s *ShowDiagnosticsStatement) String() string { return "SHOW DIAGNOSTICS" }
+func (s *ShowDiagnosticsStatement) String() string {
+	var buf bytes.Buffer
+	_, _ = buf.WriteString("SHOW DIAGNOSTICS ")
+	if s.Module != "" {
+		_, _ = buf.WriteString("FOR ")
+		_, _ = buf.WriteString(s.Module)
+	}
+	return buf.String()
+}
 
 // RequiredPrivileges returns the privilege required to execute a ShowDiagnosticsStatement
 func (s *ShowDiagnosticsStatement) RequiredPrivileges() ExecutionPrivileges {
@@ -1853,12 +2111,17 @@ type ShowTagKeysStatement struct {
 	// Fields to sort results by
 	SortFields SortFields
 
-	// Maximum number of rows to be returned.
+	// Maximum number of tag keys per measurement. Unlimited if zero.
-	// Unlimited if zero.
 	Limit int
 
-	// Returns rows starting at an offset from the first row.
+	// Returns tag keys starting at an offset from the first row.
 	Offset int
+
+	// Maxiumum number of series to be returned. Unlimited if zero.
+	SLimit int
+
+	// Returns series starting at an offset from the first one.
+	SOffset int
 }
 
 // String returns a string representation of the statement.
@@ -1886,6 +2149,14 @@ func (s *ShowTagKeysStatement) String() string {
 		_, _ = buf.WriteString(" OFFSET ")
 		_, _ = buf.WriteString(strconv.Itoa(s.Offset))
 	}
+	if s.SLimit > 0 {
+		_, _ = buf.WriteString(" SLIMIT ")
+		_, _ = buf.WriteString(strconv.Itoa(s.SLimit))
+	}
+	if s.SOffset > 0 {
+		_, _ = buf.WriteString(" SOFFSET ")
+		_, _ = buf.WriteString(strconv.Itoa(s.SOffset))
+	}
 	return buf.String()
 }
 
@@ -2100,37 +2371,21 @@ func (a Dimensions) String() string {
 
 // Normalize returns the interval and tag dimensions separately.
 // Returns 0 if no time interval is specified.
-// Returns an error if multiple time dimensions exist or if non-VarRef dimensions are specified.
+func (a Dimensions) Normalize() (time.Duration, []string) {
-func (a Dimensions) Normalize() (time.Duration, []string, error) {
 	var dur time.Duration
 	var tags []string
 
 	for _, dim := range a {
 		switch expr := dim.Expr.(type) {
 		case *Call:
-			// Ensure the call is time() and it only has one duration argument.
+			lit, _ := expr.Args[0].(*DurationLiteral)
-			// If we already have a duration
+			dur = lit.Val
-			if expr.Name != "time" {
-				return 0, nil, errors.New("only time() calls allowed in dimensions")
-			} else if len(expr.Args) != 1 {
-				return 0, nil, errors.New("time dimension expected one argument")
-			} else if lit, ok := expr.Args[0].(*DurationLiteral); !ok {
-				return 0, nil, errors.New("time dimension must have one duration argument")
-			} else if dur != 0 {
-				return 0, nil, errors.New("multiple time dimensions not allowed")
-			} else {
-				dur = lit.Val
-			}
-
 		case *VarRef:
 			tags = append(tags, expr.Val)
-
-		default:
-			return 0, nil, errors.New("only time and tag dimensions allowed")
 		}
 	}
 
-	return dur, tags, nil
+	return dur, tags
 }
 
 // Dimension represents an expression that a select statement is grouped by.
@@ -2159,6 +2414,7 @@ type Measurement struct {
 	RetentionPolicy string
 	Name            string
 	Regex           *RegexLiteral
+	IsTarget        bool
 }
 
 // String returns a string representation of the measurement.
@@ -2217,6 +2473,47 @@ func (c *Call) String() string {
 	return fmt.Sprintf("%s(%s)", c.Name, strings.Join(str, ", "))
 }
 
+// Fields will extract any field names from the call.  Only specific calls support this.
+func (c *Call) Fields() []string {
+	switch c.Name {
+	case "top", "bottom":
+		// maintain the order the user specified in the query
+		keyMap := make(map[string]struct{})
+		keys := []string{}
+		for i, a := range c.Args {
+			if i == 0 {
+				// special case, first argument is always the name of the function regardless of the field name
+				keys = append(keys, c.Name)
+				continue
+			}
+			switch v := a.(type) {
+			case *VarRef:
+				if _, ok := keyMap[v.Val]; !ok {
+					keyMap[v.Val] = struct{}{}
+					keys = append(keys, v.Val)
+				}
+			}
+		}
+		return keys
+	case "min", "max", "first", "last", "sum", "mean":
+		// maintain the order the user specified in the query
+		keyMap := make(map[string]struct{})
+		keys := []string{}
+		for _, a := range c.Args {
+			switch v := a.(type) {
+			case *VarRef:
+				if _, ok := keyMap[v.Val]; !ok {
+					keyMap[v.Val] = struct{}{}
+					keys = append(keys, v.Val)
+				}
+			}
+		}
+		return keys
+	default:
+		panic(fmt.Sprintf("*call.Fields is unable to provide information on %s", c.Name))
+	}
+}
+
 // Distinct represents a DISTINCT expression.
 type Distinct struct {
 	// Identifier following DISTINCT
@@ -2525,6 +2822,10 @@ func Walk(v Visitor, node Node) {
 			Walk(v, c)
 		}
 
+	case *DropSeriesStatement:
+		Walk(v, n.Sources)
+		Walk(v, n.Condition)
+
 	case *Field:
 		Walk(v, n.Expr)
 
@@ -2772,6 +3073,13 @@ func evalBinaryExpr(expr *BinaryExpr, m map[string]interface{}) interface{} {
 	return nil
 }
 
+// EvalBool evaluates expr and returns true if result is a boolean true.
+// Otherwise returns false.
+func EvalBool(expr Expr, m map[string]interface{}) bool {
+	v, _ := Eval(expr, m).(bool)
+	return v
+}
+
 // Reduce evaluates expr using the available values in valuer.
 // References that don't exist in valuer are ignored.
 func Reduce(expr Expr, valuer Valuer) Expr {

Godeps/_workspace/src/github.com/influxdb/influxdb/influxql/ast_test.go

@@ -451,7 +451,7 @@ func TestSelectStatement_IsRawQuerySet(t *testing.T) {
 			isRaw: false,
 		},
 		{
-			stmt:  "select mean(*) from foo group by *",
+			stmt:  "select mean(value) from foo group by *",
 			isRaw: false,
 		},
 	}

Godeps/_workspace/src/github.com/influxdb/influxdb/influxql/functions_test.go

@@ -1,534 +0,0 @@
-package influxql
-
-import (
-	"reflect"
-	"testing"
-	"time"
-
-	"github.com/davecgh/go-spew/spew"
-)
-
-import "sort"
-
-type point struct {
-	seriesKey string
-	time      int64
-	value     interface{}
-}
-
-type testIterator struct {
-	values []point
-}
-
-func (t *testIterator) Next() (timestamp int64, value interface{}) {
-	if len(t.values) > 0 {
-		v := t.values[0]
-		t.values = t.values[1:]
-		return v.time, v.value
-	}
-
-	return -1, nil
-}
-
-func TestMapMeanNoValues(t *testing.T) {
-	iter := &testIterator{}
-	if got := MapMean(iter); got != nil {
-		t.Errorf("output mismatch: exp nil got %v", got)
-	}
-}
-
-func TestMapMean(t *testing.T) {
-
-	tests := []struct {
-		input  []point
-		output *meanMapOutput
-	}{
-		{ // Single point
-			input:  []point{point{"0", 1, 1.0}},
-			output: &meanMapOutput{1, 1, Float64Type},
-		},
-		{ // Two points
-			input: []point{
-				point{"0", 1, 2.0},
-				point{"0", 2, 8.0},
-			},
-			output: &meanMapOutput{2, 5.0, Float64Type},
-		},
-	}
-
-	for _, test := range tests {
-		iter := &testIterator{
-			values: test.input,
-		}
-
-		got := MapMean(iter)
-		if got == nil {
-			t.Fatalf("MapMean(%v): output mismatch: exp %v got %v", test.input, test.output, got)
-		}
-
-		if got.(*meanMapOutput).Count != test.output.Count || got.(*meanMapOutput).Mean != test.output.Mean {
-			t.Errorf("output mismatch: exp %v got %v", test.output, got)
-		}
-	}
-}
-func TestInitializeMapFuncPercentile(t *testing.T) {
-	// No args
-	c := &Call{
-		Name: "percentile",
-		Args: []Expr{},
-	}
-	_, err := InitializeMapFunc(c)
-	if err == nil {
-		t.Errorf("InitializeMapFunc(%v) expected error. got nil", c)
-	}
-
-	if exp := "expected two arguments for percentile()"; err.Error() != exp {
-		t.Errorf("InitializeMapFunc(%v) mismatch. exp %v got %v", c, exp, err.Error())
-	}
-
-	// No percentile arg
-	c = &Call{
-		Name: "percentile",
-		Args: []Expr{
-			&VarRef{Val: "field1"},
-		},
-	}
-
-	_, err = InitializeMapFunc(c)
-	if err == nil {
-		t.Errorf("InitializeMapFunc(%v) expected error. got nil", c)
-	}
-
-	if exp := "expected two arguments for percentile()"; err.Error() != exp {
-		t.Errorf("InitializeMapFunc(%v) mismatch. exp %v got %v", c, exp, err.Error())
-	}
-}
-
-func TestInitializeMapFuncDerivative(t *testing.T) {
-
-	for _, fn := range []string{"derivative", "non_negative_derivative"} {
-		// No args should fail
-		c := &Call{
-			Name: fn,
-			Args: []Expr{},
-		}
-
-		_, err := InitializeMapFunc(c)
-		if err == nil {
-			t.Errorf("InitializeMapFunc(%v) expected error.  got nil", c)
-		}
-
-		// Single field arg should return MapEcho
-		c = &Call{
-			Name: fn,
-			Args: []Expr{
-				&VarRef{Val: " field1"},
-				&DurationLiteral{Val: time.Hour},
-			},
-		}
-
-		_, err = InitializeMapFunc(c)
-		if err != nil {
-			t.Errorf("InitializeMapFunc(%v) unexpected error.  got %v", c, err)
-		}
-
-		// Nested Aggregate func should return the map func for the nested aggregate
-		c = &Call{
-			Name: fn,
-			Args: []Expr{
-				&Call{Name: "mean", Args: []Expr{&VarRef{Val: "field1"}}},
-				&DurationLiteral{Val: time.Hour},
-			},
-		}
-
-		_, err = InitializeMapFunc(c)
-		if err != nil {
-			t.Errorf("InitializeMapFunc(%v) unexpected error.  got %v", c, err)
-		}
-	}
-}
-
-func TestInitializeReduceFuncPercentile(t *testing.T) {
-	// No args
-	c := &Call{
-		Name: "percentile",
-		Args: []Expr{},
-	}
-	_, err := InitializeReduceFunc(c)
-	if err == nil {
-		t.Errorf("InitializedReduceFunc(%v) expected error. got nil", c)
-	}
-
-	if exp := "expected float argument in percentile()"; err.Error() != exp {
-		t.Errorf("InitializedReduceFunc(%v) mismatch. exp %v got %v", c, exp, err.Error())
-	}
-
-	// No percentile arg
-	c = &Call{
-		Name: "percentile",
-		Args: []Expr{
-			&VarRef{Val: "field1"},
-		},
-	}
-
-	_, err = InitializeReduceFunc(c)
-	if err == nil {
-		t.Errorf("InitializedReduceFunc(%v) expected error. got nil", c)
-	}
-
-	if exp := "expected float argument in percentile()"; err.Error() != exp {
-		t.Errorf("InitializedReduceFunc(%v) mismatch. exp %v got %v", c, exp, err.Error())
-	}
-}
-
-func TestReducePercentileNil(t *testing.T) {
-
-	// ReducePercentile should ignore nil values when calculating the percentile
-	fn := ReducePercentile(100)
-	input := []interface{}{
-		nil,
-	}
-
-	got := fn(input)
-	if got != nil {
-		t.Fatalf("ReducePercentile(100) returned wrong type. exp nil got %v", got)
-	}
-}
-
-func TestMapDistinct(t *testing.T) {
-	const ( // prove that we're ignoring seriesKey
-		seriesKey1 = "1"
-		seriesKey2 = "2"
-	)
-
-	const ( // prove that we're ignoring time
-		timeId1 = iota + 1
-		timeId2
-		timeId3
-		timeId4
-		timeId5
-		timeId6
-	)
-
-	iter := &testIterator{
-		values: []point{
-			{seriesKey1, timeId1, uint64(1)},
-			{seriesKey1, timeId2, uint64(1)},
-			{seriesKey1, timeId3, "1"},
-			{seriesKey2, timeId4, uint64(1)},
-			{seriesKey2, timeId5, float64(1.0)},
-			{seriesKey2, timeId6, "1"},
-		},
-	}
-
-	values := MapDistinct(iter).(distinctValues)
-
-	if exp, got := 3, len(values); exp != got {
-		t.Errorf("Wrong number of values. exp %v got %v", exp, got)
-	}
-
-	sort.Sort(values)
-
-	exp := distinctValues{
-		uint64(1),
-		float64(1),
-		"1",
-	}
-
-	if !reflect.DeepEqual(values, exp) {
-		t.Errorf("Wrong values. exp %v got %v", spew.Sdump(exp), spew.Sdump(values))
-	}
-}
-
-func TestMapDistinctNil(t *testing.T) {
-	iter := &testIterator{
-		values: []point{},
-	}
-
-	values := MapDistinct(iter)
-
-	if values != nil {
-		t.Errorf("Wrong values. exp nil got %v", spew.Sdump(values))
-	}
-}
-
-func TestReduceDistinct(t *testing.T) {
-	v1 := distinctValues{
-		"2",
-		"1",
-		float64(2.0),
-		float64(1),
-		uint64(2),
-		uint64(1),
-		true,
-		false,
-	}
-
-	expect := distinctValues{
-		uint64(1),
-		float64(1),
-		uint64(2),
-		float64(2),
-		false,
-		true,
-		"1",
-		"2",
-	}
-
-	got := ReduceDistinct([]interface{}{v1, v1, expect})
-
-	if !reflect.DeepEqual(got, expect) {
-		t.Errorf("Wrong values. exp %v got %v", spew.Sdump(expect), spew.Sdump(got))
-	}
-}
-
-func TestReduceDistinctNil(t *testing.T) {
-	tests := []struct {
-		name   string
-		values []interface{}
-	}{
-		{
-			name:   "nil values",
-			values: nil,
-		},
-		{
-			name:   "nil mapper",
-			values: []interface{}{nil},
-		},
-		{
-			name:   "no mappers",
-			values: []interface{}{},
-		},
-		{
-			name:   "empty mappper (len 1)",
-			values: []interface{}{distinctValues{}},
-		},
-		{
-			name:   "empty mappper (len 2)",
-			values: []interface{}{distinctValues{}, distinctValues{}},
-		},
-	}
-
-	for _, test := range tests {
-		t.Log(test.name)
-		got := ReduceDistinct(test.values)
-		if got != nil {
-			t.Errorf("Wrong values. exp nil got %v", spew.Sdump(got))
-		}
-	}
-}
-
-func Test_distinctValues_Sort(t *testing.T) {
-	values := distinctValues{
-		"2",
-		"1",
-		float64(2.0),
-		float64(1),
-		uint64(2),
-		uint64(1),
-		true,
-		false,
-	}
-
-	expect := distinctValues{
-		uint64(1),
-		float64(1),
-		uint64(2),
-		float64(2),
-		false,
-		true,
-		"1",
-		"2",
-	}
-
-	sort.Sort(values)
-
-	if !reflect.DeepEqual(values, expect) {
-		t.Errorf("Wrong values. exp %v got %v", spew.Sdump(expect), spew.Sdump(values))
-	}
-}
-
-func TestMapCountDistinct(t *testing.T) {
-	const ( // prove that we're ignoring seriesKey
-		seriesKey1 = "1"
-		seriesKey2 = "2"
-	)
-
-	const ( // prove that we're ignoring time
-		timeId1 = iota + 1
-		timeId2
-		timeId3
-		timeId4
-		timeId5
-		timeId6
-		timeId7
-	)
-
-	iter := &testIterator{
-		values: []point{
-			{seriesKey1, timeId1, uint64(1)},
-			{seriesKey1, timeId2, uint64(1)},
-			{seriesKey1, timeId3, "1"},
-			{seriesKey2, timeId4, uint64(1)},
-			{seriesKey2, timeId5, float64(1.0)},
-			{seriesKey2, timeId6, "1"},
-			{seriesKey2, timeId7, true},
-		},
-	}
-
-	values := MapCountDistinct(iter).(map[interface{}]struct{})
-
-	if exp, got := 4, len(values); exp != got {
-		t.Errorf("Wrong number of values. exp %v got %v", exp, got)
-	}
-
-	exp := map[interface{}]struct{}{
-		uint64(1):  struct{}{},
-		float64(1): struct{}{},
-		"1":        struct{}{},
-		true:       struct{}{},
-	}
-
-	if !reflect.DeepEqual(values, exp) {
-		t.Errorf("Wrong values. exp %v got %v", spew.Sdump(exp), spew.Sdump(values))
-	}
-}
-
-func TestMapCountDistinctNil(t *testing.T) {
-	iter := &testIterator{
-		values: []point{},
-	}
-
-	values := MapCountDistinct(iter)
-
-	if values != nil {
-		t.Errorf("Wrong values. exp nil got %v", spew.Sdump(values))
-	}
-}
-
-func TestReduceCountDistinct(t *testing.T) {
-	v1 := map[interface{}]struct{}{
-		"2":          struct{}{},
-		"1":          struct{}{},
-		float64(2.0): struct{}{},
-		float64(1):   struct{}{},
-		uint64(2):    struct{}{},
-		uint64(1):    struct{}{},
-		true:         struct{}{},
-		false:        struct{}{},
-	}
-
-	v2 := map[interface{}]struct{}{
-		uint64(1):  struct{}{},
-		float64(1): struct{}{},
-		uint64(2):  struct{}{},
-		float64(2): struct{}{},
-		false:      struct{}{},
-		true:       struct{}{},
-		"1":        struct{}{},
-		"2":        struct{}{},
-	}
-
-	exp := 8
-	got := ReduceCountDistinct([]interface{}{v1, v1, v2})
-
-	if !reflect.DeepEqual(got, exp) {
-		t.Errorf("Wrong values. exp %v got %v", spew.Sdump(exp), spew.Sdump(got))
-	}
-}
-
-func TestReduceCountDistinctNil(t *testing.T) {
-	emptyResults := make(map[interface{}]struct{})
-	tests := []struct {
-		name   string
-		values []interface{}
-	}{
-		{
-			name:   "nil values",
-			values: nil,
-		},
-		{
-			name:   "nil mapper",
-			values: []interface{}{nil},
-		},
-		{
-			name:   "no mappers",
-			values: []interface{}{},
-		},
-		{
-			name:   "empty mappper (len 1)",
-			values: []interface{}{emptyResults},
-		},
-		{
-			name:   "empty mappper (len 2)",
-			values: []interface{}{emptyResults, emptyResults},
-		},
-	}
-
-	for _, test := range tests {
-		t.Log(test.name)
-		got := ReduceCountDistinct(test.values)
-		if got != 0 {
-			t.Errorf("Wrong values. exp nil got %v", spew.Sdump(got))
-		}
-	}
-}
-
-var getSortedRangeData = []float64{
-	60, 61, 62, 63, 64, 65, 66, 67, 68, 69,
-	20, 21, 22, 23, 24, 25, 26, 27, 28, 29,
-	0, 1, 2, 3, 4, 5, 6, 7, 8, 9,
-	40, 41, 42, 43, 44, 45, 46, 47, 48, 49,
-	10, 11, 12, 13, 14, 15, 16, 17, 18, 19,
-	50, 51, 52, 53, 54, 55, 56, 57, 58, 59,
-	30, 31, 32, 33, 34, 35, 36, 37, 38, 39,
-}
-
-var getSortedRangeTests = []struct {
-	name     string
-	data     []float64
-	start    int
-	count    int
-	expected []float64
-}{
-	{"first 5", getSortedRangeData, 0, 5, []float64{0, 1, 2, 3, 4}},
-	{"0 length", getSortedRangeData, 8, 0, []float64{}},
-	{"past end of data", getSortedRangeData, len(getSortedRangeData) - 3, 5, []float64{67, 68, 69}},
-}
-
-func TestGetSortedRange(t *testing.T) {
-	for _, tt := range getSortedRangeTests {
-		results := getSortedRange(tt.data, tt.start, tt.count)
-		if len(results) != len(tt.expected) {
-			t.Errorf("Test %s error.  Expected getSortedRange to return %v but got %v", tt.name, tt.expected, results)
-		}
-		for i, point := range tt.expected {
-			if point != results[i] {
-				t.Errorf("Test %s error. getSortedRange returned wrong result for index %v.  Expected %v but got %v", tt.name, i, point, results[i])
-			}
-		}
-	}
-}
-
-var benchGetSortedRangeResults []float64
-
-func BenchmarkGetSortedRangeByPivot(b *testing.B) {
-	data := make([]float64, len(getSortedRangeData))
-	var results []float64
-	for i := 0; i < b.N; i++ {
-		copy(data, getSortedRangeData)
-		results = getSortedRange(data, 8, 15)
-	}
-	benchGetSortedRangeResults = results
-}
-
-func BenchmarkGetSortedRangeBySort(b *testing.B) {
-	data := make([]float64, len(getSortedRangeData))
-	var results []float64
-	for i := 0; i < b.N; i++ {
-		copy(data, getSortedRangeData)
-		sort.Float64s(data)
-		results = data[8:23]
-	}
-	benchGetSortedRangeResults = results
-}

Godeps/_workspace/src/github.com/influxdb/influxdb/influxql/parser.go

@@ -7,6 +7,7 @@ import (
 	"io"
 	"math"
 	"regexp"
+	"sort"
 	"strconv"
 	"strings"
 	"time"
@@ -129,6 +130,8 @@ func (p *Parser) parseShowStatement() (Statement, error) {
 		return nil, newParseError(tokstr(tok, lit), []string{"POLICIES"}, pos)
 	case SERIES:
 		return p.parseShowSeriesStatement()
+	case SHARDS:
+		return p.parseShowShardsStatement()
 	case STATS:
 		return p.parseShowStatsStatement()
 	case DIAGNOSTICS:
@@ -145,7 +148,24 @@ func (p *Parser) parseShowStatement() (Statement, error) {
 		return p.parseShowUsersStatement()
 	}
 
-	return nil, newParseError(tokstr(tok, lit), []string{"CONTINUOUS", "DATABASES", "FIELD", "GRANTS", "MEASUREMENTS", "RETENTION", "SERIES", "SERVERS", "TAG", "USERS"}, pos)
+	showQueryKeywords := []string{
+		"CONTINUOUS",
+		"DATABASES",
+		"FIELD",
+		"GRANTS",
+		"MEASUREMENTS",
+		"RETENTION",
+		"SERIES",
+		"SERVERS",
+		"TAG",
+		"USERS",
+		"STATS",
+		"DIAGNOSTICS",
+		"SHARDS",
+	}
+	sort.Strings(showQueryKeywords)
+
+	return nil, newParseError(tokstr(tok, lit), showQueryKeywords, pos)
 }
 
 // parseCreateStatement parses a string and returns a create statement.
@@ -188,6 +208,8 @@ func (p *Parser) parseDropStatement() (Statement, error) {
 		return p.parseDropRetentionPolicyStatement()
 	} else if tok == USER {
 		return p.parseDropUserStatement()
+	} else if tok == SERVER {
+		return p.parseDropServerStatement()
 	}
 
 	return nil, newParseError(tokstr(tok, lit), []string{"SERIES", "CONTINUOUS", "MEASUREMENT"}, pos)
@@ -291,8 +313,8 @@ func (p *Parser) parseCreateRetentionPolicyStatement() (*CreateRetentionPolicySt
 	// Parse optional DEFAULT token.
 	if tok, pos, lit = p.scanIgnoreWhitespace(); tok == DEFAULT {
 		stmt.Default = true
-	} else {
+	} else if tok != EOF && tok != SEMICOLON {
-		p.unscan()
+		return nil, newParseError(tokstr(tok, lit), []string{"DEFAULT"}, pos)
 	}
 
 	return stmt, nil
@@ -488,6 +510,9 @@ func (p *Parser) parseSegmentedIdents() ([]string, error) {
 		if ch := p.peekRune(); ch == '/' {
 			// Next segment is a regex so we're done.
 			break
+		} else if ch == ':' {
+			// Next segment is context-specific so let caller handle it.
+			break
 		} else if ch == '.' {
 			// Add an empty identifier.
 			idents = append(idents, "")
@@ -799,7 +824,18 @@ func (p *Parser) parseTarget(tr targetRequirement) (*Target, error) {
 		return nil, err
 	}
 
-	t := &Target{Measurement: &Measurement{}}
+	if len(idents) < 3 {
+		// Check for source measurement reference.
+		if ch := p.peekRune(); ch == ':' {
+			if err := p.parseTokens([]Token{COLON, MEASUREMENT}); err != nil {
+				return nil, err
+			}
+			// Append empty measurement name.
+			idents = append(idents, "")
+		}
+	}
+
+	t := &Target{Measurement: &Measurement{IsTarget: true}}
 
 	switch len(idents) {
 	case 1:
@@ -963,6 +999,16 @@ func (p *Parser) parseShowTagKeysStatement() (*ShowTagKeysStatement, error) {
 		return nil, err
 	}
 
+	// Parse series limit: "SLIMIT <n>".
+	if stmt.SLimit, err = p.parseOptionalTokenAndInt(SLIMIT); err != nil {
+		return nil, err
+	}
+
+	// Parse series offset: "SOFFSET <n>".
+	if stmt.SOffset, err = p.parseOptionalTokenAndInt(SOFFSET); err != nil {
+		return nil, err
+	}
+
 	return stmt, nil
 }
 
@@ -1134,6 +1180,27 @@ func (p *Parser) parseDropSeriesStatement() (*DropSeriesStatement, error) {
 	return stmt, nil
 }
 
+// parseDropServerStatement parses a string and returns a DropServerStatement.
+// This function assumes the "DROP SERVER" tokens have already been consumed.
+func (p *Parser) parseDropServerStatement() (*DropServerStatement, error) {
+	s := &DropServerStatement{}
+	var err error
+
+	// Parse the server's ID.
+	if s.NodeID, err = p.parseUInt64(); err != nil {
+		return nil, err
+	}
+
+	// Parse optional FORCE token.
+	if tok, pos, lit := p.scanIgnoreWhitespace(); tok == FORCE {
+		s.Force = true
+	} else if tok != EOF && tok != SEMICOLON {
+		return nil, newParseError(tokstr(tok, lit), []string{"FORCE"}, pos)
+	}
+
+	return s, nil
+}
+
 // parseShowContinuousQueriesStatement parses a string and returns a ShowContinuousQueriesStatement.
 // This function assumes the "SHOW CONTINUOUS" tokens have already been consumed.
 func (p *Parser) parseShowContinuousQueriesStatement() (*ShowContinuousQueriesStatement, error) {
@@ -1250,6 +1317,16 @@ func (p *Parser) parseCreateContinuousQueryStatement() (*CreateContinuousQuerySt
 func (p *Parser) parseCreateDatabaseStatement() (*CreateDatabaseStatement, error) {
 	stmt := &CreateDatabaseStatement{}
 
+	// Look for "IF NOT EXISTS"
+	if tok, _, _ := p.scanIgnoreWhitespace(); tok == IF {
+		if err := p.parseTokens([]Token{NOT, EXISTS}); err != nil {
+			return nil, err
+		}
+		stmt.IfNotExists = true
+	} else {
+		p.unscan()
+	}
+
 	// Parse the name of the database to be created.
 	lit, err := p.parseIdent()
 	if err != nil {
@@ -1385,14 +1462,20 @@ func (p *Parser) parseRetentionPolicy() (name string, dfault bool, err error) {
 	return
 }
 
+// parseShowShardsStatement parses a string for "SHOW SHARDS" statement.
+// This function assumes the "SHOW SHARDS" tokens have already been consumed.
+func (p *Parser) parseShowShardsStatement() (*ShowShardsStatement, error) {
+	return &ShowShardsStatement{}, nil
+}
+
 // parseShowStatsStatement parses a string and returns a ShowStatsStatement.
 // This function assumes the "SHOW STATS" tokens have already been consumed.
 func (p *Parser) parseShowStatsStatement() (*ShowStatsStatement, error) {
 	stmt := &ShowStatsStatement{}
 	var err error
 
-	if tok, _, _ := p.scanIgnoreWhitespace(); tok == ON {
+	if tok, _, _ := p.scanIgnoreWhitespace(); tok == FOR {
-		stmt.Host, err = p.parseString()
+		stmt.Module, err = p.parseString()
 	} else {
 		p.unscan()
 	}
@@ -1403,7 +1486,15 @@ func (p *Parser) parseShowStatsStatement() (*ShowStatsStatement, error) {
 // parseShowDiagnostics parses a string and returns a ShowDiagnosticsStatement.
 func (p *Parser) parseShowDiagnosticsStatement() (*ShowDiagnosticsStatement, error) {
 	stmt := &ShowDiagnosticsStatement{}
-	return stmt, nil
+	var err error
+
+	if tok, _, _ := p.scanIgnoreWhitespace(); tok == FOR {
+		stmt.Module, err = p.parseString()
+	} else {
+		p.unscan()
+	}
+
+	return stmt, err
 }
 
 // parseDropContinuousQueriesStatement parses a string and returns a DropContinuousQueryStatement.
@@ -1441,13 +1532,6 @@ func (p *Parser) parseDropContinuousQueryStatement() (*DropContinuousQueryStatem
 func (p *Parser) parseFields() (Fields, error) {
 	var fields Fields
 
-	// Check for "*" (i.e., "all fields")
-	if tok, _, _ := p.scanIgnoreWhitespace(); tok == MUL {
-		fields = append(fields, &Field{&Wildcard{}, ""})
-		return fields, nil
-	}
-	p.unscan()
-
 	for {
 		// Parse the field.
 		f, err := p.parseField()
@@ -1777,24 +1861,29 @@ func (p *Parser) parseOrderBy() (SortFields, error) {
 func (p *Parser) parseSortFields() (SortFields, error) {
 	var fields SortFields
 
-	// If first token is ASC or DESC, all fields are sorted.
+	tok, pos, lit := p.scanIgnoreWhitespace()
-	if tok, pos, lit := p.scanIgnoreWhitespace(); tok == ASC || tok == DESC {
+
-		if tok == DESC {
+	switch tok {
-			// Token must be ASC, until other sort orders are supported.
+	// The first field after an order by may not have a field name (e.g. ORDER BY ASC)
-			return nil, errors.New("only ORDER BY time ASC supported at this time")
+	case ASC, DESC:
+		fields = append(fields, &SortField{Ascending: (tok == ASC)})
+	// If it's a token, parse it as a sort field.  At least one is required.
+	case IDENT:
+		p.unscan()
+		field, err := p.parseSortField()
+		if err != nil {
+			return nil, err
 		}
-		return append(fields, &SortField{Ascending: (tok == ASC)}), nil
+
-	} else if tok != IDENT {
+		if lit != "time" {
+			return nil, errors.New("only ORDER BY time supported at this time")
+		}
+
+		fields = append(fields, field)
+	// Parse error...
+	default:
 		return nil, newParseError(tokstr(tok, lit), []string{"identifier", "ASC", "DESC"}, pos)
 	}
-	p.unscan()
-
-	// At least one field is required.
-	field, err := p.parseSortField()
-	if err != nil {
-		return nil, err
-	}
-	fields = append(fields, field)
 
 	// Parse additional fields.
 	for {
@@ -1813,9 +1902,8 @@ func (p *Parser) parseSortFields() (SortFields, error) {
 		fields = append(fields, field)
 	}
 
-	// First SortField must be time ASC, until other sort orders are supported.
+	if len(fields) > 1 {
-	if len(fields) > 1 || fields[0].Name != "time" || !fields[0].Ascending {
+		return nil, errors.New("only ORDER BY time supported at this time")
-		return nil, errors.New("only ORDER BY time ASC supported at this time")
 	}
 
 	return fields, nil

Godeps/_workspace/src/github.com/influxdb/influxdb/influxql/parser_test.go

@@ -73,11 +73,45 @@ func TestParser_ParseStatement(t *testing.T) {
 				Sources: []influxql.Source{&influxql.Measurement{Name: "myseries"}},
 			},
 		},
+		{
+			s: `SELECT * FROM myseries GROUP BY *`,
+			stmt: &influxql.SelectStatement{
+				IsRawQuery: true,
+				Fields: []*influxql.Field{
+					{Expr: &influxql.Wildcard{}},
+				},
+				Sources:    []influxql.Source{&influxql.Measurement{Name: "myseries"}},
+				Dimensions: []*influxql.Dimension{{Expr: &influxql.Wildcard{}}},
+			},
+		},
+		{
+			s: `SELECT field1, * FROM myseries GROUP BY *`,
+			stmt: &influxql.SelectStatement{
+				IsRawQuery: true,
+				Fields: []*influxql.Field{
+					{Expr: &influxql.VarRef{Val: "field1"}},
+					{Expr: &influxql.Wildcard{}},
+				},
+				Sources:    []influxql.Source{&influxql.Measurement{Name: "myseries"}},
+				Dimensions: []*influxql.Dimension{{Expr: &influxql.Wildcard{}}},
+			},
+		},
+		{
+			s: `SELECT *, field1 FROM myseries GROUP BY *`,
+			stmt: &influxql.SelectStatement{
+				IsRawQuery: true,
+				Fields: []*influxql.Field{
+					{Expr: &influxql.Wildcard{}},
+					{Expr: &influxql.VarRef{Val: "field1"}},
+				},
+				Sources:    []influxql.Source{&influxql.Measurement{Name: "myseries"}},
+				Dimensions: []*influxql.Dimension{{Expr: &influxql.Wildcard{}}},
+			},
+		},
 
 		// SELECT statement
 		{
-			skip: true,
+			s: fmt.Sprintf(`SELECT mean(field1), sum(field2) ,count(field3) AS field_x FROM myseries WHERE host = 'hosta.influxdb.org' and time > '%s' GROUP BY time(10h) ORDER BY DESC LIMIT 20 OFFSET 10;`, now.UTC().Format(time.RFC3339Nano)),
-			s:    fmt.Sprintf(`SELECT mean(field1), sum(field2) ,count(field3) AS field_x FROM myseries WHERE host = 'hosta.influxdb.org' and time > '%s' GROUP BY time(10h) ORDER BY ASC LIMIT 20 OFFSET 10;`, now.UTC().Format(time.RFC3339Nano)),
 			stmt: &influxql.SelectStatement{
 				IsRawQuery: false,
 				Fields: []*influxql.Field{
@@ -101,12 +135,32 @@ func TestParser_ParseStatement(t *testing.T) {
 				},
 				Dimensions: []*influxql.Dimension{{Expr: &influxql.Call{Name: "time", Args: []influxql.Expr{&influxql.DurationLiteral{Val: 10 * time.Hour}}}}},
 				SortFields: []*influxql.SortField{
-					{Ascending: true},
+					{Ascending: false},
 				},
 				Limit:  20,
 				Offset: 10,
 			},
 		},
+		{
+			s: `SELECT "foo.bar.baz" AS foo FROM myseries`,
+			stmt: &influxql.SelectStatement{
+				IsRawQuery: true,
+				Fields: []*influxql.Field{
+					{Expr: &influxql.VarRef{Val: "foo.bar.baz"}, Alias: "foo"},
+				},
+				Sources: []influxql.Source{&influxql.Measurement{Name: "myseries"}},
+			},
+		},
+		{
+			s: `SELECT "foo.bar.baz" AS foo FROM foo`,
+			stmt: &influxql.SelectStatement{
+				IsRawQuery: true,
+				Fields: []*influxql.Field{
+					{Expr: &influxql.VarRef{Val: "foo.bar.baz"}, Alias: "foo"},
+				},
+				Sources: []influxql.Source{&influxql.Measurement{Name: "foo"}},
+			},
+		},
 
 		// derivative
 		{
@@ -120,6 +174,22 @@ func TestParser_ParseStatement(t *testing.T) {
 			},
 		},
 
+		{
+			s: fmt.Sprintf(`SELECT derivative(field1, 1h) FROM myseries WHERE time > '%s'`, now.UTC().Format(time.RFC3339Nano)),
+			stmt: &influxql.SelectStatement{
+				IsRawQuery: false,
+				Fields: []*influxql.Field{
+					{Expr: &influxql.Call{Name: "derivative", Args: []influxql.Expr{&influxql.VarRef{Val: "field1"}, &influxql.DurationLiteral{Val: time.Hour}}}},
+				},
+				Sources: []influxql.Source{&influxql.Measurement{Name: "myseries"}},
+				Condition: &influxql.BinaryExpr{
+					Op:  influxql.GT,
+					LHS: &influxql.VarRef{Val: "time"},
+					RHS: &influxql.TimeLiteral{Val: now.UTC()},
+				},
+			},
+		},
+
 		{
 			s: `SELECT derivative(mean(field1), 1h) FROM myseries;`,
 			stmt: &influxql.SelectStatement{
@@ -214,6 +284,65 @@ func TestParser_ParseStatement(t *testing.T) {
 			},
 		},
 
+		// select percentile statements
+		{
+			s: `select percentile("field1", 2.0) from cpu`,
+			stmt: &influxql.SelectStatement{
+				IsRawQuery: false,
+				Fields: []*influxql.Field{
+					{Expr: &influxql.Call{Name: "percentile", Args: []influxql.Expr{&influxql.VarRef{Val: "field1"}, &influxql.NumberLiteral{Val: 2.0}}}},
+				},
+				Sources: []influxql.Source{&influxql.Measurement{Name: "cpu"}},
+			},
+		},
+
+		// select top statements
+		{
+			s: `select top("field1", 2) from cpu`,
+			stmt: &influxql.SelectStatement{
+				IsRawQuery: false,
+				Fields: []*influxql.Field{
+					{Expr: &influxql.Call{Name: "top", Args: []influxql.Expr{&influxql.VarRef{Val: "field1"}, &influxql.NumberLiteral{Val: 2}}}},
+				},
+				Sources: []influxql.Source{&influxql.Measurement{Name: "cpu"}},
+			},
+		},
+
+		{
+			s: `select top(field1, 2) from cpu`,
+			stmt: &influxql.SelectStatement{
+				IsRawQuery: false,
+				Fields: []*influxql.Field{
+					{Expr: &influxql.Call{Name: "top", Args: []influxql.Expr{&influxql.VarRef{Val: "field1"}, &influxql.NumberLiteral{Val: 2}}}},
+				},
+				Sources: []influxql.Source{&influxql.Measurement{Name: "cpu"}},
+			},
+		},
+
+		{
+			s: `select top(field1, 2), tag1 from cpu`,
+			stmt: &influxql.SelectStatement{
+				IsRawQuery: false,
+				Fields: []*influxql.Field{
+					{Expr: &influxql.Call{Name: "top", Args: []influxql.Expr{&influxql.VarRef{Val: "field1"}, &influxql.NumberLiteral{Val: 2}}}},
+					{Expr: &influxql.VarRef{Val: "tag1"}},
+				},
+				Sources: []influxql.Source{&influxql.Measurement{Name: "cpu"}},
+			},
+		},
+
+		{
+			s: `select top(field1, tag1, 2), tag1 from cpu`,
+			stmt: &influxql.SelectStatement{
+				IsRawQuery: false,
+				Fields: []*influxql.Field{
+					{Expr: &influxql.Call{Name: "top", Args: []influxql.Expr{&influxql.VarRef{Val: "field1"}, &influxql.VarRef{Val: "tag1"}, &influxql.NumberLiteral{Val: 2}}}},
+					{Expr: &influxql.VarRef{Val: "tag1"}},
+				},
+				Sources: []influxql.Source{&influxql.Measurement{Name: "cpu"}},
+			},
+		},
+
 		// select distinct statements
 		{
 			s: `select distinct(field1) from cpu`,
@@ -587,17 +716,17 @@ func TestParser_ParseStatement(t *testing.T) {
 		// SHOW SERIES WHERE with ORDER BY and LIMIT
 		{
 			skip: true,
-			s:    `SHOW SERIES WHERE region = 'uswest' ORDER BY ASC, field1, field2 DESC LIMIT 10`,
+			s:    `SHOW SERIES WHERE region = 'order by desc' ORDER BY DESC, field1, field2 DESC LIMIT 10`,
 			stmt: &influxql.ShowSeriesStatement{
 				Condition: &influxql.BinaryExpr{
 					Op:  influxql.EQ,
 					LHS: &influxql.VarRef{Val: "region"},
-					RHS: &influxql.StringLiteral{Val: "uswest"},
+					RHS: &influxql.StringLiteral{Val: "order by desc"},
 				},
 				SortFields: []*influxql.SortField{
-					{Ascending: true},
+					&influxql.SortField{Ascending: false},
-					{Name: "field1"},
+					&influxql.SortField{Name: "field1", Ascending: true},
-					{Name: "field2"},
+					&influxql.SortField{Name: "field2"},
 				},
 				Limit: 10,
 			},
@@ -638,6 +767,74 @@ func TestParser_ParseStatement(t *testing.T) {
 			},
 		},
 
+		// SHOW TAG KEYS with LIMIT
+		{
+			s: `SHOW TAG KEYS FROM src LIMIT 2`,
+			stmt: &influxql.ShowTagKeysStatement{
+				Sources: []influxql.Source{&influxql.Measurement{Name: "src"}},
+				Limit:   2,
+			},
+		},
+
+		// SHOW TAG KEYS with OFFSET
+		{
+			s: `SHOW TAG KEYS FROM src OFFSET 1`,
+			stmt: &influxql.ShowTagKeysStatement{
+				Sources: []influxql.Source{&influxql.Measurement{Name: "src"}},
+				Offset:  1,
+			},
+		},
+
+		// SHOW TAG KEYS with LIMIT and OFFSET
+		{
+			s: `SHOW TAG KEYS FROM src LIMIT 2 OFFSET 1`,
+			stmt: &influxql.ShowTagKeysStatement{
+				Sources: []influxql.Source{&influxql.Measurement{Name: "src"}},
+				Limit:   2,
+				Offset:  1,
+			},
+		},
+
+		// SHOW TAG KEYS with SLIMIT
+		{
+			s: `SHOW TAG KEYS FROM src SLIMIT 2`,
+			stmt: &influxql.ShowTagKeysStatement{
+				Sources: []influxql.Source{&influxql.Measurement{Name: "src"}},
+				SLimit:  2,
+			},
+		},
+
+		// SHOW TAG KEYS with SOFFSET
+		{
+			s: `SHOW TAG KEYS FROM src SOFFSET 1`,
+			stmt: &influxql.ShowTagKeysStatement{
+				Sources: []influxql.Source{&influxql.Measurement{Name: "src"}},
+				SOffset: 1,
+			},
+		},
+
+		// SHOW TAG KEYS with SLIMIT and SOFFSET
+		{
+			s: `SHOW TAG KEYS FROM src SLIMIT 2 SOFFSET 1`,
+			stmt: &influxql.ShowTagKeysStatement{
+				Sources: []influxql.Source{&influxql.Measurement{Name: "src"}},
+				SLimit:  2,
+				SOffset: 1,
+			},
+		},
+
+		// SHOW TAG KEYS with LIMIT, OFFSET, SLIMIT, and SOFFSET
+		{
+			s: `SHOW TAG KEYS FROM src LIMIT 4 OFFSET 3 SLIMIT 2 SOFFSET 1`,
+			stmt: &influxql.ShowTagKeysStatement{
+				Sources: []influxql.Source{&influxql.Measurement{Name: "src"}},
+				Limit:   4,
+				Offset:  3,
+				SLimit:  2,
+				SOffset: 1,
+			},
+		},
+
 		// SHOW TAG KEYS FROM /<regex>/
 		{
 			s: `SHOW TAG KEYS FROM /[cg]pu/`,
@@ -816,6 +1013,16 @@ func TestParser_ParseStatement(t *testing.T) {
 			},
 		},
 
+		// DROP SERVER statement
+		{
+			s:    `DROP SERVER 123`,
+			stmt: &influxql.DropServerStatement{NodeID: 123},
+		},
+		{
+			s:    `DROP SERVER 123 FORCE`,
+			stmt: &influxql.DropServerStatement{NodeID: 123, Force: true},
+		},
+
 		// SHOW CONTINUOUS QUERIES statement
 		{
 			s:    `SHOW CONTINUOUS QUERIES`,
@@ -830,7 +1037,7 @@ func TestParser_ParseStatement(t *testing.T) {
 				Database: "testdb",
 				Source: &influxql.SelectStatement{
 					Fields:  []*influxql.Field{{Expr: &influxql.Call{Name: "count", Args: []influxql.Expr{&influxql.VarRef{Val: "field1"}}}}},
-					Target:  &influxql.Target{Measurement: &influxql.Measurement{Name: "measure1"}},
+					Target:  &influxql.Target{Measurement: &influxql.Measurement{Name: "measure1", IsTarget: true}},
 					Sources: []influxql.Source{&influxql.Measurement{Name: "myseries"}},
 					Dimensions: []*influxql.Dimension{
 						{
@@ -854,7 +1061,7 @@ func TestParser_ParseStatement(t *testing.T) {
 				Source: &influxql.SelectStatement{
 					IsRawQuery: true,
 					Fields:     []*influxql.Field{{Expr: &influxql.Wildcard{}}},
-					Target:     &influxql.Target{Measurement: &influxql.Measurement{Name: "measure1"}},
+					Target:     &influxql.Target{Measurement: &influxql.Measurement{Name: "measure1", IsTarget: true}},
 					Sources:    []influxql.Source{&influxql.Measurement{Name: "cpu_load_short"}},
 				},
 			},
@@ -869,7 +1076,7 @@ func TestParser_ParseStatement(t *testing.T) {
 				Source: &influxql.SelectStatement{
 					Fields: []*influxql.Field{{Expr: &influxql.Call{Name: "count", Args: []influxql.Expr{&influxql.VarRef{Val: "field1"}}}}},
 					Target: &influxql.Target{
-						Measurement: &influxql.Measurement{RetentionPolicy: "1h.policy1", Name: "cpu.load"},
+						Measurement: &influxql.Measurement{RetentionPolicy: "1h.policy1", Name: "cpu.load", IsTarget: true},
 					},
 					Sources: []influxql.Source{&influxql.Measurement{Name: "myseries"}},
 					Dimensions: []*influxql.Dimension{
@@ -896,7 +1103,7 @@ func TestParser_ParseStatement(t *testing.T) {
 					IsRawQuery: true,
 					Fields:     []*influxql.Field{{Expr: &influxql.VarRef{Val: "value"}}},
 					Target: &influxql.Target{
-						Measurement: &influxql.Measurement{RetentionPolicy: "policy1", Name: "value"},
+						Measurement: &influxql.Measurement{RetentionPolicy: "policy1", Name: "value", IsTarget: true},
 					},
 					Sources: []influxql.Source{&influxql.Measurement{Name: "myseries"}},
 				},
@@ -914,18 +1121,52 @@ func TestParser_ParseStatement(t *testing.T) {
 					Fields: []*influxql.Field{{Expr: &influxql.VarRef{Val: "transmit_rx"}},
 						{Expr: &influxql.VarRef{Val: "transmit_tx"}}},
 					Target: &influxql.Target{
-						Measurement: &influxql.Measurement{RetentionPolicy: "policy1", Name: "network"},
+						Measurement: &influxql.Measurement{RetentionPolicy: "policy1", Name: "network", IsTarget: true},
 					},
 					Sources: []influxql.Source{&influxql.Measurement{Name: "myseries"}},
 				},
 			},
 		},
 
+		// CREATE CONTINUOUS QUERY with backreference measurement name
+		{
+			s: `CREATE CONTINUOUS QUERY myquery ON testdb BEGIN SELECT mean(value) INTO "policy1".:measurement FROM /^[a-z]+.*/ GROUP BY time(1m) END`,
+			stmt: &influxql.CreateContinuousQueryStatement{
+				Name:     "myquery",
+				Database: "testdb",
+				Source: &influxql.SelectStatement{
+					Fields: []*influxql.Field{{Expr: &influxql.Call{Name: "mean", Args: []influxql.Expr{&influxql.VarRef{Val: "value"}}}}},
+					Target: &influxql.Target{
+						Measurement: &influxql.Measurement{RetentionPolicy: "policy1", IsTarget: true},
+					},
+					Sources: []influxql.Source{&influxql.Measurement{Regex: &influxql.RegexLiteral{Val: regexp.MustCompile(`^[a-z]+.*`)}}},
+					Dimensions: []*influxql.Dimension{
+						{
+							Expr: &influxql.Call{
+								Name: "time",
+								Args: []influxql.Expr{
+									&influxql.DurationLiteral{Val: 1 * time.Minute},
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+
 		// CREATE DATABASE statement
 		{
 			s: `CREATE DATABASE testdb`,
 			stmt: &influxql.CreateDatabaseStatement{
-				Name: "testdb",
+				Name:        "testdb",
+				IfNotExists: false,
+			},
+		},
+		{
+			s: `CREATE DATABASE IF NOT EXISTS testdb`,
+			stmt: &influxql.CreateDatabaseStatement{
+				Name:        "testdb",
+				IfNotExists: true,
 			},
 		},
 
@@ -1181,20 +1422,20 @@ func TestParser_ParseStatement(t *testing.T) {
 		{
 			s: `SHOW STATS`,
 			stmt: &influxql.ShowStatsStatement{
-				Host: "",
+				Module: "",
 			},
 		},
 		{
-			s: `SHOW STATS ON 'servera'`,
+			s: `SHOW STATS FOR 'cluster'`,
 			stmt: &influxql.ShowStatsStatement{
-				Host: "servera",
+				Module: "cluster",
 			},
 		},
+
+		// SHOW SHARDS
 		{
-			s: `SHOW STATS ON '192.167.1.44'`,
+			s:    `SHOW SHARDS`,
-			stmt: &influxql.ShowStatsStatement{
+			stmt: &influxql.ShowShardsStatement{},
-				Host: "192.167.1.44",
-			},
 		},
 
 		// SHOW DIAGNOSTICS
@@ -1202,6 +1443,12 @@ func TestParser_ParseStatement(t *testing.T) {
 			s:    `SHOW DIAGNOSTICS`,
 			stmt: &influxql.ShowDiagnosticsStatement{},
 		},
+		{
+			s: `SHOW DIAGNOSTICS FOR 'build'`,
+			stmt: &influxql.ShowDiagnosticsStatement{
+				Module: "build",
+			},
+		},
 
 		// Errors
 		{s: ``, err: `found EOF, expected SELECT, DELETE, SHOW, CREATE, DROP, GRANT, REVOKE, ALTER, SET at line 1, char 1`},
@@ -1213,6 +1460,21 @@ func TestParser_ParseStatement(t *testing.T) {
 		{s: `SELECT field1 FROM myseries GROUP`, err: `found EOF, expected BY at line 1, char 35`},
 		{s: `SELECT field1 FROM myseries LIMIT`, err: `found EOF, expected number at line 1, char 35`},
 		{s: `SELECT field1 FROM myseries LIMIT 10.5`, err: `fractional parts not allowed in LIMIT at line 1, char 35`},
+		{s: `SELECT top() FROM myseries`, err: `invalid number of arguments for top, expected at least 2, got 0`},
+		{s: `SELECT top(field1) FROM myseries`, err: `invalid number of arguments for top, expected at least 2, got 1`},
+		{s: `SELECT top(field1,foo) FROM myseries`, err: `expected integer as last argument in top(), found foo`},
+		{s: `SELECT top(field1,host,'server',foo) FROM myseries`, err: `expected integer as last argument in top(), found foo`},
+		{s: `SELECT top(field1,5,'server',2) FROM myseries`, err: `only fields or tags are allowed in top(), found 5.000`},
+		{s: `SELECT top(field1,max(foo),'server',2) FROM myseries`, err: `only fields or tags are allowed in top(), found max(foo)`},
+		{s: `SELECT bottom() FROM myseries`, err: `invalid number of arguments for bottom, expected at least 2, got 0`},
+		{s: `SELECT bottom(field1) FROM myseries`, err: `invalid number of arguments for bottom, expected at least 2, got 1`},
+		{s: `SELECT bottom(field1,foo) FROM myseries`, err: `expected integer as last argument in bottom(), found foo`},
+		{s: `SELECT bottom(field1,host,'server',foo) FROM myseries`, err: `expected integer as last argument in bottom(), found foo`},
+		{s: `SELECT bottom(field1,5,'server',2) FROM myseries`, err: `only fields or tags are allowed in bottom(), found 5.000`},
+		{s: `SELECT bottom(field1,max(foo),'server',2) FROM myseries`, err: `only fields or tags are allowed in bottom(), found max(foo)`},
+		{s: `SELECT percentile() FROM myseries`, err: `invalid number of arguments for percentile, expected 2, got 0`},
+		{s: `SELECT percentile(field1) FROM myseries`, err: `invalid number of arguments for percentile, expected 2, got 1`},
+		{s: `SELECT percentile(field1, foo) FROM myseries`, err: `expected float argument in percentile()`},
 		{s: `SELECT field1 FROM myseries OFFSET`, err: `found EOF, expected number at line 1, char 36`},
 		{s: `SELECT field1 FROM myseries OFFSET 10.5`, err: `fractional parts not allowed in OFFSET at line 1, char 36`},
 		{s: `SELECT field1 FROM myseries ORDER`, err: `found EOF, expected BY at line 1, char 35`},
@@ -1220,19 +1482,20 @@ func TestParser_ParseStatement(t *testing.T) {
 		{s: `SELECT field1 FROM myseries ORDER BY /`, err: `found /, expected identifier, ASC, DESC at line 1, char 38`},
 		{s: `SELECT field1 FROM myseries ORDER BY 1`, err: `found 1, expected identifier, ASC, DESC at line 1, char 38`},
 		{s: `SELECT field1 FROM myseries ORDER BY time ASC,`, err: `found EOF, expected identifier at line 1, char 47`},
-		{s: `SELECT field1 FROM myseries ORDER BY DESC`, err: `only ORDER BY time ASC supported at this time`},
+		{s: `SELECT field1 FROM myseries ORDER BY time, field1`, err: `only ORDER BY time supported at this time`},
-		{s: `SELECT field1 FROM myseries ORDER BY field1`, err: `only ORDER BY time ASC supported at this time`},
-		{s: `SELECT field1 FROM myseries ORDER BY time DESC`, err: `only ORDER BY time ASC supported at this time`},
-		{s: `SELECT field1 FROM myseries ORDER BY time, field1`, err: `only ORDER BY time ASC supported at this time`},
 		{s: `SELECT field1 AS`, err: `found EOF, expected identifier at line 1, char 18`},
 		{s: `SELECT field1 FROM foo group by time(1s)`, err: `GROUP BY requires at least one aggregate function`},
 		{s: `SELECT count(value), value FROM foo`, err: `mixing aggregate and non-aggregate queries is not supported`},
 		{s: `SELECT count(value) FROM foo group by time(1s)`, err: `aggregate functions with GROUP BY time require a WHERE time clause`},
 		{s: `SELECT count(value) FROM foo group by time(1s) where host = 'hosta.influxdb.org'`, err: `aggregate functions with GROUP BY time require a WHERE time clause`},
+		{s: `SELECT count(value) FROM foo group by time`, err: `time() is a function and expects at least one argument`},
+		{s: `SELECT count(value) FROM foo group by 'time'`, err: `only time and tag dimensions allowed`},
+		{s: `SELECT count(value) FROM foo where time > now() and time < now() group by time()`, err: `time dimension expected one argument`},
+		{s: `SELECT count(value) FROM foo where time > now() and time < now() group by time(b)`, err: `time dimension must have one duration argument`},
+		{s: `SELECT count(value) FROM foo where time > now() and time < now() group by time(1s), time(2s)`, err: `multiple time dimensions not allowed`},
 		{s: `SELECT field1 FROM 12`, err: `found 12, expected identifier at line 1, char 20`},
 		{s: `SELECT 1000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 FROM myseries`, err: `unable to parse number at line 1, char 8`},
 		{s: `SELECT 10.5h FROM myseries`, err: `found h, expected FROM at line 1, char 12`},
-		{s: `SELECT derivative(field1), field1 FROM myseries`, err: `mixing aggregate and non-aggregate queries is not supported`},
 		{s: `SELECT distinct(field1), sum(field1) FROM myseries`, err: `aggregate function distinct() can not be combined with other functions or fields`},
 		{s: `SELECT distinct(field1), field2 FROM myseries`, err: `aggregate function distinct() can not be combined with other functions or fields`},
 		{s: `SELECT distinct(field1, field2) FROM myseries`, err: `distinct function can only have one argument`},
@@ -1243,15 +1506,18 @@ func TestParser_ParseStatement(t *testing.T) {
 		{s: `SELECT count(distinct field1, field2) FROM myseries`, err: `count(distinct <field>) can only have one argument`},
 		{s: `select count(distinct(too, many, arguments)) from myseries`, err: `count(distinct <field>) can only have one argument`},
 		{s: `select count() from myseries`, err: `invalid number of arguments for count, expected 1, got 0`},
+		{s: `SELECT derivative(), field1 FROM myseries`, err: `mixing aggregate and non-aggregate queries is not supported`},
 		{s: `select derivative() from myseries`, err: `invalid number of arguments for derivative, expected at least 1 but no more than 2, got 0`},
 		{s: `select derivative(mean(value), 1h, 3) from myseries`, err: `invalid number of arguments for derivative, expected at least 1 but no more than 2, got 3`},
+		{s: `SELECT derivative(value) FROM myseries group by time(1h)`, err: `aggregate function required inside the call to derivative`},
+		{s: `SELECT non_negative_derivative(), field1 FROM myseries`, err: `mixing aggregate and non-aggregate queries is not supported`},
+		{s: `select non_negative_derivative() from myseries`, err: `invalid number of arguments for non_negative_derivative, expected at least 1 but no more than 2, got 0`},
+		{s: `select non_negative_derivative(mean(value), 1h, 3) from myseries`, err: `invalid number of arguments for non_negative_derivative, expected at least 1 but no more than 2, got 3`},
+		{s: `SELECT non_negative_derivative(value) FROM myseries group by time(1h)`, err: `aggregate function required inside the call to non_negative_derivative`},
 		{s: `SELECT field1 from myseries WHERE host =~ 'asd' LIMIT 1`, err: `found asd, expected regex at line 1, char 42`},
 		{s: `SELECT value > 2 FROM cpu`, err: `invalid operator > in SELECT clause at line 1, char 8; operator is intended for WHERE clause`},
 		{s: `SELECT value = 2 FROM cpu`, err: `invalid operator = in SELECT clause at line 1, char 8; operator is intended for WHERE clause`},
 		{s: `SELECT s =~ /foo/ FROM cpu`, err: `invalid operator =~ in SELECT clause at line 1, char 8; operator is intended for WHERE clause`},
-		{s: `SELECT foo, * from cpu`, err: `wildcards can not be combined with other fields`},
-		{s: `SELECT *, * from cpu`, err: `found ,, expected FROM at line 1, char 9`},
-		{s: `SELECT *, foo from cpu`, err: `found ,, expected FROM at line 1, char 9`},
 		{s: `DELETE`, err: `found EOF, expected FROM at line 1, char 8`},
 		{s: `DELETE FROM`, err: `found EOF, expected identifier at line 1, char 13`},
 		{s: `DELETE FROM myseries WHERE`, err: `found EOF, expected identifier, string, number, bool at line 1, char 28`},
@@ -1259,14 +1525,18 @@ func TestParser_ParseStatement(t *testing.T) {
 		{s: `DROP SERIES`, err: `found EOF, expected FROM, WHERE at line 1, char 13`},
 		{s: `DROP SERIES FROM`, err: `found EOF, expected identifier at line 1, char 18`},
 		{s: `DROP SERIES FROM src WHERE`, err: `found EOF, expected identifier, string, number, bool at line 1, char 28`},
+		{s: `DROP SERVER`, err: `found EOF, expected number at line 1, char 13`},
+		{s: `DROP SERVER abc`, err: `found abc, expected number at line 1, char 13`},
+		{s: `DROP SERVER 1 1`, err: `found 1, expected FORCE at line 1, char 15`},
 		{s: `SHOW CONTINUOUS`, err: `found EOF, expected QUERIES at line 1, char 17`},
 		{s: `SHOW RETENTION`, err: `found EOF, expected POLICIES at line 1, char 16`},
 		{s: `SHOW RETENTION ON`, err: `found ON, expected POLICIES at line 1, char 16`},
 		{s: `SHOW RETENTION POLICIES`, err: `found EOF, expected ON at line 1, char 25`},
 		{s: `SHOW RETENTION POLICIES mydb`, err: `found mydb, expected ON at line 1, char 25`},
 		{s: `SHOW RETENTION POLICIES ON`, err: `found EOF, expected identifier at line 1, char 28`},
-		{s: `SHOW FOO`, err: `found FOO, expected CONTINUOUS, DATABASES, FIELD, GRANTS, MEASUREMENTS, RETENTION, SERIES, SERVERS, TAG, USERS at line 1, char 6`},
+		{s: `SHOW FOO`, err: `found FOO, expected CONTINUOUS, DATABASES, DIAGNOSTICS, FIELD, GRANTS, MEASUREMENTS, RETENTION, SERIES, SERVERS, SHARDS, STATS, TAG, USERS at line 1, char 6`},
-		{s: `SHOW STATS ON`, err: `found EOF, expected string at line 1, char 15`},
+		{s: `SHOW STATS FOR`, err: `found EOF, expected string at line 1, char 16`},
+		{s: `SHOW DIAGNOSTICS FOR`, err: `found EOF, expected string at line 1, char 22`},
 		{s: `SHOW GRANTS`, err: `found EOF, expected FOR at line 1, char 13`},
 		{s: `SHOW GRANTS FOR`, err: `found EOF, expected identifier at line 1, char 17`},
 		{s: `DROP CONTINUOUS`, err: `found EOF, expected QUERY at line 1, char 17`},
@@ -1276,6 +1546,10 @@ func TestParser_ParseStatement(t *testing.T) {
 		{s: `CREATE CONTINUOUS`, err: `found EOF, expected QUERY at line 1, char 19`},
 		{s: `CREATE CONTINUOUS QUERY`, err: `found EOF, expected identifier at line 1, char 25`},
 		{s: `DROP FOO`, err: `found FOO, expected SERIES, CONTINUOUS, MEASUREMENT at line 1, char 6`},
+		{s: `CREATE DATABASE`, err: `found EOF, expected identifier at line 1, char 17`},
+		{s: `CREATE DATABASE IF`, err: `found EOF, expected NOT at line 1, char 20`},
+		{s: `CREATE DATABASE IF NOT`, err: `found EOF, expected EXISTS at line 1, char 24`},
+		{s: `CREATE DATABASE IF NOT EXISTS`, err: `found EOF, expected identifier at line 1, char 31`},
 		{s: `DROP DATABASE`, err: `found EOF, expected identifier at line 1, char 15`},
 		{s: `DROP RETENTION`, err: `found EOF, expected POLICY at line 1, char 16`},
 		{s: `DROP RETENTION POLICY`, err: `found EOF, expected identifier at line 1, char 23`},
@@ -1363,6 +1637,7 @@ func TestParser_ParseStatement(t *testing.T) {
 		{s: `CREATE RETENTION POLICY policy1 ON testdb DURATION 1h REPLICATION 3.14`, err: `number must be an integer at line 1, char 67`},
 		{s: `CREATE RETENTION POLICY policy1 ON testdb DURATION 1h REPLICATION 0`, err: `invalid value 0: must be 1 <= n <= 2147483647 at line 1, char 67`},
 		{s: `CREATE RETENTION POLICY policy1 ON testdb DURATION 1h REPLICATION bad`, err: `found bad, expected number at line 1, char 67`},
+		{s: `CREATE RETENTION POLICY policy1 ON testdb DURATION 1h REPLICATION 1 foo`, err: `found foo, expected DEFAULT at line 1, char 69`},
 		{s: `ALTER`, err: `found EOF, expected RETENTION at line 1, char 7`},
 		{s: `ALTER RETENTION`, err: `found EOF, expected POLICY at line 1, char 17`},
 		{s: `ALTER RETENTION POLICY`, err: `found EOF, expected identifier at line 1, char 24`},
@@ -1396,7 +1671,8 @@ func TestParser_ParseStatement(t *testing.T) {
 		if !reflect.DeepEqual(tt.err, errstring(err)) {
 			t.Errorf("%d. %q: error mismatch:\n  exp=%s\n  got=%s\n\n", i, tt.s, tt.err, err)
 		} else if tt.err == "" && !reflect.DeepEqual(tt.stmt, stmt) {
-			t.Logf("\nexp=%s\ngot=%s\n", mustMarshalJSON(tt.stmt), mustMarshalJSON(stmt))
+			t.Logf("\n# %s\nexp=%s\ngot=%s\n", tt.s, mustMarshalJSON(tt.stmt), mustMarshalJSON(stmt))
+			t.Logf("\nSQL exp=%s\nSQL got=%s\n", tt.stmt.String(), stmt.String())
 			t.Errorf("%d. %q\n\nstmt mismatch:\n\nexp=%#v\n\ngot=%#v\n\n", i, tt.s, tt.stmt, stmt)
 		}
 	}

Godeps/_workspace/src/github.com/influxdb/influxdb/influxql/result.go

@@ -3,8 +3,8 @@ package influxql
 import (
 	"encoding/json"
 	"errors"
-	"hash/fnv"
+
-	"sort"
+	"github.com/influxdb/influxdb/models"
 )
 
 // TagSet is a fundamental concept within the query system. It represents a composite series,
@@ -22,66 +22,13 @@ func (t *TagSet) AddFilter(key string, filter Expr) {
 	t.Filters = append(t.Filters, filter)
 }
 
-// Row represents a single row returned from the execution of a statement.
-type Row struct {
-	Name    string            `json:"name,omitempty"`
-	Tags    map[string]string `json:"tags,omitempty"`
-	Columns []string          `json:"columns,omitempty"`
-	Values  [][]interface{}   `json:"values,omitempty"`
-	Err     error             `json:"err,omitempty"`
-}
-
-// SameSeries returns true if r contains values for the same series as o.
-func (r *Row) SameSeries(o *Row) bool {
-	return r.tagsHash() == o.tagsHash() && r.Name == o.Name
-}
-
-// tagsHash returns a hash of tag key/value pairs.
-func (r *Row) tagsHash() uint64 {
-	h := fnv.New64a()
-	keys := r.tagsKeys()
-	for _, k := range keys {
-		h.Write([]byte(k))
-		h.Write([]byte(r.Tags[k]))
-	}
-	return h.Sum64()
-}
-
-// tagKeys returns a sorted list of tag keys.
-func (r *Row) tagsKeys() []string {
-	a := make([]string, 0, len(r.Tags))
-	for k := range r.Tags {
-		a = append(a, k)
-	}
-	sort.Strings(a)
-	return a
-}
-
 // Rows represents a list of rows that can be sorted consistently by name/tag.
-type Rows []*Row
-
-func (p Rows) Len() int { return len(p) }
-
-func (p Rows) Less(i, j int) bool {
-	// Sort by name first.
-	if p[i].Name != p[j].Name {
-		return p[i].Name < p[j].Name
-	}
-
-	// Sort by tag set hash. Tags don't have a meaningful sort order so we
-	// just compute a hash and sort by that instead. This allows the tests
-	// to receive rows in a predictable order every time.
-	return p[i].tagsHash() < p[j].tagsHash()
-}
-
-func (p Rows) Swap(i, j int) { p[i], p[j] = p[j], p[i] }
-
 // Result represents a resultset returned from a single statement.
 type Result struct {
 	// StatementID is just the statement's position in the query. It's used
 	// to combine statement results if they're being buffered in memory.
 	StatementID int `json:"-"`
-	Series      Rows
+	Series      models.Rows
 	Err         error
 }
 
@@ -89,8 +36,8 @@ type Result struct {
 func (r *Result) MarshalJSON() ([]byte, error) {
 	// Define a struct that outputs "error" as a string.
 	var o struct {
-		Series []*Row `json:"series,omitempty"`
+		Series []*models.Row `json:"series,omitempty"`
-		Err    string `json:"error,omitempty"`
+		Err    string        `json:"error,omitempty"`
 	}
 
 	// Copy fields to output struct.
@@ -105,8 +52,8 @@ func (r *Result) MarshalJSON() ([]byte, error) {
 // UnmarshalJSON decodes the data into the Result struct
 func (r *Result) UnmarshalJSON(b []byte) error {
 	var o struct {
-		Series []*Row `json:"series,omitempty"`
+		Series []*models.Row `json:"series,omitempty"`
-		Err    string `json:"error,omitempty"`
+		Err    string        `json:"error,omitempty"`
 	}
 
 	err := json.Unmarshal(b, &o)

Godeps/_workspace/src/github.com/influxdb/influxdb/influxql/scanner.go

@@ -95,6 +95,8 @@ func (s *Scanner) Scan() (tok Token, pos Pos, lit string) {
 		return COMMA, pos, ""
 	case ';':
 		return SEMICOLON, pos, ""
+	case ':':
+		return COLON, pos, ""
 	}
 
 	return ILLEGAL, pos, string(ch0)

Godeps/_workspace/src/github.com/influxdb/influxdb/influxql/scanner_test.go

@@ -136,8 +136,10 @@ func TestScanner_Scan(t *testing.T) {
 		{s: `KEYS`, tok: influxql.KEYS},
 		{s: `LIMIT`, tok: influxql.LIMIT},
 		{s: `SHOW`, tok: influxql.SHOW},
+		{s: `SHARDS`, tok: influxql.SHARDS},
 		{s: `MEASUREMENT`, tok: influxql.MEASUREMENT},
 		{s: `MEASUREMENTS`, tok: influxql.MEASUREMENTS},
+		{s: `NOT`, tok: influxql.NOT},
 		{s: `OFFSET`, tok: influxql.OFFSET},
 		{s: `ON`, tok: influxql.ON},
 		{s: `ORDER`, tok: influxql.ORDER},
@@ -152,6 +154,8 @@ func TestScanner_Scan(t *testing.T) {
 		{s: `REVOKE`, tok: influxql.REVOKE},
 		{s: `SELECT`, tok: influxql.SELECT},
 		{s: `SERIES`, tok: influxql.SERIES},
+		{s: `SERVER`, tok: influxql.SERVER},
+		{s: `SERVERS`, tok: influxql.SERVERS},
 		{s: `TAG`, tok: influxql.TAG},
 		{s: `TO`, tok: influxql.TO},
 		{s: `USER`, tok: influxql.USER},

Godeps/_workspace/src/github.com/influxdb/influxdb/influxql/token.go

@@ -50,6 +50,7 @@ const (
 	LPAREN    // (
 	RPAREN    // )
 	COMMA     // ,
+	COLON     // :
 	SEMICOLON // ;
 	DOT       // .
 
@@ -76,6 +77,7 @@ const (
 	EXPLAIN
 	FIELD
 	FOR
+	FORCE
 	FROM
 	GRANT
 	GRANTS
@@ -91,6 +93,7 @@ const (
 	LIMIT
 	MEASUREMENT
 	MEASUREMENTS
+	NOT
 	OFFSET
 	ON
 	ORDER
@@ -106,9 +109,11 @@ const (
 	REVOKE
 	SELECT
 	SERIES
+	SERVER
 	SERVERS
 	SET
 	SHOW
+	SHARDS
 	SLIMIT
 	STATS
 	DIAGNOSTICS
@@ -159,6 +164,7 @@ var tokens = [...]string{
 	LPAREN:    "(",
 	RPAREN:    ")",
 	COMMA:     ",",
+	COLON:     ":",
 	SEMICOLON: ";",
 	DOT:       ".",
 
@@ -183,6 +189,7 @@ var tokens = [...]string{
 	EXPLAIN:      "EXPLAIN",
 	FIELD:        "FIELD",
 	FOR:          "FOR",
+	FORCE:        "FORCE",
 	FROM:         "FROM",
 	GRANT:        "GRANT",
 	GRANTS:       "GRANTS",
@@ -198,6 +205,7 @@ var tokens = [...]string{
 	LIMIT:        "LIMIT",
 	MEASUREMENT:  "MEASUREMENT",
 	MEASUREMENTS: "MEASUREMENTS",
+	NOT:          "NOT",
 	OFFSET:       "OFFSET",
 	ON:           "ON",
 	ORDER:        "ORDER",
@@ -213,9 +221,11 @@ var tokens = [...]string{
 	REVOKE:       "REVOKE",
 	SELECT:       "SELECT",
 	SERIES:       "SERIES",
+	SERVER:       "SERVER",
 	SERVERS:      "SERVERS",
 	SET:          "SET",
 	SHOW:         "SHOW",
+	SHARDS:       "SHARDS",
 	SLIMIT:       "SLIMIT",
 	SOFFSET:      "SOFFSET",
 	STATS:        "STATS",

Godeps/_workspace/src/github.com/influxdb/influxdb/influxvar.go

@@ -0,0 +1,45 @@
+package influxdb
+
+import (
+	"expvar"
+	"sync"
+)
+
+var expvarMu sync.Mutex
+
+// NewStatistics returns an expvar-based map with the given key. Within that map
+// is another map. Within there "name" is the Measurement name, "tags" are the tags,
+// and values are placed at the key "values".
+func NewStatistics(key, name string, tags map[string]string) *expvar.Map {
+	expvarMu.Lock()
+	defer expvarMu.Unlock()
+
+	// Add expvar for this service.
+	var v expvar.Var
+	if v = expvar.Get(key); v == nil {
+		v = expvar.NewMap(key)
+	}
+	m := v.(*expvar.Map)
+
+	// Set the name
+	nameVar := &expvar.String{}
+	nameVar.Set(name)
+	m.Set("name", nameVar)
+
+	// Set the tags
+	tagsVar := &expvar.Map{}
+	tagsVar.Init()
+	for k, v := range tags {
+		value := &expvar.String{}
+		value.Set(v)
+		tagsVar.Set(k, value)
+	}
+	m.Set("tags", tagsVar)
+
+	// Create and set the values entry used for actual stats.
+	statMap := &expvar.Map{}
+	statMap.Init()
+	m.Set("values", statMap)
+
+	return statMap
+}

Godeps/_workspace/src/github.com/influxdb/influxdb/meta/data.go

@@ -74,14 +74,71 @@ func (data *Data) CreateNode(host string) error {
 }
 
 // DeleteNode removes a node from the metadata.
-func (data *Data) DeleteNode(id uint64) error {
+func (data *Data) DeleteNode(id uint64, force bool) error {
-	for i := range data.Nodes {
+	// Node has to be larger than 0 to be real
-		if data.Nodes[i].ID == id {
+	if id == 0 {
-			data.Nodes = append(data.Nodes[:i], data.Nodes[i+1:]...)
+		return ErrNodeIDRequired
-			return nil
+	}
+	// Is this a valid node?
+	nodeInfo := data.Node(id)
+	if nodeInfo == nil {
+		return ErrNodeNotFound
+	}
+
+	// Am I the only node?  If so, nothing to do
+	if len(data.Nodes) == 1 {
+		return ErrNodeUnableToDropFinalNode
+	}
+
+	// Determine if there are any any non-replicated nodes and force was not specified
+	if !force {
+		for _, d := range data.Databases {
+			for _, rp := range d.RetentionPolicies {
+				// ignore replicated retention policies
+				if rp.ReplicaN > 1 {
+					continue
+				}
+				for _, sg := range rp.ShardGroups {
+					for _, s := range sg.Shards {
+						if s.OwnedBy(id) && len(s.Owners) == 1 {
+							return ErrShardNotReplicated
+						}
+					}
+				}
+			}
 		}
 	}
-	return ErrNodeNotFound
+
+	// Remove node id from all shard infos
+	for di, d := range data.Databases {
+		for ri, rp := range d.RetentionPolicies {
+			for sgi, sg := range rp.ShardGroups {
+				for si, s := range sg.Shards {
+					if s.OwnedBy(id) {
+						var owners []ShardOwner
+						for _, o := range s.Owners {
+							if o.NodeID != id {
+								owners = append(owners, o)
+							}
+						}
+						data.Databases[di].RetentionPolicies[ri].ShardGroups[sgi].Shards[si].Owners = owners
+					}
+				}
+			}
+		}
+	}
+
+	// Remove this node from the in memory nodes
+	var nodes []NodeInfo
+	for _, n := range data.Nodes {
+		if n.ID == id {
+			continue
+		}
+		nodes = append(nodes, n)
+	}
+	data.Nodes = nodes
+
+	return nil
 }
 
 // Database returns a database by name.
@@ -132,7 +189,7 @@ func (data *Data) RetentionPolicy(database, name string) (*RetentionPolicyInfo,
 			return &di.RetentionPolicies[i], nil
 		}
 	}
-	return nil, ErrRetentionPolicyNotFound
+	return nil, nil
 }
 
 // CreateRetentionPolicy creates a new retention policy on a database.
@@ -172,6 +229,11 @@ func (data *Data) DropRetentionPolicy(database, name string) error {
 		return ErrDatabaseNotFound
 	}
 
+	// Prohibit dropping the default retention policy.
+	if di.DefaultRetentionPolicy == name {
+		return ErrRetentionPolicyDefault
+	}
+
 	// Remove from list.
 	for i := range di.RetentionPolicies {
 		if di.RetentionPolicies[i].Name == name {
@@ -273,7 +335,6 @@ func (data *Data) ShardGroupsByTimeRange(database, policy string, tmin, tmax tim
 		}
 		groups = append(groups, g)
 	}
-	sort.Sort(ShardGroupInfos(groups))
 	return groups, nil
 }
 
@@ -344,13 +405,16 @@ func (data *Data) CreateShardGroup(database, policy string, timestamp time.Time)
 		si := &sgi.Shards[i]
 		for j := 0; j < replicaN; j++ {
 			nodeID := data.Nodes[nodeIndex%len(data.Nodes)].ID
-			si.OwnerIDs = append(si.OwnerIDs, nodeID)
+			si.Owners = append(si.Owners, ShardOwner{NodeID: nodeID})
 			nodeIndex++
 		}
 	}
 
-	// Retention policy has a new shard group, so update the policy.
+	// Retention policy has a new shard group, so update the policy. Shard
+	// Groups must be stored in sorted order, as other parts of the system
+	// assume this to be the case.
 	rpi.ShardGroups = append(rpi.ShardGroups, sgi)
+	sort.Sort(ShardGroupInfos(rpi.ShardGroups))
 
 	return nil
 }
@@ -662,6 +726,31 @@ func (di DatabaseInfo) RetentionPolicy(name string) *RetentionPolicyInfo {
 	return nil
 }
 
+// ShardInfos returns a list of all shards' info for the database.
+func (di DatabaseInfo) ShardInfos() []ShardInfo {
+	shards := map[uint64]*ShardInfo{}
+	for i := range di.RetentionPolicies {
+		for j := range di.RetentionPolicies[i].ShardGroups {
+			sg := di.RetentionPolicies[i].ShardGroups[j]
+			// Skip deleted shard groups
+			if sg.Deleted() {
+				continue
+			}
+			for k := range sg.Shards {
+				si := &di.RetentionPolicies[i].ShardGroups[j].Shards[k]
+				shards[si.ID] = si
+			}
+		}
+	}
+
+	infos := make([]ShardInfo, 0, len(shards))
+	for _, info := range shards {
+		infos = append(infos, *info)
+	}
+
+	return infos
+}
+
 // clone returns a deep copy of di.
 func (di DatabaseInfo) clone() DatabaseInfo {
 	other := di
@@ -917,14 +1006,14 @@ func (sgi *ShardGroupInfo) unmarshal(pb *internal.ShardGroupInfo) {
 
 // ShardInfo represents metadata about a shard.
 type ShardInfo struct {
-	ID       uint64
+	ID     uint64
-	OwnerIDs []uint64
+	Owners []ShardOwner
 }
 
 // OwnedBy returns whether the shard's owner IDs includes nodeID.
 func (si ShardInfo) OwnedBy(nodeID uint64) bool {
-	for _, id := range si.OwnerIDs {
+	for _, so := range si.Owners {
-		if id == nodeID {
+		if so.NodeID == nodeID {
 			return true
 		}
 	}
@@ -935,9 +1024,11 @@ func (si ShardInfo) OwnedBy(nodeID uint64) bool {
 func (si ShardInfo) clone() ShardInfo {
 	other := si
 
-	if si.OwnerIDs != nil {
+	if si.Owners != nil {
-		other.OwnerIDs = make([]uint64, len(si.OwnerIDs))
+		other.Owners = make([]ShardOwner, len(si.Owners))
-		copy(other.OwnerIDs, si.OwnerIDs)
+		for i := range si.Owners {
+			other.Owners[i] = si.Owners[i].clone()
+		}
 	}
 
 	return other
@@ -949,17 +1040,64 @@ func (si ShardInfo) marshal() *internal.ShardInfo {
 		ID: proto.Uint64(si.ID),
 	}
 
-	pb.OwnerIDs = make([]uint64, len(si.OwnerIDs))
+	pb.Owners = make([]*internal.ShardOwner, len(si.Owners))
-	copy(pb.OwnerIDs, si.OwnerIDs)
+	for i := range si.Owners {
+		pb.Owners[i] = si.Owners[i].marshal()
+	}
 
 	return pb
 }
 
+// UnmarshalBinary decodes the object from a binary format.
+func (si *ShardInfo) UnmarshalBinary(buf []byte) error {
+	var pb internal.ShardInfo
+	if err := proto.Unmarshal(buf, &pb); err != nil {
+		return err
+	}
+	si.unmarshal(&pb)
+	return nil
+}
+
 // unmarshal deserializes from a protobuf representation.
 func (si *ShardInfo) unmarshal(pb *internal.ShardInfo) {
 	si.ID = pb.GetID()
-	si.OwnerIDs = make([]uint64, len(pb.GetOwnerIDs()))
+
-	copy(si.OwnerIDs, pb.GetOwnerIDs())
+	// If deprecated "OwnerIDs" exists then convert it to "Owners" format.
+	if len(pb.GetOwnerIDs()) > 0 {
+		si.Owners = make([]ShardOwner, len(pb.GetOwnerIDs()))
+		for i, x := range pb.GetOwnerIDs() {
+			si.Owners[i].unmarshal(&internal.ShardOwner{
+				NodeID: proto.Uint64(x),
+			})
+		}
+	} else if len(pb.GetOwners()) > 0 {
+		si.Owners = make([]ShardOwner, len(pb.GetOwners()))
+		for i, x := range pb.GetOwners() {
+			si.Owners[i].unmarshal(x)
+		}
+	}
+}
+
+// ShardOwner represents a node that owns a shard.
+type ShardOwner struct {
+	NodeID uint64
+}
+
+// clone returns a deep copy of so.
+func (so ShardOwner) clone() ShardOwner {
+	return so
+}
+
+// marshal serializes to a protobuf representation.
+func (so ShardOwner) marshal() *internal.ShardOwner {
+	return &internal.ShardOwner{
+		NodeID: proto.Uint64(so.NodeID),
+	}
+}
+
+// unmarshal deserializes from a protobuf representation.
+func (so *ShardOwner) unmarshal(pb *internal.ShardOwner) {
+	so.NodeID = pb.GetNodeID()
 }
 
 // ContinuousQueryInfo represents metadata about a continuous query.

Godeps/_workspace/src/github.com/influxdb/influxdb/meta/data_test.go

@@ -9,8 +9,10 @@ import (
 	"time"
 
 	"github.com/davecgh/go-spew/spew"
+	"github.com/gogo/protobuf/proto"
 	"github.com/influxdb/influxdb/influxql"
 	"github.com/influxdb/influxdb/meta"
+	"github.com/influxdb/influxdb/meta/internal"
 )
 
 // Ensure a node can be created.
@@ -24,7 +26,7 @@ func TestData_CreateNode(t *testing.T) {
 }
 
 // Ensure a node can be removed.
-func TestData_DeleteNode(t *testing.T) {
+func TestData_DeleteNode_Basic(t *testing.T) {
 	var data meta.Data
 	if err := data.CreateNode("host0"); err != nil {
 		t.Fatal(err)
@@ -34,7 +36,7 @@ func TestData_DeleteNode(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	if err := data.DeleteNode(1); err != nil {
+	if err := data.DeleteNode(1, false); err != nil {
 		t.Fatal(err)
 	} else if len(data.Nodes) != 2 {
 		t.Fatalf("unexpected node count: %d", len(data.Nodes))
@@ -45,6 +47,49 @@ func TestData_DeleteNode(t *testing.T) {
 	}
 }
 
+// Ensure a node can be removed with shard info in play
+func TestData_DeleteNode_Shards(t *testing.T) {
+	var data meta.Data
+	if err := data.CreateNode("host0"); err != nil {
+		t.Fatal(err)
+	} else if err = data.CreateNode("host1"); err != nil {
+		t.Fatal(err)
+	} else if err := data.CreateNode("host2"); err != nil {
+		t.Fatal(err)
+	} else if err := data.CreateNode("host3"); err != nil {
+		t.Fatal(err)
+	}
+
+	if err := data.CreateDatabase("mydb"); err != nil {
+		t.Fatal(err)
+	}
+
+	rpi := &meta.RetentionPolicyInfo{
+		Name:     "myrp",
+		ReplicaN: 3,
+	}
+	if err := data.CreateRetentionPolicy("mydb", rpi); err != nil {
+		t.Fatal(err)
+	}
+	if err := data.CreateShardGroup("mydb", "myrp", time.Now()); err != nil {
+		t.Fatal(err)
+	}
+	if len(data.Databases[0].RetentionPolicies[0].ShardGroups[0].Shards[0].Owners) != 3 {
+		t.Fatal("wrong number of shard owners")
+	}
+	if err := data.DeleteNode(2, false); err != nil {
+		t.Fatal(err)
+	}
+	if got, exp := len(data.Databases[0].RetentionPolicies[0].ShardGroups[0].Shards[0].Owners), 2; exp != got {
+		t.Fatalf("wrong number of shard owners, got %d, exp %d", got, exp)
+	}
+	for _, s := range data.Databases[0].RetentionPolicies[0].ShardGroups[0].Shards {
+		if s.OwnedBy(2) {
+			t.Fatal("shard still owned by delted node")
+		}
+	}
+}
+
 // Ensure a database can be created.
 func TestData_CreateDatabase(t *testing.T) {
 	var data meta.Data
@@ -299,7 +344,13 @@ func TestData_CreateShardGroup(t *testing.T) {
 		StartTime: time.Date(2000, time.January, 1, 0, 0, 0, 0, time.UTC),
 		EndTime:   time.Date(2000, time.January, 1, 1, 0, 0, 0, time.UTC),
 		Shards: []meta.ShardInfo{
-			{ID: 1, OwnerIDs: []uint64{1, 2}},
+			{
+				ID: 1,
+				Owners: []meta.ShardOwner{
+					{NodeID: 1},
+					{NodeID: 2},
+				},
+			},
 		},
 	}) {
 		t.Fatalf("unexpected shard group: %#v", sgi)
@@ -570,8 +621,12 @@ func TestData_Clone(t *testing.T) {
 								EndTime:   time.Date(2000, time.February, 1, 0, 0, 0, 0, time.UTC),
 								Shards: []meta.ShardInfo{
 									{
-										ID:       200,
+										ID: 200,
-										OwnerIDs: []uint64{1, 3, 4},
+										Owners: []meta.ShardOwner{
+											{NodeID: 1},
+											{NodeID: 3},
+											{NodeID: 4},
+										},
 									},
 								},
 							},
@@ -605,8 +660,8 @@ func TestData_Clone(t *testing.T) {
 	}
 
 	// Ensure that changing data in the clone does not affect the original.
-	other.Databases[0].RetentionPolicies[0].ShardGroups[0].Shards[0].OwnerIDs[1] = 9
+	other.Databases[0].RetentionPolicies[0].ShardGroups[0].Shards[0].Owners[1].NodeID = 9
-	if v := data.Databases[0].RetentionPolicies[0].ShardGroups[0].Shards[0].OwnerIDs[1]; v != 3 {
+	if v := data.Databases[0].RetentionPolicies[0].ShardGroups[0].Shards[0].Owners[1].NodeID; v != 3 {
 		t.Fatalf("editing clone changed original: %v", v)
 	}
 }
@@ -637,8 +692,12 @@ func TestData_MarshalBinary(t *testing.T) {
 								EndTime:   time.Date(2000, time.February, 1, 0, 0, 0, 0, time.UTC),
 								Shards: []meta.ShardInfo{
 									{
-										ID:       200,
+										ID: 200,
-										OwnerIDs: []uint64{1, 3, 4},
+										Owners: []meta.ShardOwner{
+											{NodeID: 1},
+											{NodeID: 3},
+											{NodeID: 4},
+										},
 									},
 								},
 							},
@@ -682,3 +741,33 @@ func TestData_MarshalBinary(t *testing.T) {
 		t.Fatalf("unexpected users: %#v", other.Users)
 	}
 }
+
+// Ensure shards with deprecated "OwnerIDs" can be decoded.
+func TestShardInfo_UnmarshalBinary_OwnerIDs(t *testing.T) {
+	// Encode deprecated form to bytes.
+	buf, err := proto.Marshal(&internal.ShardInfo{
+		ID:       proto.Uint64(1),
+		OwnerIDs: []uint64{10, 20, 30},
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Decode deprecated form.
+	var si meta.ShardInfo
+	if err := si.UnmarshalBinary(buf); err != nil {
+		t.Fatal(err)
+	}
+
+	// Verify data is migrated correctly.
+	if !reflect.DeepEqual(si, meta.ShardInfo{
+		ID: 1,
+		Owners: []meta.ShardOwner{
+			{NodeID: 10},
+			{NodeID: 20},
+			{NodeID: 30},
+		},
+	}) {
+		t.Fatalf("unexpected shard info: %s", spew.Sdump(si))
+	}
+}

Godeps/_workspace/src/github.com/influxdb/influxdb/meta/errors.go

@@ -7,102 +7,114 @@ import (
 
 var (
 	// ErrStoreOpen is returned when opening an already open store.
-	ErrStoreOpen = errors.New("store already open")
+	ErrStoreOpen = newError("store already open")
 
 	// ErrStoreClosed is returned when closing an already closed store.
-	ErrStoreClosed = errors.New("raft store already closed")
+	ErrStoreClosed = newError("raft store already closed")
 
 	// ErrTooManyPeers is returned when more than 3 peers are used.
-	ErrTooManyPeers = errors.New("too many peers; influxdb v0.9.0 is limited to 3 nodes in a cluster")
+	ErrTooManyPeers = newError("too many peers; influxdb v0.9.0 is limited to 3 nodes in a cluster")
 )
 
 var (
 	// ErrNodeExists is returned when creating an already existing node.
-	ErrNodeExists = errors.New("node already exists")
+	ErrNodeExists = newError("node already exists")
 
 	// ErrNodeNotFound is returned when mutating a node that doesn't exist.
-	ErrNodeNotFound = errors.New("node not found")
+	ErrNodeNotFound = newError("node not found")
 
 	// ErrNodesRequired is returned when at least one node is required for an operation.
 	// This occurs when creating a shard group.
-	ErrNodesRequired = errors.New("at least one node required")
+	ErrNodesRequired = newError("at least one node required")
+
+	// ErrNodeIDRequired is returned when using a zero node id.
+	ErrNodeIDRequired = newError("node id must be greater than 0")
+
+	// ErrNodeUnableToDropSingleNode is returned if the node being dropped is the last
+	// node in the cluster
+	ErrNodeUnableToDropFinalNode = newError("unable to drop the final node in a cluster")
+
+	// ErrNodeRaft is returned when attempting an operation prohibted for a Raft-node.
+	ErrNodeRaft = newError("node is a Raft node")
 )
 
 var (
 	// ErrDatabaseExists is returned when creating an already existing database.
-	ErrDatabaseExists = errors.New("database already exists")
+	ErrDatabaseExists = newError("database already exists")
 
 	// ErrDatabaseNotFound is returned when mutating a database that doesn't exist.
-	ErrDatabaseNotFound = errors.New("database not found")
+	ErrDatabaseNotFound = newError("database not found")
 
 	// ErrDatabaseNameRequired is returned when creating a database without a name.
-	ErrDatabaseNameRequired = errors.New("database name required")
+	ErrDatabaseNameRequired = newError("database name required")
 )
 
 var (
 	// ErrRetentionPolicyExists is returned when creating an already existing policy.
-	ErrRetentionPolicyExists = errors.New("retention policy already exists")
+	ErrRetentionPolicyExists = newError("retention policy already exists")
+
+	// ErrRetentionPolicyDefault is returned when attempting a prohibited operation
+	// on a default retention policy.
+	ErrRetentionPolicyDefault = newError("retention policy is default")
 
 	// ErrRetentionPolicyNotFound is returned when mutating a policy that doesn't exist.
-	ErrRetentionPolicyNotFound = errors.New("retention policy not found")
+	ErrRetentionPolicyNotFound = newError("retention policy not found")
 
 	// ErrRetentionPolicyNameRequired is returned when creating a policy without a name.
-	ErrRetentionPolicyNameRequired = errors.New("retention policy name required")
+	ErrRetentionPolicyNameRequired = newError("retention policy name required")
 
 	// ErrRetentionPolicyNameExists is returned when renaming a policy to
 	// the same name as another existing policy.
-	ErrRetentionPolicyNameExists = errors.New("retention policy name already exists")
+	ErrRetentionPolicyNameExists = newError("retention policy name already exists")
 
 	// ErrRetentionPolicyDurationTooLow is returned when updating a retention
 	// policy that has a duration lower than the allowed minimum.
-	ErrRetentionPolicyDurationTooLow = errors.New(fmt.Sprintf("retention policy duration must be at least %s",
+	ErrRetentionPolicyDurationTooLow = newError(fmt.Sprintf("retention policy duration must be at least %s",
 		RetentionPolicyMinDuration))
 
 	// ErrReplicationFactorTooLow is returned when the replication factor is not in an
 	// acceptable range.
-	ErrReplicationFactorTooLow = errors.New("replication factor must be greater than 0")
+	ErrReplicationFactorTooLow = newError("replication factor must be greater than 0")
 )
 
 var (
 	// ErrShardGroupExists is returned when creating an already existing shard group.
-	ErrShardGroupExists = errors.New("shard group already exists")
+	ErrShardGroupExists = newError("shard group already exists")
 
 	// ErrShardGroupNotFound is returned when mutating a shard group that doesn't exist.
-	ErrShardGroupNotFound = errors.New("shard group not found")
+	ErrShardGroupNotFound = newError("shard group not found")
+
+	// ErrShardNotReplicated is returned if the node requested to be dropped has
+	// the last copy of a shard present and the force keyword was not used
+	ErrShardNotReplicated = newError("shard not replicated")
 )
 
 var (
 	// ErrContinuousQueryExists is returned when creating an already existing continuous query.
-	ErrContinuousQueryExists = errors.New("continuous query already exists")
+	ErrContinuousQueryExists = newError("continuous query already exists")
 
 	// ErrContinuousQueryNotFound is returned when removing a continuous query that doesn't exist.
-	ErrContinuousQueryNotFound = errors.New("continuous query not found")
+	ErrContinuousQueryNotFound = newError("continuous query not found")
 )
 
 var (
 	// ErrUserExists is returned when creating an already existing user.
-	ErrUserExists = errors.New("user already exists")
+	ErrUserExists = newError("user already exists")
 
 	// ErrUserNotFound is returned when mutating a user that doesn't exist.
-	ErrUserNotFound = errors.New("user not found")
+	ErrUserNotFound = newError("user not found")
 
 	// ErrUsernameRequired is returned when creating a user without a username.
-	ErrUsernameRequired = errors.New("username required")
+	ErrUsernameRequired = newError("username required")
 )
 
-var errs = [...]error{
-	ErrStoreOpen, ErrStoreClosed,
-	ErrNodeExists, ErrNodeNotFound,
-	ErrDatabaseExists, ErrDatabaseNotFound, ErrDatabaseNameRequired,
-}
-
 // errLookup stores a mapping of error strings to well defined error types.
 var errLookup = make(map[string]error)
 
-func init() {
+func newError(msg string) error {
-	for _, err := range errs {
+	err := errors.New(msg)
-		errLookup[err.Error()] = err
+	errLookup[err.Error()] = err
-	}
+	return err
 }
 
 // lookupError returns a known error reference, if one exists.

Godeps/_workspace/src/github.com/influxdb/influxdb/meta/internal/meta.pb.go

@@ -15,6 +15,7 @@ It has these top-level messages:
 	RetentionPolicyInfo
 	ShardGroupInfo
 	ShardInfo
+	ShardOwner
 	ContinuousQueryInfo
 	UserInfo
 	UserPrivilege
@@ -49,10 +50,12 @@ It has these top-level messages:
 package internal
 
 import proto "github.com/gogo/protobuf/proto"
+import fmt "fmt"
 import math "math"
 
 // Reference imports to suppress errors if they are not otherwise used.
 var _ = proto.Marshal
+var _ = fmt.Errorf
 var _ = math.Inf
 
 type RPCType int32
@@ -176,15 +179,15 @@ func (x *Command_Type) UnmarshalJSON(data []byte) error {
 }
 
 type Data struct {
-	Term             *uint64         `protobuf:"varint,1,req" json:"Term,omitempty"`
+	Term             *uint64         `protobuf:"varint,1,req,name=Term" json:"Term,omitempty"`
-	Index            *uint64         `protobuf:"varint,2,req" json:"Index,omitempty"`
+	Index            *uint64         `protobuf:"varint,2,req,name=Index" json:"Index,omitempty"`
-	ClusterID        *uint64         `protobuf:"varint,3,req" json:"ClusterID,omitempty"`
+	ClusterID        *uint64         `protobuf:"varint,3,req,name=ClusterID" json:"ClusterID,omitempty"`
-	Nodes            []*NodeInfo     `protobuf:"bytes,4,rep" json:"Nodes,omitempty"`
+	Nodes            []*NodeInfo     `protobuf:"bytes,4,rep,name=Nodes" json:"Nodes,omitempty"`
-	Databases        []*DatabaseInfo `protobuf:"bytes,5,rep" json:"Databases,omitempty"`
+	Databases        []*DatabaseInfo `protobuf:"bytes,5,rep,name=Databases" json:"Databases,omitempty"`
-	Users            []*UserInfo     `protobuf:"bytes,6,rep" json:"Users,omitempty"`
+	Users            []*UserInfo     `protobuf:"bytes,6,rep,name=Users" json:"Users,omitempty"`
-	MaxNodeID        *uint64         `protobuf:"varint,7,req" json:"MaxNodeID,omitempty"`
+	MaxNodeID        *uint64         `protobuf:"varint,7,req,name=MaxNodeID" json:"MaxNodeID,omitempty"`
-	MaxShardGroupID  *uint64         `protobuf:"varint,8,req" json:"MaxShardGroupID,omitempty"`
+	MaxShardGroupID  *uint64         `protobuf:"varint,8,req,name=MaxShardGroupID" json:"MaxShardGroupID,omitempty"`
-	MaxShardID       *uint64         `protobuf:"varint,9,req" json:"MaxShardID,omitempty"`
+	MaxShardID       *uint64         `protobuf:"varint,9,req,name=MaxShardID" json:"MaxShardID,omitempty"`
 	XXX_unrecognized []byte          `json:"-"`
 }
 
@@ -256,8 +259,8 @@ func (m *Data) GetMaxShardID() uint64 {
 }
 
 type NodeInfo struct {
-	ID               *uint64 `protobuf:"varint,1,req" json:"ID,omitempty"`
+	ID               *uint64 `protobuf:"varint,1,req,name=ID" json:"ID,omitempty"`
-	Host             *string `protobuf:"bytes,2,req" json:"Host,omitempty"`
+	Host             *string `protobuf:"bytes,2,req,name=Host" json:"Host,omitempty"`
 	XXX_unrecognized []byte  `json:"-"`
 }
 
@@ -280,10 +283,10 @@ func (m *NodeInfo) GetHost() string {
 }
 
 type DatabaseInfo struct {
-	Name                   *string                `protobuf:"bytes,1,req" json:"Name,omitempty"`
+	Name                   *string                `protobuf:"bytes,1,req,name=Name" json:"Name,omitempty"`
-	DefaultRetentionPolicy *string                `protobuf:"bytes,2,req" json:"DefaultRetentionPolicy,omitempty"`
+	DefaultRetentionPolicy *string                `protobuf:"bytes,2,req,name=DefaultRetentionPolicy" json:"DefaultRetentionPolicy,omitempty"`
-	RetentionPolicies      []*RetentionPolicyInfo `protobuf:"bytes,3,rep" json:"RetentionPolicies,omitempty"`
+	RetentionPolicies      []*RetentionPolicyInfo `protobuf:"bytes,3,rep,name=RetentionPolicies" json:"RetentionPolicies,omitempty"`
-	ContinuousQueries      []*ContinuousQueryInfo `protobuf:"bytes,4,rep" json:"ContinuousQueries,omitempty"`
+	ContinuousQueries      []*ContinuousQueryInfo `protobuf:"bytes,4,rep,name=ContinuousQueries" json:"ContinuousQueries,omitempty"`
 	XXX_unrecognized       []byte                 `json:"-"`
 }
 
@@ -320,11 +323,11 @@ func (m *DatabaseInfo) GetContinuousQueries() []*ContinuousQueryInfo {
 }
 
 type RetentionPolicyInfo struct {
-	Name               *string           `protobuf:"bytes,1,req" json:"Name,omitempty"`
+	Name               *string           `protobuf:"bytes,1,req,name=Name" json:"Name,omitempty"`
-	Duration           *int64            `protobuf:"varint,2,req" json:"Duration,omitempty"`
+	Duration           *int64            `protobuf:"varint,2,req,name=Duration" json:"Duration,omitempty"`
-	ShardGroupDuration *int64            `protobuf:"varint,3,req" json:"ShardGroupDuration,omitempty"`
+	ShardGroupDuration *int64            `protobuf:"varint,3,req,name=ShardGroupDuration" json:"ShardGroupDuration,omitempty"`
-	ReplicaN           *uint32           `protobuf:"varint,4,req" json:"ReplicaN,omitempty"`
+	ReplicaN           *uint32           `protobuf:"varint,4,req,name=ReplicaN" json:"ReplicaN,omitempty"`
-	ShardGroups        []*ShardGroupInfo `protobuf:"bytes,5,rep" json:"ShardGroups,omitempty"`
+	ShardGroups        []*ShardGroupInfo `protobuf:"bytes,5,rep,name=ShardGroups" json:"ShardGroups,omitempty"`
 	XXX_unrecognized   []byte            `json:"-"`
 }
 
@@ -368,11 +371,11 @@ func (m *RetentionPolicyInfo) GetShardGroups() []*ShardGroupInfo {
 }
 
 type ShardGroupInfo struct {
-	ID               *uint64      `protobuf:"varint,1,req" json:"ID,omitempty"`
+	ID               *uint64      `protobuf:"varint,1,req,name=ID" json:"ID,omitempty"`
-	StartTime        *int64       `protobuf:"varint,2,req" json:"StartTime,omitempty"`
+	StartTime        *int64       `protobuf:"varint,2,req,name=StartTime" json:"StartTime,omitempty"`
-	EndTime          *int64       `protobuf:"varint,3,req" json:"EndTime,omitempty"`
+	EndTime          *int64       `protobuf:"varint,3,req,name=EndTime" json:"EndTime,omitempty"`
-	DeletedAt        *int64       `protobuf:"varint,4,req" json:"DeletedAt,omitempty"`
+	DeletedAt        *int64       `protobuf:"varint,4,req,name=DeletedAt" json:"DeletedAt,omitempty"`
-	Shards           []*ShardInfo `protobuf:"bytes,5,rep" json:"Shards,omitempty"`
+	Shards           []*ShardInfo `protobuf:"bytes,5,rep,name=Shards" json:"Shards,omitempty"`
 	XXX_unrecognized []byte       `json:"-"`
 }
 
@@ -416,9 +419,10 @@ func (m *ShardGroupInfo) GetShards() []*ShardInfo {
 }
 
 type ShardInfo struct {
-	ID               *uint64  `protobuf:"varint,1,req" json:"ID,omitempty"`
+	ID               *uint64       `protobuf:"varint,1,req,name=ID" json:"ID,omitempty"`
-	OwnerIDs         []uint64 `protobuf:"varint,2,rep" json:"OwnerIDs,omitempty"`
+	OwnerIDs         []uint64      `protobuf:"varint,2,rep,name=OwnerIDs" json:"OwnerIDs,omitempty"`
-	XXX_unrecognized []byte   `json:"-"`
+	Owners           []*ShardOwner `protobuf:"bytes,3,rep,name=Owners" json:"Owners,omitempty"`
+	XXX_unrecognized []byte        `json:"-"`
 }
 
 func (m *ShardInfo) Reset()         { *m = ShardInfo{} }
@@ -439,9 +443,32 @@ func (m *ShardInfo) GetOwnerIDs() []uint64 {
 	return nil
 }
 
+func (m *ShardInfo) GetOwners() []*ShardOwner {
+	if m != nil {
+		return m.Owners
+	}
+	return nil
+}
+
+type ShardOwner struct {
+	NodeID           *uint64 `protobuf:"varint,1,req,name=NodeID" json:"NodeID,omitempty"`
+	XXX_unrecognized []byte  `json:"-"`
+}
+
+func (m *ShardOwner) Reset()         { *m = ShardOwner{} }
+func (m *ShardOwner) String() string { return proto.CompactTextString(m) }
+func (*ShardOwner) ProtoMessage()    {}
+
+func (m *ShardOwner) GetNodeID() uint64 {
+	if m != nil && m.NodeID != nil {
+		return *m.NodeID
+	}
+	return 0
+}
+
 type ContinuousQueryInfo struct {
-	Name             *string `protobuf:"bytes,1,req" json:"Name,omitempty"`
+	Name             *string `protobuf:"bytes,1,req,name=Name" json:"Name,omitempty"`
-	Query            *string `protobuf:"bytes,2,req" json:"Query,omitempty"`
+	Query            *string `protobuf:"bytes,2,req,name=Query" json:"Query,omitempty"`
 	XXX_unrecognized []byte  `json:"-"`
 }
 
@@ -464,10 +491,10 @@ func (m *ContinuousQueryInfo) GetQuery() string {
 }
 
 type UserInfo struct {
-	Name             *string          `protobuf:"bytes,1,req" json:"Name,omitempty"`
+	Name             *string          `protobuf:"bytes,1,req,name=Name" json:"Name,omitempty"`
-	Hash             *string          `protobuf:"bytes,2,req" json:"Hash,omitempty"`
+	Hash             *string          `protobuf:"bytes,2,req,name=Hash" json:"Hash,omitempty"`
-	Admin            *bool            `protobuf:"varint,3,req" json:"Admin,omitempty"`
+	Admin            *bool            `protobuf:"varint,3,req,name=Admin" json:"Admin,omitempty"`
-	Privileges       []*UserPrivilege `protobuf:"bytes,4,rep" json:"Privileges,omitempty"`
+	Privileges       []*UserPrivilege `protobuf:"bytes,4,rep,name=Privileges" json:"Privileges,omitempty"`
 	XXX_unrecognized []byte           `json:"-"`
 }
 
@@ -504,8 +531,8 @@ func (m *UserInfo) GetPrivileges() []*UserPrivilege {
 }
 
 type UserPrivilege struct {
-	Database         *string `protobuf:"bytes,1,req" json:"Database,omitempty"`
+	Database         *string `protobuf:"bytes,1,req,name=Database" json:"Database,omitempty"`
-	Privilege        *int32  `protobuf:"varint,2,req" json:"Privilege,omitempty"`
+	Privilege        *int32  `protobuf:"varint,2,req,name=Privilege" json:"Privilege,omitempty"`
 	XXX_unrecognized []byte  `json:"-"`
 }
 
@@ -559,8 +586,8 @@ func (m *Command) GetType() Command_Type {
 }
 
 type CreateNodeCommand struct {
-	Host             *string `protobuf:"bytes,1,req" json:"Host,omitempty"`
+	Host             *string `protobuf:"bytes,1,req,name=Host" json:"Host,omitempty"`
-	Rand             *uint64 `protobuf:"varint,2,req" json:"Rand,omitempty"`
+	Rand             *uint64 `protobuf:"varint,2,req,name=Rand" json:"Rand,omitempty"`
 	XXX_unrecognized []byte  `json:"-"`
 }
 
@@ -591,7 +618,8 @@ var E_CreateNodeCommand_Command = &proto.ExtensionDesc{
 }
 
 type DeleteNodeCommand struct {
-	ID               *uint64 `protobuf:"varint,1,req" json:"ID,omitempty"`
+	ID               *uint64 `protobuf:"varint,1,req,name=ID" json:"ID,omitempty"`
+	Force            *bool   `protobuf:"varint,2,req,name=Force" json:"Force,omitempty"`
 	XXX_unrecognized []byte  `json:"-"`
 }
 
@@ -606,6 +634,13 @@ func (m *DeleteNodeCommand) GetID() uint64 {
 	return 0
 }
 
+func (m *DeleteNodeCommand) GetForce() bool {
+	if m != nil && m.Force != nil {
+		return *m.Force
+	}
+	return false
+}
+
 var E_DeleteNodeCommand_Command = &proto.ExtensionDesc{
 	ExtendedType:  (*Command)(nil),
 	ExtensionType: (*DeleteNodeCommand)(nil),
@@ -615,7 +650,7 @@ var E_DeleteNodeCommand_Command = &proto.ExtensionDesc{
 }
 
 type CreateDatabaseCommand struct {
-	Name             *string `protobuf:"bytes,1,req" json:"Name,omitempty"`
+	Name             *string `protobuf:"bytes,1,req,name=Name" json:"Name,omitempty"`
 	XXX_unrecognized []byte  `json:"-"`
 }
 
@@ -639,7 +674,7 @@ var E_CreateDatabaseCommand_Command = &proto.ExtensionDesc{
 }
 
 type DropDatabaseCommand struct {
-	Name             *string `protobuf:"bytes,1,req" json:"Name,omitempty"`
+	Name             *string `protobuf:"bytes,1,req,name=Name" json:"Name,omitempty"`
 	XXX_unrecognized []byte  `json:"-"`
 }
 
@@ -663,8 +698,8 @@ var E_DropDatabaseCommand_Command = &proto.ExtensionDesc{
 }
 
 type CreateRetentionPolicyCommand struct {
-	Database         *string              `protobuf:"bytes,1,req" json:"Database,omitempty"`
+	Database         *string              `protobuf:"bytes,1,req,name=Database" json:"Database,omitempty"`
-	RetentionPolicy  *RetentionPolicyInfo `protobuf:"bytes,2,req" json:"RetentionPolicy,omitempty"`
+	RetentionPolicy  *RetentionPolicyInfo `protobuf:"bytes,2,req,name=RetentionPolicy" json:"RetentionPolicy,omitempty"`
 	XXX_unrecognized []byte               `json:"-"`
 }
 
@@ -695,8 +730,8 @@ var E_CreateRetentionPolicyCommand_Command = &proto.ExtensionDesc{
 }
 
 type DropRetentionPolicyCommand struct {
-	Database         *string `protobuf:"bytes,1,req" json:"Database,omitempty"`
+	Database         *string `protobuf:"bytes,1,req,name=Database" json:"Database,omitempty"`
-	Name             *string `protobuf:"bytes,2,req" json:"Name,omitempty"`
+	Name             *string `protobuf:"bytes,2,req,name=Name" json:"Name,omitempty"`
 	XXX_unrecognized []byte  `json:"-"`
 }
 
@@ -727,8 +762,8 @@ var E_DropRetentionPolicyCommand_Command = &proto.ExtensionDesc{
 }
 
 type SetDefaultRetentionPolicyCommand struct {
-	Database         *string `protobuf:"bytes,1,req" json:"Database,omitempty"`
+	Database         *string `protobuf:"bytes,1,req,name=Database" json:"Database,omitempty"`
-	Name             *string `protobuf:"bytes,2,req" json:"Name,omitempty"`
+	Name             *string `protobuf:"bytes,2,req,name=Name" json:"Name,omitempty"`
 	XXX_unrecognized []byte  `json:"-"`
 }
 
@@ -759,11 +794,11 @@ var E_SetDefaultRetentionPolicyCommand_Command = &proto.ExtensionDesc{
 }
 
 type UpdateRetentionPolicyCommand struct {
-	Database         *string `protobuf:"bytes,1,req" json:"Database,omitempty"`
+	Database         *string `protobuf:"bytes,1,req,name=Database" json:"Database,omitempty"`
-	Name             *string `protobuf:"bytes,2,req" json:"Name,omitempty"`
+	Name             *string `protobuf:"bytes,2,req,name=Name" json:"Name,omitempty"`
-	NewName          *string `protobuf:"bytes,3,opt" json:"NewName,omitempty"`
+	NewName          *string `protobuf:"bytes,3,opt,name=NewName" json:"NewName,omitempty"`
-	Duration         *int64  `protobuf:"varint,4,opt" json:"Duration,omitempty"`
+	Duration         *int64  `protobuf:"varint,4,opt,name=Duration" json:"Duration,omitempty"`
-	ReplicaN         *uint32 `protobuf:"varint,5,opt" json:"ReplicaN,omitempty"`
+	ReplicaN         *uint32 `protobuf:"varint,5,opt,name=ReplicaN" json:"ReplicaN,omitempty"`
 	XXX_unrecognized []byte  `json:"-"`
 }
 
@@ -815,9 +850,9 @@ var E_UpdateRetentionPolicyCommand_Command = &proto.ExtensionDesc{
 }
 
 type CreateShardGroupCommand struct {
-	Database         *string `protobuf:"bytes,1,req" json:"Database,omitempty"`
+	Database         *string `protobuf:"bytes,1,req,name=Database" json:"Database,omitempty"`
-	Policy           *string `protobuf:"bytes,2,req" json:"Policy,omitempty"`
+	Policy           *string `protobuf:"bytes,2,req,name=Policy" json:"Policy,omitempty"`
-	Timestamp        *int64  `protobuf:"varint,3,req" json:"Timestamp,omitempty"`
+	Timestamp        *int64  `protobuf:"varint,3,req,name=Timestamp" json:"Timestamp,omitempty"`
 	XXX_unrecognized []byte  `json:"-"`
 }
 
@@ -855,9 +890,9 @@ var E_CreateShardGroupCommand_Command = &proto.ExtensionDesc{
 }
 
 type DeleteShardGroupCommand struct {
-	Database         *string `protobuf:"bytes,1,req" json:"Database,omitempty"`
+	Database         *string `protobuf:"bytes,1,req,name=Database" json:"Database,omitempty"`
-	Policy           *string `protobuf:"bytes,2,req" json:"Policy,omitempty"`
+	Policy           *string `protobuf:"bytes,2,req,name=Policy" json:"Policy,omitempty"`
-	ShardGroupID     *uint64 `protobuf:"varint,3,req" json:"ShardGroupID,omitempty"`
+	ShardGroupID     *uint64 `protobuf:"varint,3,req,name=ShardGroupID" json:"ShardGroupID,omitempty"`
 	XXX_unrecognized []byte  `json:"-"`
 }
 
@@ -895,9 +930,9 @@ var E_DeleteShardGroupCommand_Command = &proto.ExtensionDesc{
 }
 
 type CreateContinuousQueryCommand struct {
-	Database         *string `protobuf:"bytes,1,req" json:"Database,omitempty"`
+	Database         *string `protobuf:"bytes,1,req,name=Database" json:"Database,omitempty"`
-	Name             *string `protobuf:"bytes,2,req" json:"Name,omitempty"`
+	Name             *string `protobuf:"bytes,2,req,name=Name" json:"Name,omitempty"`
-	Query            *string `protobuf:"bytes,3,req" json:"Query,omitempty"`
+	Query            *string `protobuf:"bytes,3,req,name=Query" json:"Query,omitempty"`
 	XXX_unrecognized []byte  `json:"-"`
 }
 
@@ -935,8 +970,8 @@ var E_CreateContinuousQueryCommand_Command = &proto.ExtensionDesc{
 }
 
 type DropContinuousQueryCommand struct {
-	Database         *string `protobuf:"bytes,1,req" json:"Database,omitempty"`
+	Database         *string `protobuf:"bytes,1,req,name=Database" json:"Database,omitempty"`
-	Name             *string `protobuf:"bytes,2,req" json:"Name,omitempty"`
+	Name             *string `protobuf:"bytes,2,req,name=Name" json:"Name,omitempty"`
 	XXX_unrecognized []byte  `json:"-"`
 }
 
@@ -967,9 +1002,9 @@ var E_DropContinuousQueryCommand_Command = &proto.ExtensionDesc{
 }
 
 type CreateUserCommand struct {
-	Name             *string `protobuf:"bytes,1,req" json:"Name,omitempty"`
+	Name             *string `protobuf:"bytes,1,req,name=Name" json:"Name,omitempty"`
-	Hash             *string `protobuf:"bytes,2,req" json:"Hash,omitempty"`
+	Hash             *string `protobuf:"bytes,2,req,name=Hash" json:"Hash,omitempty"`
-	Admin            *bool   `protobuf:"varint,3,req" json:"Admin,omitempty"`
+	Admin            *bool   `protobuf:"varint,3,req,name=Admin" json:"Admin,omitempty"`
 	XXX_unrecognized []byte  `json:"-"`
 }
 
@@ -1007,7 +1042,7 @@ var E_CreateUserCommand_Command = &proto.ExtensionDesc{
 }
 
 type DropUserCommand struct {
-	Name             *string `protobuf:"bytes,1,req" json:"Name,omitempty"`
+	Name             *string `protobuf:"bytes,1,req,name=Name" json:"Name,omitempty"`
 	XXX_unrecognized []byte  `json:"-"`
 }
 
@@ -1031,8 +1066,8 @@ var E_DropUserCommand_Command = &proto.ExtensionDesc{
 }
 
 type UpdateUserCommand struct {
-	Name             *string `protobuf:"bytes,1,req" json:"Name,omitempty"`
+	Name             *string `protobuf:"bytes,1,req,name=Name" json:"Name,omitempty"`
-	Hash             *string `protobuf:"bytes,2,req" json:"Hash,omitempty"`
+	Hash             *string `protobuf:"bytes,2,req,name=Hash" json:"Hash,omitempty"`
 	XXX_unrecognized []byte  `json:"-"`
 }
 
@@ -1063,9 +1098,9 @@ var E_UpdateUserCommand_Command = &proto.ExtensionDesc{
 }
 
 type SetPrivilegeCommand struct {
-	Username         *string `protobuf:"bytes,1,req" json:"Username,omitempty"`
+	Username         *string `protobuf:"bytes,1,req,name=Username" json:"Username,omitempty"`
-	Database         *string `protobuf:"bytes,2,req" json:"Database,omitempty"`
+	Database         *string `protobuf:"bytes,2,req,name=Database" json:"Database,omitempty"`
-	Privilege        *int32  `protobuf:"varint,3,req" json:"Privilege,omitempty"`
+	Privilege        *int32  `protobuf:"varint,3,req,name=Privilege" json:"Privilege,omitempty"`
 	XXX_unrecognized []byte  `json:"-"`
 }
 
@@ -1103,7 +1138,7 @@ var E_SetPrivilegeCommand_Command = &proto.ExtensionDesc{
 }
 
 type SetDataCommand struct {
-	Data             *Data  `protobuf:"bytes,1,req" json:"Data,omitempty"`
+	Data             *Data  `protobuf:"bytes,1,req,name=Data" json:"Data,omitempty"`
 	XXX_unrecognized []byte `json:"-"`
 }
 
@@ -1127,8 +1162,8 @@ var E_SetDataCommand_Command = &proto.ExtensionDesc{
 }
 
 type SetAdminPrivilegeCommand struct {
-	Username         *string `protobuf:"bytes,1,req" json:"Username,omitempty"`
+	Username         *string `protobuf:"bytes,1,req,name=Username" json:"Username,omitempty"`
-	Admin            *bool   `protobuf:"varint,2,req" json:"Admin,omitempty"`
+	Admin            *bool   `protobuf:"varint,2,req,name=Admin" json:"Admin,omitempty"`
 	XXX_unrecognized []byte  `json:"-"`
 }
 
@@ -1159,8 +1194,8 @@ var E_SetAdminPrivilegeCommand_Command = &proto.ExtensionDesc{
 }
 
 type UpdateNodeCommand struct {
-	ID               *uint64 `protobuf:"varint,1,req" json:"ID,omitempty"`
+	ID               *uint64 `protobuf:"varint,1,req,name=ID" json:"ID,omitempty"`
-	Host             *string `protobuf:"bytes,2,req" json:"Host,omitempty"`
+	Host             *string `protobuf:"bytes,2,req,name=Host" json:"Host,omitempty"`
 	XXX_unrecognized []byte  `json:"-"`
 }
 
@@ -1191,9 +1226,9 @@ var E_UpdateNodeCommand_Command = &proto.ExtensionDesc{
 }
 
 type Response struct {
-	OK               *bool   `protobuf:"varint,1,req" json:"OK,omitempty"`
+	OK               *bool   `protobuf:"varint,1,req,name=OK" json:"OK,omitempty"`
-	Error            *string `protobuf:"bytes,2,opt" json:"Error,omitempty"`
+	Error            *string `protobuf:"bytes,2,opt,name=Error" json:"Error,omitempty"`
-	Index            *uint64 `protobuf:"varint,3,opt" json:"Index,omitempty"`
+	Index            *uint64 `protobuf:"varint,3,opt,name=Index" json:"Index,omitempty"`
 	XXX_unrecognized []byte  `json:"-"`
 }
 
@@ -1223,8 +1258,8 @@ func (m *Response) GetIndex() uint64 {
 }
 
 type ResponseHeader struct {
-	OK               *bool   `protobuf:"varint,1,req" json:"OK,omitempty"`
+	OK               *bool   `protobuf:"varint,1,req,name=OK" json:"OK,omitempty"`
-	Error            *string `protobuf:"bytes,2,opt" json:"Error,omitempty"`
+	Error            *string `protobuf:"bytes,2,opt,name=Error" json:"Error,omitempty"`
 	XXX_unrecognized []byte  `json:"-"`
 }
 
@@ -1247,7 +1282,7 @@ func (m *ResponseHeader) GetError() string {
 }
 
 type ErrorResponse struct {
-	Header           *ResponseHeader `protobuf:"bytes,1,req" json:"Header,omitempty"`
+	Header           *ResponseHeader `protobuf:"bytes,1,req,name=Header" json:"Header,omitempty"`
 	XXX_unrecognized []byte          `json:"-"`
 }
 
@@ -1263,9 +1298,9 @@ func (m *ErrorResponse) GetHeader() *ResponseHeader {
 }
 
 type FetchDataRequest struct {
-	Index            *uint64 `protobuf:"varint,1,req" json:"Index,omitempty"`
+	Index            *uint64 `protobuf:"varint,1,req,name=Index" json:"Index,omitempty"`
-	Term             *uint64 `protobuf:"varint,2,req" json:"Term,omitempty"`
+	Term             *uint64 `protobuf:"varint,2,req,name=Term" json:"Term,omitempty"`
-	Blocking         *bool   `protobuf:"varint,3,opt,def=0" json:"Blocking,omitempty"`
+	Blocking         *bool   `protobuf:"varint,3,opt,name=Blocking,def=0" json:"Blocking,omitempty"`
 	XXX_unrecognized []byte  `json:"-"`
 }
 
@@ -1297,10 +1332,10 @@ func (m *FetchDataRequest) GetBlocking() bool {
 }
 
 type FetchDataResponse struct {
-	Header           *ResponseHeader `protobuf:"bytes,1,req" json:"Header,omitempty"`
+	Header           *ResponseHeader `protobuf:"bytes,1,req,name=Header" json:"Header,omitempty"`
-	Index            *uint64         `protobuf:"varint,2,req" json:"Index,omitempty"`
+	Index            *uint64         `protobuf:"varint,2,req,name=Index" json:"Index,omitempty"`
-	Term             *uint64         `protobuf:"varint,3,req" json:"Term,omitempty"`
+	Term             *uint64         `protobuf:"varint,3,req,name=Term" json:"Term,omitempty"`
-	Data             []byte          `protobuf:"bytes,4,opt" json:"Data,omitempty"`
+	Data             []byte          `protobuf:"bytes,4,opt,name=Data" json:"Data,omitempty"`
 	XXX_unrecognized []byte          `json:"-"`
 }
 
@@ -1337,7 +1372,7 @@ func (m *FetchDataResponse) GetData() []byte {
 }
 
 type JoinRequest struct {
-	Addr             *string `protobuf:"bytes,1,req" json:"Addr,omitempty"`
+	Addr             *string `protobuf:"bytes,1,req,name=Addr" json:"Addr,omitempty"`
 	XXX_unrecognized []byte  `json:"-"`
 }
 
@@ -1353,14 +1388,14 @@ func (m *JoinRequest) GetAddr() string {
 }
 
 type JoinResponse struct {
-	Header *ResponseHeader `protobuf:"bytes,1,req" json:"Header,omitempty"`
+	Header *ResponseHeader `protobuf:"bytes,1,req,name=Header" json:"Header,omitempty"`
 	// Indicates that this node should take part in the raft cluster.
-	EnableRaft *bool `protobuf:"varint,2,opt" json:"EnableRaft,omitempty"`
+	EnableRaft *bool `protobuf:"varint,2,opt,name=EnableRaft" json:"EnableRaft,omitempty"`
 	// The addresses of raft peers to use if joining as a raft member. If not joining
 	// as a raft member, these are the nodes running raft.
-	RaftNodes []string `protobuf:"bytes,3,rep" json:"RaftNodes,omitempty"`
+	RaftNodes []string `protobuf:"bytes,3,rep,name=RaftNodes" json:"RaftNodes,omitempty"`
 	// The node ID assigned to the requesting node.
-	NodeID           *uint64 `protobuf:"varint,4,opt" json:"NodeID,omitempty"`
+	NodeID           *uint64 `protobuf:"varint,4,opt,name=NodeID" json:"NodeID,omitempty"`
 	XXX_unrecognized []byte  `json:"-"`
 }
 

Godeps/_workspace/src/github.com/influxdb/influxdb/meta/internal/meta.proto

@@ -49,8 +49,13 @@ message ShardGroupInfo {
 }
 
 message ShardInfo {
-	required uint64 ID = 1;
+    required uint64 ID = 1;
-	repeated uint64 OwnerIDs = 2;
+    repeated uint64 OwnerIDs = 2 [deprecated=true];
+    repeated ShardOwner Owners = 3;
+}
+
+message ShardOwner {
+    required uint64 NodeID = 1;
 }
 
 message ContinuousQueryInfo {
@@ -118,6 +123,7 @@ message DeleteNodeCommand {
         optional DeleteNodeCommand command = 102;
     }
 	required uint64 ID = 1;
+	required bool Force = 2;
 }
 
 message CreateDatabaseCommand {

Godeps/_workspace/src/github.com/influxdb/influxdb/meta/rpc_test.go

@@ -122,7 +122,7 @@ func TestRPCFetchDataMatchesBlocking(t *testing.T) {
 
 	// Simulate the rmote index changing and unblocking
 	fs.mu.Lock()
-	fs.md.Index = 100
+	fs.md = &Data{Index: 100}
 	fs.mu.Unlock()
 	close(fs.blockChan)
 	wg.Wait()

Godeps/_workspace/src/github.com/influxdb/influxdb/meta/statement_executor.go

@@ -1,17 +1,24 @@
 package meta
 
 import (
+	"bytes"
 	"fmt"
+	"strconv"
+	"time"
 
 	"github.com/influxdb/influxdb/influxql"
+	"github.com/influxdb/influxdb/models"
 )
 
 // StatementExecutor translates InfluxQL queries to meta store methods.
 type StatementExecutor struct {
 	Store interface {
+		Node(id uint64) (ni *NodeInfo, err error)
 		Nodes() ([]NodeInfo, error)
 		Peers() ([]string, error)
+		Leader() string
 
+		DeleteNode(nodeID uint64, force bool) error
 		Database(name string) (*DatabaseInfo, error)
 		Databases() ([]DatabaseInfo, error)
 		CreateDatabase(name string) (*DatabaseInfo, error)
@@ -80,8 +87,12 @@ func (e *StatementExecutor) ExecuteStatement(stmt influxql.Statement) *influxql.
 		return e.executeDropContinuousQueryStatement(stmt)
 	case *influxql.ShowContinuousQueriesStatement:
 		return e.executeShowContinuousQueriesStatement(stmt)
+	case *influxql.ShowShardsStatement:
+		return e.executeShowShardsStatement(stmt)
 	case *influxql.ShowStatsStatement:
 		return e.executeShowStatsStatement(stmt)
+	case *influxql.DropServerStatement:
+		return e.executeDropServerStatement(stmt)
 	default:
 		panic(fmt.Sprintf("unsupported statement type: %T", stmt))
 	}
@@ -89,6 +100,9 @@ func (e *StatementExecutor) ExecuteStatement(stmt influxql.Statement) *influxql.
 
 func (e *StatementExecutor) executeCreateDatabaseStatement(q *influxql.CreateDatabaseStatement) *influxql.Result {
 	_, err := e.Store.CreateDatabase(q.Name)
+	if err == ErrDatabaseExists && q.IfNotExists {
+		err = nil
+	}
 	return &influxql.Result{Err: err}
 }
 
@@ -102,11 +116,11 @@ func (e *StatementExecutor) executeShowDatabasesStatement(q *influxql.ShowDataba
 		return &influxql.Result{Err: err}
 	}
 
-	row := &influxql.Row{Name: "databases", Columns: []string{"name"}}
+	row := &models.Row{Name: "databases", Columns: []string{"name"}}
 	for _, di := range dis {
 		row.Values = append(row.Values, []interface{}{di.Name})
 	}
-	return &influxql.Result{Series: []*influxql.Row{row}}
+	return &influxql.Result{Series: []*models.Row{row}}
 }
 
 func (e *StatementExecutor) executeShowGrantsForUserStatement(q *influxql.ShowGrantsForUserStatement) *influxql.Result {
@@ -115,11 +129,11 @@ func (e *StatementExecutor) executeShowGrantsForUserStatement(q *influxql.ShowGr
 		return &influxql.Result{Err: err}
 	}
 
-	row := &influxql.Row{Columns: []string{"database", "privilege"}}
+	row := &models.Row{Columns: []string{"database", "privilege"}}
 	for d, p := range priv {
 		row.Values = append(row.Values, []interface{}{d, p.String()})
 	}
-	return &influxql.Result{Series: []*influxql.Row{row}}
+	return &influxql.Result{Series: []*models.Row{row}}
 }
 
 func (e *StatementExecutor) executeShowServersStatement(q *influxql.ShowServersStatement) *influxql.Result {
@@ -133,11 +147,35 @@ func (e *StatementExecutor) executeShowServersStatement(q *influxql.ShowServersS
 		return &influxql.Result{Err: err}
 	}
 
-	row := &influxql.Row{Columns: []string{"id", "cluster_addr", "raft"}}
+	leader := e.Store.Leader()
+
+	row := &models.Row{Columns: []string{"id", "cluster_addr", "raft", "raft-leader"}}
 	for _, ni := range nis {
-		row.Values = append(row.Values, []interface{}{ni.ID, ni.Host, contains(peers, ni.Host)})
+		row.Values = append(row.Values, []interface{}{ni.ID, ni.Host, contains(peers, ni.Host), leader == ni.Host})
 	}
-	return &influxql.Result{Series: []*influxql.Row{row}}
+	return &influxql.Result{Series: []*models.Row{row}}
+}
+
+func (e *StatementExecutor) executeDropServerStatement(q *influxql.DropServerStatement) *influxql.Result {
+	ni, err := e.Store.Node(q.NodeID)
+	if err != nil {
+		return &influxql.Result{Err: err}
+	}
+	if ni == nil {
+		return &influxql.Result{Err: ErrNodeNotFound}
+	}
+
+	// Dropping only non-Raft nodes supported.
+	peers, err := e.Store.Peers()
+	if err != nil {
+		return &influxql.Result{Err: err}
+	}
+	if contains(peers, ni.Host) {
+		return &influxql.Result{Err: ErrNodeRaft}
+	}
+
+	err = e.Store.DeleteNode(q.NodeID, q.Force)
+	return &influxql.Result{Err: err}
 }
 
 func (e *StatementExecutor) executeCreateUserStatement(q *influxql.CreateUserStatement) *influxql.Result {
@@ -159,11 +197,11 @@ func (e *StatementExecutor) executeShowUsersStatement(q *influxql.ShowUsersState
 		return &influxql.Result{Err: err}
 	}
 
-	row := &influxql.Row{Columns: []string{"user", "admin"}}
+	row := &models.Row{Columns: []string{"user", "admin"}}
 	for _, ui := range uis {
 		row.Values = append(row.Values, []interface{}{ui.Name, ui.Admin})
 	}
-	return &influxql.Result{Series: []*influxql.Row{row}}
+	return &influxql.Result{Series: []*models.Row{row}}
 }
 
 func (e *StatementExecutor) executeGrantStatement(stmt *influxql.GrantStatement) *influxql.Result {
@@ -245,11 +283,11 @@ func (e *StatementExecutor) executeShowRetentionPoliciesStatement(q *influxql.Sh
 		return &influxql.Result{Err: ErrDatabaseNotFound}
 	}
 
-	row := &influxql.Row{Columns: []string{"name", "duration", "replicaN", "default"}}
+	row := &models.Row{Columns: []string{"name", "duration", "replicaN", "default"}}
 	for _, rpi := range di.RetentionPolicies {
 		row.Values = append(row.Values, []interface{}{rpi.Name, rpi.Duration.String(), rpi.ReplicaN, di.DefaultRetentionPolicy == rpi.Name})
 	}
-	return &influxql.Result{Series: []*influxql.Row{row}}
+	return &influxql.Result{Series: []*models.Row{row}}
 }
 
 func (e *StatementExecutor) executeCreateContinuousQueryStatement(q *influxql.CreateContinuousQueryStatement) *influxql.Result {
@@ -270,9 +308,9 @@ func (e *StatementExecutor) executeShowContinuousQueriesStatement(stmt *influxql
 		return &influxql.Result{Err: err}
 	}
 
-	rows := []*influxql.Row{}
+	rows := []*models.Row{}
 	for _, di := range dis {
-		row := &influxql.Row{Columns: []string{"name", "query"}, Name: di.Name}
+		row := &models.Row{Columns: []string{"name", "query"}, Name: di.Name}
 		for _, cqi := range di.ContinuousQueries {
 			row.Values = append(row.Values, []interface{}{cqi.Name, cqi.Query})
 		}
@@ -281,6 +319,50 @@ func (e *StatementExecutor) executeShowContinuousQueriesStatement(stmt *influxql
 	return &influxql.Result{Series: rows}
 }
 
+func (e *StatementExecutor) executeShowShardsStatement(stmt *influxql.ShowShardsStatement) *influxql.Result {
+	dis, err := e.Store.Databases()
+	if err != nil {
+		return &influxql.Result{Err: err}
+	}
+
+	rows := []*models.Row{}
+	for _, di := range dis {
+		row := &models.Row{Columns: []string{"id", "start_time", "end_time", "expiry_time", "owners"}, Name: di.Name}
+		for _, rpi := range di.RetentionPolicies {
+			for _, sgi := range rpi.ShardGroups {
+				for _, si := range sgi.Shards {
+					ownerIDs := make([]uint64, len(si.Owners))
+					for i, owner := range si.Owners {
+						ownerIDs[i] = owner.NodeID
+					}
+
+					row.Values = append(row.Values, []interface{}{
+						si.ID,
+						sgi.StartTime.UTC().Format(time.RFC3339),
+						sgi.EndTime.UTC().Format(time.RFC3339),
+						sgi.EndTime.Add(rpi.Duration).UTC().Format(time.RFC3339),
+						joinUint64(ownerIDs),
+					})
+				}
+			}
+		}
+		rows = append(rows, row)
+	}
+	return &influxql.Result{Series: rows}
+}
+
 func (e *StatementExecutor) executeShowStatsStatement(stmt *influxql.ShowStatsStatement) *influxql.Result {
 	return &influxql.Result{Err: fmt.Errorf("SHOW STATS is not implemented yet")}
 }
+
+// joinUint64 returns a comma-delimited string of uint64 numbers.
+func joinUint64(a []uint64) string {
+	var buf bytes.Buffer
+	for i, x := range a {
+		buf.WriteString(strconv.FormatUint(x, 10))
+		if i < len(a)-1 {
+			buf.WriteRune(',')
+		}
+	}
+	return buf.String()
+}

Godeps/_workspace/src/github.com/influxdb/influxdb/meta/statement_executor_test.go

@@ -9,6 +9,7 @@ import (
 	"github.com/davecgh/go-spew/spew"
 	"github.com/influxdb/influxdb/influxql"
 	"github.com/influxdb/influxdb/meta"
+	"github.com/influxdb/influxdb/models"
 )
 
 // Ensure a CREATE DATABASE statement can be executed.
@@ -57,7 +58,7 @@ func TestStatementExecutor_ExecuteStatement_ShowDatabases(t *testing.T) {
 
 	if res := e.ExecuteStatement(influxql.MustParseStatement(`SHOW DATABASES`)); res.Err != nil {
 		t.Fatal(res.Err)
-	} else if !reflect.DeepEqual(res.Series, influxql.Rows{
+	} else if !reflect.DeepEqual(res.Series, models.Rows{
 		{
 			Name:    "databases",
 			Columns: []string{"name"},
@@ -99,7 +100,7 @@ func TestStatementExecutor_ExecuteStatement_ShowGrantsFor(t *testing.T) {
 
 	if res := e.ExecuteStatement(influxql.MustParseStatement(`SHOW GRANTS FOR dejan`)); res.Err != nil {
 		t.Fatal(res.Err)
-	} else if !reflect.DeepEqual(res.Series, influxql.Rows{
+	} else if !reflect.DeepEqual(res.Series, models.Rows{
 		{
 			Columns: []string{"database", "privilege"},
 			Values: [][]interface{}{
@@ -124,15 +125,18 @@ func TestStatementExecutor_ExecuteStatement_ShowServers(t *testing.T) {
 	e.Store.PeersFn = func() ([]string, error) {
 		return []string{"node0"}, nil
 	}
+	e.Store.LeaderFn = func() string {
+		return "node0"
+	}
 
 	if res := e.ExecuteStatement(influxql.MustParseStatement(`SHOW SERVERS`)); res.Err != nil {
 		t.Fatal(res.Err)
-	} else if !reflect.DeepEqual(res.Series, influxql.Rows{
+	} else if !reflect.DeepEqual(res.Series, models.Rows{
 		{
-			Columns: []string{"id", "cluster_addr", "raft"},
+			Columns: []string{"id", "cluster_addr", "raft", "raft-leader"},
 			Values: [][]interface{}{
-				{uint64(1), "node0", true},
+				{uint64(1), "node0", true, true},
-				{uint64(2), "node1", false},
+				{uint64(2), "node1", false, false},
 			},
 		},
 	}) {
@@ -140,6 +144,45 @@ func TestStatementExecutor_ExecuteStatement_ShowServers(t *testing.T) {
 	}
 }
 
+// Ensure a DROP SERVER statement can be executed.
+func TestStatementExecutor_ExecuteStatement_DropServer(t *testing.T) {
+	e := NewStatementExecutor()
+	e.Store.PeersFn = func() ([]string, error) {
+		return []string{"node1"}, nil
+	}
+
+	// Ensure non-existent nodes do not cause a problem.
+	e.Store.NodeFn = func(id uint64) (*meta.NodeInfo, error) {
+		return nil, nil
+	}
+	if res := e.ExecuteStatement(influxql.MustParseStatement(`DROP SERVER 666`)); res.Err != meta.ErrNodeNotFound {
+		t.Fatalf("unexpected error: %s", res.Err)
+	}
+
+	// Make a node exist.
+	e.Store.NodeFn = func(id uint64) (*meta.NodeInfo, error) {
+		return &meta.NodeInfo{
+			ID: 1, Host: "node1",
+		}, nil
+	}
+
+	// Ensure Raft nodes cannot be dropped.
+	if res := e.ExecuteStatement(influxql.MustParseStatement(`DROP SERVER 1`)); res.Err != meta.ErrNodeRaft {
+		t.Fatalf("unexpected error: %s", res.Err)
+	}
+
+	// Ensure non-Raft nodes can be dropped.
+	e.Store.PeersFn = func() ([]string, error) {
+		return []string{"node2"}, nil
+	}
+	e.Store.DeleteNodeFn = func(id uint64, force bool) error {
+		return nil
+	}
+	if res := e.ExecuteStatement(influxql.MustParseStatement(`DROP SERVER 1`)); res.Err != nil {
+		t.Fatalf("unexpected error: %s", res.Err)
+	}
+}
+
 // Ensure a SHOW SERVERS statement returns errors from the store.
 func TestStatementExecutor_ExecuteStatement_ShowServers_Err(t *testing.T) {
 	e := NewStatementExecutor()
@@ -257,7 +300,7 @@ func TestStatementExecutor_ExecuteStatement_ShowUsers(t *testing.T) {
 
 	if res := e.ExecuteStatement(influxql.MustParseStatement(`SHOW USERS`)); res.Err != nil {
 		t.Fatal(res.Err)
-	} else if !reflect.DeepEqual(res.Series, influxql.Rows{
+	} else if !reflect.DeepEqual(res.Series, models.Rows{
 		{
 			Columns: []string{"user", "admin"},
 			Values: [][]interface{}{
@@ -580,7 +623,7 @@ func TestStatementExecutor_ExecuteStatement_ShowRetentionPolicies(t *testing.T)
 
 	if res := e.ExecuteStatement(influxql.MustParseStatement(`SHOW RETENTION POLICIES ON db0`)); res.Err != nil {
 		t.Fatal(res.Err)
-	} else if !reflect.DeepEqual(res.Series, influxql.Rows{
+	} else if !reflect.DeepEqual(res.Series, models.Rows{
 		{
 			Columns: []string{"name", "duration", "replicaN", "default"},
 			Values: [][]interface{}{
@@ -625,13 +668,13 @@ func TestStatementExecutor_ExecuteStatement_CreateContinuousQuery(t *testing.T)
 			t.Fatalf("unexpected database: %s", database)
 		} else if name != "cq0" {
 			t.Fatalf("unexpected name: %s", name)
-		} else if query != `CREATE CONTINUOUS QUERY cq0 ON db0 BEGIN SELECT count(*) INTO db1 FROM db0 GROUP BY time(1h) END` {
+		} else if query != `CREATE CONTINUOUS QUERY cq0 ON db0 BEGIN SELECT count(field1) INTO db1 FROM db0 GROUP BY time(1h) END` {
 			t.Fatalf("unexpected query: %s", query)
 		}
 		return nil
 	}
 
-	stmt := influxql.MustParseStatement(`CREATE CONTINUOUS QUERY cq0 ON db0 BEGIN SELECT count(*) INTO db1 FROM db0 GROUP BY time(1h) END`)
+	stmt := influxql.MustParseStatement(`CREATE CONTINUOUS QUERY cq0 ON db0 BEGIN SELECT count(field1) INTO db1 FROM db0 GROUP BY time(1h) END`)
 	if res := e.ExecuteStatement(stmt); res.Err != nil {
 		t.Fatal(res.Err)
 	} else if res.Series != nil {
@@ -646,7 +689,7 @@ func TestStatementExecutor_ExecuteStatement_CreateContinuousQuery_Err(t *testing
 		return errors.New("marker")
 	}
 
-	stmt := influxql.MustParseStatement(`CREATE CONTINUOUS QUERY cq0 ON db0 BEGIN SELECT count(*) INTO db1 FROM db0 GROUP BY time(1h) END`)
+	stmt := influxql.MustParseStatement(`CREATE CONTINUOUS QUERY cq0 ON db0 BEGIN SELECT count(field1) INTO db1 FROM db0 GROUP BY time(1h) END`)
 	if res := e.ExecuteStatement(stmt); res.Err == nil || res.Err.Error() != "marker" {
 		t.Fatalf("unexpected error: %s", res.Err)
 	}
@@ -693,14 +736,14 @@ func TestStatementExecutor_ExecuteStatement_ShowContinuousQueries(t *testing.T)
 			{
 				Name: "db0",
 				ContinuousQueries: []meta.ContinuousQueryInfo{
-					{Name: "cq0", Query: "SELECT count(*) INTO db1 FROM db0"},
+					{Name: "cq0", Query: "SELECT count(field1) INTO db1 FROM db0"},
-					{Name: "cq1", Query: "SELECT count(*) INTO db2 FROM db0"},
+					{Name: "cq1", Query: "SELECT count(field1) INTO db2 FROM db0"},
 				},
 			},
 			{
 				Name: "db1",
 				ContinuousQueries: []meta.ContinuousQueryInfo{
-					{Name: "cq2", Query: "SELECT count(*) INTO db3 FROM db1"},
+					{Name: "cq2", Query: "SELECT count(field1) INTO db3 FROM db1"},
 				},
 			},
 		}, nil
@@ -709,20 +752,20 @@ func TestStatementExecutor_ExecuteStatement_ShowContinuousQueries(t *testing.T)
 	stmt := influxql.MustParseStatement(`SHOW CONTINUOUS QUERIES`)
 	if res := e.ExecuteStatement(stmt); res.Err != nil {
 		t.Fatal(res.Err)
-	} else if !reflect.DeepEqual(res.Series, influxql.Rows{
+	} else if !reflect.DeepEqual(res.Series, models.Rows{
 		{
 			Name:    "db0",
 			Columns: []string{"name", "query"},
 			Values: [][]interface{}{
-				{"cq0", "SELECT count(*) INTO db1 FROM db0"},
+				{"cq0", "SELECT count(field1) INTO db1 FROM db0"},
-				{"cq1", "SELECT count(*) INTO db2 FROM db0"},
+				{"cq1", "SELECT count(field1) INTO db2 FROM db0"},
 			},
 		},
 		{
 			Name:    "db1",
 			Columns: []string{"name", "query"},
 			Values: [][]interface{}{
-				{"cq2", "SELECT count(*) INTO db3 FROM db1"},
+				{"cq2", "SELECT count(field1) INTO db3 FROM db1"},
 			},
 		},
 	}) {
@@ -755,7 +798,7 @@ func TestStatementExecutor_ExecuteStatement_Unsupported(t *testing.T) {
 
 		// Execute a SELECT statement.
 		NewStatementExecutor().ExecuteStatement(
-			influxql.MustParseStatement(`SELECT count(*) FROM db0`),
+			influxql.MustParseStatement(`SELECT count(field1) FROM db0`),
 		)
 	}()
 
@@ -765,6 +808,57 @@ func TestStatementExecutor_ExecuteStatement_Unsupported(t *testing.T) {
 	}
 }
 
+// Ensure a SHOW SHARDS statement can be executed.
+func TestStatementExecutor_ExecuteStatement_ShowShards(t *testing.T) {
+	e := NewStatementExecutor()
+	e.Store.DatabasesFn = func() ([]meta.DatabaseInfo, error) {
+		return []meta.DatabaseInfo{
+			{
+				Name: "foo",
+				RetentionPolicies: []meta.RetentionPolicyInfo{
+					{
+						Duration: time.Second,
+						ShardGroups: []meta.ShardGroupInfo{
+							{
+								StartTime: time.Unix(0, 0),
+								EndTime:   time.Unix(1, 0),
+								Shards: []meta.ShardInfo{
+									{
+										ID: 1,
+										Owners: []meta.ShardOwner{
+											{NodeID: 1},
+											{NodeID: 2},
+											{NodeID: 3},
+										},
+									},
+									{
+										ID: 2,
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+		}, nil
+	}
+
+	if res := e.ExecuteStatement(influxql.MustParseStatement(`SHOW SHARDS`)); res.Err != nil {
+		t.Fatal(res.Err)
+	} else if !reflect.DeepEqual(res.Series, models.Rows{
+		{
+			Name:    "foo",
+			Columns: []string{"id", "start_time", "end_time", "expiry_time", "owners"},
+			Values: [][]interface{}{
+				{uint64(1), "1970-01-01T00:00:00Z", "1970-01-01T00:00:01Z", "1970-01-01T00:00:02Z", "1,2,3"},
+				{uint64(2), "1970-01-01T00:00:00Z", "1970-01-01T00:00:01Z", "1970-01-01T00:00:02Z", ""},
+			},
+		},
+	}) {
+		t.Fatalf("unexpected rows: %s", spew.Sdump(res.Series))
+	}
+}
+
 // StatementExecutor represents a test wrapper for meta.StatementExecutor.
 type StatementExecutor struct {
 	*meta.StatementExecutor
@@ -780,12 +874,15 @@ func NewStatementExecutor() *StatementExecutor {
 
 // StatementExecutorStore represents a mock implementation of StatementExecutor.Store.
 type StatementExecutorStore struct {
+	NodeFn                      func(id uint64) (*meta.NodeInfo, error)
 	NodesFn                     func() ([]meta.NodeInfo, error)
 	PeersFn                     func() ([]string, error)
+	LeaderFn                    func() string
 	DatabaseFn                  func(name string) (*meta.DatabaseInfo, error)
 	DatabasesFn                 func() ([]meta.DatabaseInfo, error)
 	CreateDatabaseFn            func(name string) (*meta.DatabaseInfo, error)
 	DropDatabaseFn              func(name string) error
+	DeleteNodeFn                func(nodeID uint64, force bool) error
 	DefaultRetentionPolicyFn    func(database string) (*meta.RetentionPolicyInfo, error)
 	CreateRetentionPolicyFn     func(database string, rpi *meta.RetentionPolicyInfo) (*meta.RetentionPolicyInfo, error)
 	UpdateRetentionPolicyFn     func(database, name string, rpu *meta.RetentionPolicyUpdate) error
@@ -804,6 +901,10 @@ type StatementExecutorStore struct {
 	DropContinuousQueryFn       func(database, name string) error
 }
 
+func (s *StatementExecutorStore) Node(id uint64) (*meta.NodeInfo, error) {
+	return s.NodeFn(id)
+}
+
 func (s *StatementExecutorStore) Nodes() ([]meta.NodeInfo, error) {
 	return s.NodesFn()
 }
@@ -812,6 +913,17 @@ func (s *StatementExecutorStore) Peers() ([]string, error) {
 	return s.PeersFn()
 }
 
+func (s *StatementExecutorStore) Leader() string {
+	if s.LeaderFn != nil {
+		return s.LeaderFn()
+	}
+	return ""
+}
+
+func (s *StatementExecutorStore) DeleteNode(nodeID uint64, force bool) error {
+	return s.DeleteNodeFn(nodeID, force)
+}
+
 func (s *StatementExecutorStore) Database(name string) (*meta.DatabaseInfo, error) {
 	return s.DatabaseFn(name)
 }

Godeps/_workspace/src/github.com/influxdb/influxdb/meta/store.go

@@ -254,7 +254,10 @@ func (s *Store) Open() error {
 		close(s.ready)
 	}
 
-	return nil
+	// Wait for a leader to be elected so we know the raft log is loaded
+	// and up to date
+	<-s.ready
+	return s.WaitForLeader(0)
 }
 
 // syncNodeInfo continuously tries to update the current nodes hostname
@@ -689,7 +692,7 @@ func (s *Store) handleExecConn(conn net.Conn) {
 
 		// Apply against the raft log.
 		if err := s.apply(buf); err != nil {
-			return fmt.Errorf("apply: %s", err)
+			return err
 		}
 		return nil
 	}()
@@ -820,10 +823,16 @@ func (s *Store) UpdateNode(id uint64, host string) (*NodeInfo, error) {
 }
 
 // DeleteNode removes a node from the metastore by id.
-func (s *Store) DeleteNode(id uint64) error {
+func (s *Store) DeleteNode(id uint64, force bool) error {
+	ni := s.data.Node(id)
+	if ni == nil {
+		return ErrNodeNotFound
+	}
+
 	return s.exec(internal.Command_DeleteNodeCommand, internal.E_DeleteNodeCommand_Command,
 		&internal.DeleteNodeCommand{
-			ID: proto.Uint64(id),
+			ID:    proto.Uint64(id),
+			Force: proto.Bool(force),
 		},
 	)
 }
@@ -858,6 +867,7 @@ func (s *Store) CreateDatabase(name string) (*DatabaseInfo, error) {
 	); err != nil {
 		return nil, err
 	}
+	s.Logger.Printf("database '%s' created", name)
 
 	if s.retentionAutoCreate {
 		// Read node count.
@@ -977,6 +987,7 @@ func (s *Store) CreateRetentionPolicy(database string, rpi *RetentionPolicyInfo)
 		return nil, err
 	}
 
+	s.Logger.Printf("retention policy '%s' for database '%s' created", rpi.Name, database)
 	return s.RetentionPolicy(database, rpi.Name)
 }
 
@@ -1047,8 +1058,6 @@ func (s *Store) DropRetentionPolicy(database, name string) error {
 	)
 }
 
-// FIX: CreateRetentionPolicyIfNotExists(database string, rp *RetentionPolicyInfo) (*RetentionPolicyInfo, error)
-
 // CreateShardGroup creates a new shard group in a retention policy for a given time.
 func (s *Store) CreateShardGroup(database, policy string, timestamp time.Time) (*ShardGroupInfo, error) {
 	if err := s.exec(internal.Command_CreateShardGroupCommand, internal.E_CreateShardGroupCommand_Command,
@@ -1389,38 +1398,34 @@ func (s *Store) UserCount() (count int, err error) {
 	return
 }
 
-// PrecreateShardGroups creates shard groups whose endtime is before the cutoff time passed in. This
+// PrecreateShardGroups creates shard groups whose endtime is before the 'to' time passed in, but
-// avoid the need for these shards to be created when data for the corresponding time range arrives.
+// is yet to expire before 'from'. This is to avoid the need for these shards to be created when data
-// Shard creation involves Raft consensus, and precreation avoids taking the hit at write-time.
+// for the corresponding time range arrives. Shard creation involves Raft consensus, and precreation
-func (s *Store) PrecreateShardGroups(cutoff time.Time) error {
+// avoids taking the hit at write-time.
+func (s *Store) PrecreateShardGroups(from, to time.Time) error {
 	s.read(func(data *Data) error {
 		for _, di := range data.Databases {
 			for _, rp := range di.RetentionPolicies {
-				for _, g := range rp.ShardGroups {
+				if len(rp.ShardGroups) == 0 {
-					// Check to see if it is not deleted and going to end before our interval
+					// No data was ever written to this group, or all groups have been deleted.
-					if !g.Deleted() && g.EndTime.Before(cutoff) {
+					continue
-						nextShardGroupTime := g.EndTime.Add(1 * time.Nanosecond)
+				}
+				g := rp.ShardGroups[len(rp.ShardGroups)-1] // Get the last group in time.
+				if !g.Deleted() && g.EndTime.Before(to) && g.EndTime.After(from) {
+					// Group is not deleted, will end before the future time, but is still yet to expire.
+					// This last check is important, so the system doesn't create shards groups wholly
+					// in the past.
 
-						// Check if successive shard group exists.
+					// Create successive shard group.
-						if sgi, err := s.ShardGroupByTimestamp(di.Name, rp.Name, nextShardGroupTime); err != nil {
+					nextShardGroupTime := g.EndTime.Add(1 * time.Nanosecond)
-							s.Logger.Printf("failed to check if successive shard group for group exists %d: %s",
+					if newGroup, err := s.CreateShardGroupIfNotExists(di.Name, rp.Name, nextShardGroupTime); err != nil {
-								g.ID, err.Error())
+						s.Logger.Printf("failed to create successive shard group for group %d: %s",
-							continue
+							g.ID, err.Error())
-						} else if sgi != nil && !sgi.Deleted() {
+					} else {
-							continue
+						s.Logger.Printf("new shard group %d successfully created for database %s, retention policy %s",
-						}
+							newGroup.ID, di.Name, rp.Name)
-
-						// It doesn't. Create it.
-						if newGroup, err := s.CreateShardGroupIfNotExists(di.Name, rp.Name, nextShardGroupTime); err != nil {
-							s.Logger.Printf("failed to create successive shard group for group %d: %s",
-								g.ID, err.Error())
-						} else {
-							s.Logger.Printf("new shard group %d successfully created for database %s, retention policy %s",
-								newGroup.ID, di.Name, rp.Name)
-						}
 					}
 				}
-
 			}
 		}
 		return nil
@@ -1554,7 +1559,7 @@ func (s *Store) remoteExec(b []byte) error {
 	if err := proto.Unmarshal(buf, &resp); err != nil {
 		return fmt.Errorf("unmarshal response: %s", err)
 	} else if !resp.GetOK() {
-		return fmt.Errorf("exec failed: %s", resp.GetError())
+		return lookupError(fmt.Errorf(resp.GetError()))
 	}
 
 	// Wait for local FSM to sync to index.
@@ -1707,11 +1712,14 @@ func (fsm *storeFSM) applyDeleteNodeCommand(cmd *internal.Command) interface{} {
 
 	// Copy data and update.
 	other := fsm.data.Clone()
-	if err := other.DeleteNode(v.GetID()); err != nil {
+	if err := other.DeleteNode(v.GetID(), v.GetForce()); err != nil {
 		return err
 	}
 	fsm.data = other
 
+	id := v.GetID()
+	fsm.Logger.Printf("node '%d' removed", id)
+
 	return nil
 }
 

Godeps/_workspace/src/github.com/influxdb/influxdb/meta/store_test.go

@@ -151,7 +151,7 @@ func TestStore_DeleteNode(t *testing.T) {
 	}
 
 	// Remove second node.
-	if err := s.DeleteNode(3); err != nil {
+	if err := s.DeleteNode(3, false); err != nil {
 		t.Fatal(err)
 	}
 
@@ -173,7 +173,7 @@ func TestStore_DeleteNode_ErrNodeNotFound(t *testing.T) {
 	s := MustOpenStore()
 	defer s.Close()
 
-	if err := s.DeleteNode(2); err != meta.ErrNodeNotFound {
+	if err := s.DeleteNode(2, false); err != meta.ErrNodeNotFound {
 		t.Fatalf("unexpected error: %s", err)
 	}
 }
@@ -274,6 +274,47 @@ func TestStore_CreateRetentionPolicy(t *testing.T) {
 	}
 }
 
+// Ensure the store can create and get a retention policy on a database.
+func TestStore_CreateAndGetRetentionPolicy(t *testing.T) {
+	t.Parallel()
+	s := MustOpenStore()
+	defer s.Close()
+
+	// Create an additional nodes and database.
+	if _, err := s.CreateNode("hostX"); err != nil {
+		t.Fatal(err)
+	} else if _, err := s.CreateDatabase("db0"); err != nil {
+		t.Fatal(err)
+	}
+
+	// Create policy on database.
+	if _, err := s.CreateRetentionPolicy("db0", &meta.RetentionPolicyInfo{
+		Name:     "rp0",
+		ReplicaN: 2,
+		Duration: 48 * time.Hour,
+	}); err != nil {
+		t.Fatal(err)
+	}
+
+	// Get the policy on database.
+	if rpi, err := s.RetentionPolicy("db0", "rp0"); err != nil {
+		t.Fatal(err)
+	} else if !reflect.DeepEqual(rpi, &meta.RetentionPolicyInfo{
+		Name:               "rp0",
+		ReplicaN:           2,
+		Duration:           48 * time.Hour,
+		ShardGroupDuration: 24 * time.Hour,
+	}) {
+		t.Fatalf("unexpected policy: %#v", rpi)
+	}
+
+	// Get non-existent policies.
+	if _, err := s.RetentionPolicy("db0", "rp0"); err != nil {
+		t.Fatal(err)
+	}
+
+}
+
 // Ensure the store can delete a retention policy.
 func TestStore_DropRetentionPolicy(t *testing.T) {
 	t.Parallel()
@@ -489,30 +530,57 @@ func TestStore_PrecreateShardGroup(t *testing.T) {
 	s := MustOpenStore()
 	defer s.Close()
 
-	// Create node, database, policy, & group.
+	// Create node, database, policy, & groups.
 	if _, err := s.CreateNode("host0"); err != nil {
 		t.Fatal(err)
 	} else if _, err := s.CreateDatabase("db0"); err != nil {
 		t.Fatal(err)
 	} else if _, err = s.CreateRetentionPolicy("db0", &meta.RetentionPolicyInfo{Name: "rp0", ReplicaN: 2, Duration: 1 * time.Hour}); err != nil {
 		t.Fatal(err)
-	} else if _, err := s.CreateShardGroup("db0", "rp0", time.Date(2000, time.January, 1, 0, 0, 0, 0, time.UTC)); err != nil {
+	} else if _, err = s.CreateRetentionPolicy("db0", &meta.RetentionPolicyInfo{Name: "rp1", ReplicaN: 2, Duration: 1 * time.Hour}); err != nil {
 		t.Fatal(err)
-	} else if err := s.PrecreateShardGroups(time.Date(2001, time.January, 1, 0, 0, 0, 0, time.UTC)); err != nil {
+	} else if _, err = s.CreateRetentionPolicy("db0", &meta.RetentionPolicyInfo{Name: "rp2", ReplicaN: 2, Duration: 1 * time.Hour}); err != nil {
+		t.Fatal(err)
+	} else if _, err := s.CreateShardGroup("db0", "rp0", time.Date(2001, time.January, 1, 1, 0, 0, 0, time.UTC)); err != nil {
+		t.Fatal(err)
+	} else if _, err := s.CreateShardGroup("db0", "rp1", time.Date(2000, time.January, 1, 1, 0, 0, 0, time.UTC)); err != nil {
 		t.Fatal(err)
 	}
 
+	if err := s.PrecreateShardGroups(time.Date(2001, time.January, 1, 0, 0, 0, 0, time.UTC), time.Date(2001, time.January, 1, 3, 0, 0, 0, time.UTC)); err != nil {
+		t.Fatal(err)
+	}
+
+	// rp0 should undergo precreation.
 	groups, err := s.ShardGroups("db0", "rp0")
 	if err != nil {
 		t.Fatal(err)
 	}
 	if len(groups) != 2 {
-		t.Fatalf("shard group precreation failed to create new shard group")
+		t.Fatalf("shard group precreation failed to create new shard group for rp0")
 	}
-	if groups[1].StartTime != time.Date(2000, time.January, 1, 1, 0, 0, 0, time.UTC) {
+	if groups[1].StartTime != time.Date(2001, time.January, 1, 2, 0, 0, 0, time.UTC) {
 		t.Fatalf("precreated shard group has wrong start time, exp %s, got %s",
 			time.Date(2000, time.January, 1, 1, 0, 0, 0, time.UTC), groups[1].StartTime)
 	}
+
+	// rp1 should not undergo precreation since it is completely in the past.
+	groups, err = s.ShardGroups("db0", "rp1")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(groups) != 1 {
+		t.Fatalf("shard group precreation created new shard group for rp1")
+	}
+
+	// rp2 should not undergo precreation since it has no shards.
+	groups, err = s.ShardGroups("db0", "rp2")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(groups) != 0 {
+		t.Fatalf("shard group precreation created new shard group for rp2")
+	}
 }
 
 // Ensure the store can create a new continuous query.
@@ -828,14 +896,14 @@ func TestCluster_Restart(t *testing.T) {
 		t.Fatal("no leader found")
 	}
 
-	// Add 5 more ndes, 2 should become raft peers, 3 remote raft clients
+	// Add 5 more nodes, 2 should become raft peers, 3 remote raft clients
 	for i := 0; i < 5; i++ {
 		if err := c.Join(); err != nil {
 			t.Fatalf("failed to join cluster: %v", err)
 		}
 	}
 
-	// The tests use a host host assigned listener port.  We need to re-use
+	// The tests use a host assigned listener port.  We need to re-use
 	// the original ports when the new cluster is restarted so that the existing
 	// peer store addresses can be reached.
 	addrs := []string{}
@@ -858,10 +926,25 @@ func TestCluster_Restart(t *testing.T) {
 
 	// Re-create the cluster nodes from existing disk paths and addresses
 	stores := []*Store{}
+	storeChan := make(chan *Store)
 	for i, s := range c.Stores {
-		store := MustOpenStoreWithPath(addrs[i], s.Path())
+
+		// Need to start each instance asynchronously because they have existing raft peers
+		// store.  Starting one will block indefinitely because it will not be able to become
+		// leader until another peer is available to hold an election.
+		go func(addr, path string) {
+			store := MustOpenStoreWithPath(addr, path)
+			storeChan <- store
+		}(addrs[i], s.Path())
+
+	}
+
+	// Collect up our restart meta-stores
+	for range c.Stores {
+		store := <-storeChan
 		stores = append(stores, store)
 	}
+
 	c.Stores = stores
 
 	// Wait for the cluster to stabilize

Godeps/_workspace/src/github.com/influxdb/influxdb/models/points.go

@@ -1,15 +1,27 @@
-package tsdb
+package models
 
 import (
 	"bytes"
 	"fmt"
 	"hash/fnv"
-	"math"
-	"regexp"
 	"sort"
 	"strconv"
-	"strings"
 	"time"
+
+	"github.com/influxdb/influxdb/pkg/escape"
+)
+
+var (
+	measurementEscapeCodes = map[byte][]byte{
+		',': []byte(`\,`),
+		' ': []byte(`\ `),
+	}
+
+	tagEscapeCodes = map[byte][]byte{
+		',': []byte(`\,`),
+		' ': []byte(`\ `),
+		'=': []byte(`\=`),
+	}
 )
 
 // Point defines the values that will be written to the database
@@ -34,7 +46,15 @@ type Point interface {
 	Data() []byte
 	SetData(buf []byte)
 
+	// String returns a string representation of the point object, if there is a
+	// timestamp associated with the point then it will be specified with the default
+	// precision of nanoseconds
 	String() string
+
+	// PrecisionString returns a string representation of the point object, if there
+	// is a timestamp associated with the point then it will be specified in the
+	// given unit
+	PrecisionString(precision string) string
 }
 
 // Points represents a sortable list of points by timestamp.
@@ -85,36 +105,7 @@ const (
 	minFloat64Digits = 27
 )
 
-var (
+var ()
-	// Compile the regex that detects unquoted double quote sequences
-	quoteReplacer = regexp.MustCompile(`([^\\])"`)
-
-	escapeCodes = map[byte][]byte{
-		',': []byte(`\,`),
-		'"': []byte(`\"`),
-		' ': []byte(`\ `),
-		'=': []byte(`\=`),
-	}
-
-	escapeCodesStr = map[string]string{}
-
-	measurementEscapeCodes = map[byte][]byte{
-		',': []byte(`\,`),
-		' ': []byte(`\ `),
-	}
-
-	tagEscapeCodes = map[byte][]byte{
-		',': []byte(`\,`),
-		' ': []byte(`\ `),
-		'=': []byte(`\=`),
-	}
-)
-
-func init() {
-	for k, v := range escapeCodes {
-		escapeCodesStr[string(k)] = string(v)
-	}
-}
 
 func ParsePointsString(buf string) ([]Point, error) {
 	return ParsePoints([]byte(buf))
@@ -414,7 +405,7 @@ func less(buf []byte, indices []int, i, j int) bool {
 }
 
 func isFieldEscapeChar(b byte) bool {
-	for c := range escapeCodes {
+	for c := range escape.Codes {
 		if c == b {
 			return true
 		}
@@ -569,6 +560,10 @@ func scanNumber(buf []byte, i int) (int, error) {
 	// Is negative number?
 	if i < len(buf) && buf[i] == '-' {
 		i += 1
+		// There must be more characters now, as just '-' is illegal.
+		if i == len(buf) {
+			return i, fmt.Errorf("invalid number")
+		}
 	}
 
 	// how many decimal points we've see
@@ -740,14 +735,19 @@ func skipWhitespace(buf []byte, i int) int {
 func scanLine(buf []byte, i int) (int, []byte) {
 	start := i
 	quoted := false
+	fields := false
 	for {
 		// reached the end of buf?
 		if i >= len(buf) {
 			break
 		}
 
+		if buf[i] == ' ' {
+			fields = true
+		}
+
 		// If we see a double quote, makes sure it is not escaped
-		if buf[i] == '"' && (i-1 > 0 && buf[i-1] != '\\') {
+		if fields && buf[i] == '"' && (i-1 > 0 && buf[i-1] != '\\') {
 			i += 1
 			quoted = !quoted
 			continue
@@ -891,62 +891,6 @@ func unescapeTag(in []byte) []byte {
 	return in
 }
 
-func escape(in []byte) []byte {
-	for b, esc := range escapeCodes {
-		in = bytes.Replace(in, []byte{b}, esc, -1)
-	}
-	return in
-}
-
-func escapeString(in string) string {
-	for b, esc := range escapeCodesStr {
-		in = strings.Replace(in, b, esc, -1)
-	}
-	return in
-}
-
-func unescape(in []byte) []byte {
-	i := 0
-	inLen := len(in)
-	var out []byte
-
-	for {
-		if i >= inLen {
-			break
-		}
-		if in[i] == '\\' && i+1 < inLen {
-			switch in[i+1] {
-			case ',':
-				out = append(out, ',')
-				i += 2
-				continue
-			case '"':
-				out = append(out, '"')
-				i += 2
-				continue
-			case ' ':
-				out = append(out, ' ')
-				i += 2
-				continue
-			case '=':
-				out = append(out, '=')
-				i += 2
-				continue
-			}
-		}
-		out = append(out, in[i])
-		i += 1
-	}
-	return out
-}
-
-func unescapeString(in string) string {
-	for b, esc := range escapeCodesStr {
-		in = strings.Replace(in, esc, b, -1)
-	}
-	return in
-}
-
 // escapeStringField returns a copy of in with any double quotes or
 // backslashes with escaped values
 func escapeStringField(in string) string {
@@ -1036,7 +980,7 @@ func (p *point) Name() string {
 	if p.cachedName != "" {
 		return p.cachedName
 	}
-	p.cachedName = string(unescape(p.name()))
+	p.cachedName = string(escape.Unescape(p.name()))
 	return p.cachedName
 }
 
@@ -1077,6 +1021,10 @@ func (p *point) Tags() Tags {
 			i, key = scanTo(p.key, i, '=')
 			i, value = scanTagValue(p.key, i+1)
 
+			if len(value) == 0 {
+				continue
+			}
+
 			tags[string(unescapeTag(key))] = string(unescapeTag(value))
 
 			i += 1
@@ -1162,6 +1110,14 @@ func (p *point) String() string {
 	return fmt.Sprintf("%s %s %d", p.Key(), string(p.fields), p.UnixNano())
 }
 
+func (p *point) PrecisionString(precision string) string {
+	if p.Time().IsZero() {
+		return fmt.Sprintf("%s %s", p.Key(), string(p.fields))
+	}
+	return fmt.Sprintf("%s %s %d", p.Key(), string(p.fields),
+		p.UnixNano()/p.GetPrecisionMultiplier(precision))
+}
+
 func (p *point) unmarshalBinary() Fields {
 	return newFieldsFromBinary(p.fields)
 }
@@ -1189,7 +1145,10 @@ func (t Tags) HashKey() []byte {
 	for k, v := range t {
 		ek := escapeTag([]byte(k))
 		ev := escapeTag([]byte(v))
-		escaped[string(ek)] = string(ev)
+
+		if len(ev) > 0 {
+			escaped[string(ek)] = string(ev)
+		}
 	}
 
 	// Extract keys and determine final size.
@@ -1257,7 +1216,7 @@ func newFieldsFromBinary(buf []byte) Fields {
 		if len(name) == 0 {
 			continue
 		}
-		name = unescape(name)
+		name = escape.Unescape(name)
 
 		i, valueBuf = scanFieldValue(buf, i+1)
 		if len(valueBuf) == 0 {
@@ -1291,6 +1250,10 @@ func newFieldsFromBinary(buf []byte) Fields {
 	return fields
 }
 
+// MarshalBinary encodes all the fields to their proper type and returns the binary
+// represenation
+// NOTE: uint64 is specifically not supported due to potential overflow when we decode
+// again later to an int64
 func (p Fields) MarshalBinary() []byte {
 	b := []byte{}
 	keys := make([]string, len(p))
@@ -1303,31 +1266,42 @@ func (p Fields) MarshalBinary() []byte {
 
 	for _, k := range keys {
 		v := p[k]
-		b = append(b, []byte(escapeString(k))...)
+		b = append(b, []byte(escape.String(k))...)
 		b = append(b, '=')
 		switch t := v.(type) {
 		case int:
 			b = append(b, []byte(strconv.FormatInt(int64(t), 10))...)
 			b = append(b, 'i')
-		case int32:
+		case int8:
 			b = append(b, []byte(strconv.FormatInt(int64(t), 10))...)
 			b = append(b, 'i')
-		case uint64:
+		case int16:
-			b = append(b, []byte(strconv.FormatUint(t, 10))...)
+			b = append(b, []byte(strconv.FormatInt(int64(t), 10))...)
+			b = append(b, 'i')
+		case int32:
+			b = append(b, []byte(strconv.FormatInt(int64(t), 10))...)
 			b = append(b, 'i')
 		case int64:
 			b = append(b, []byte(strconv.FormatInt(t, 10))...)
 			b = append(b, 'i')
-		case float64:
+		case uint:
-			// ensure there is a decimal in the encoded for
+			b = append(b, []byte(strconv.FormatInt(int64(t), 10))...)
-
+			b = append(b, 'i')
-			val := []byte(strconv.FormatFloat(t, 'f', -1, 64))
+		case uint8:
-			_, frac := math.Modf(t)
+			b = append(b, []byte(strconv.FormatInt(int64(t), 10))...)
-			hasDecimal := frac != 0
+			b = append(b, 'i')
+		case uint16:
+			b = append(b, []byte(strconv.FormatInt(int64(t), 10))...)
+			b = append(b, 'i')
+		case uint32:
+			b = append(b, []byte(strconv.FormatInt(int64(t), 10))...)
+			b = append(b, 'i')
+		case float32:
+			val := []byte(strconv.FormatFloat(float64(t), 'f', -1, 32))
+			b = append(b, val...)
+		case float64:
+			val := []byte(strconv.FormatFloat(t, 'f', -1, 64))
 			b = append(b, val...)
-			if !hasDecimal {
-				b = append(b, []byte(".0")...)
-			}
 		case bool:
 			b = append(b, []byte(strconv.FormatBool(t))...)
 		case []byte:

Godeps/_workspace/src/github.com/influxdb/influxdb/models/rows.go

@@ -0,0 +1,59 @@
+package models
+
+import (
+	"hash/fnv"
+	"sort"
+)
+
+// Row represents a single row returned from the execution of a statement.
+type Row struct {
+	Name    string            `json:"name,omitempty"`
+	Tags    map[string]string `json:"tags,omitempty"`
+	Columns []string          `json:"columns,omitempty"`
+	Values  [][]interface{}   `json:"values,omitempty"`
+	Err     error             `json:"err,omitempty"`
+}
+
+// SameSeries returns true if r contains values for the same series as o.
+func (r *Row) SameSeries(o *Row) bool {
+	return r.tagsHash() == o.tagsHash() && r.Name == o.Name
+}
+
+// tagsHash returns a hash of tag key/value pairs.
+func (r *Row) tagsHash() uint64 {
+	h := fnv.New64a()
+	keys := r.tagsKeys()
+	for _, k := range keys {
+		h.Write([]byte(k))
+		h.Write([]byte(r.Tags[k]))
+	}
+	return h.Sum64()
+}
+
+// tagKeys returns a sorted list of tag keys.
+func (r *Row) tagsKeys() []string {
+	a := make([]string, 0, len(r.Tags))
+	for k := range r.Tags {
+		a = append(a, k)
+	}
+	sort.Strings(a)
+	return a
+}
+
+type Rows []*Row
+
+func (p Rows) Len() int { return len(p) }
+
+func (p Rows) Less(i, j int) bool {
+	// Sort by name first.
+	if p[i].Name != p[j].Name {
+		return p[i].Name < p[j].Name
+	}
+
+	// Sort by tag set hash. Tags don't have a meaningful sort order so we
+	// just compute a hash and sort by that instead. This allows the tests
+	// to receive rows in a predictable order every time.
+	return p[i].tagsHash() < p[j].tagsHash()
+}
+
+func (p Rows) Swap(i, j int) { p[i], p[j] = p[j], p[i] }

Godeps/_workspace/src/github.com/influxdb/influxdb/monitor/README.md

@@ -0,0 +1,73 @@
+# System Monitoring
+_This functionality should be considered experimental and is subject to change._
+
+_System Monitoring_ means all statistical and diagnostic information made availabe to the user of InfluxDB system, about the system itself. Its purpose is to assist with troubleshooting and performance analysis of the database itself.
+
+## Statistics vs. Diagnostics
+A distinction is made between _statistics_ and _diagnostics_ for the purposes of monitoring. Generally a statistical quality is something that is being counted, and for which it makes sense to store persistently for historical analysis. Diagnostic information is not necessarily numerical, and may not make sense to store.
+
+An example of statistical information would be the number of points received over UDP, or the number of queries executed. Examples of diagnostic information would be a list of current Graphite TCP connections, the version of InfluxDB, or the uptime of the process.
+
+## System Statistics
+`SHOW STATS [FOR <module>]` displays statisics about subsystems within the running `influxd` process. Statistics include points received, points indexed, bytes written to disk, TCP connections handled etc. These statistics are all zero when the InfluxDB process starts. If _module_ is specified, it must be single-quoted. For example `SHOW STATS FOR 'httpd'`.
+
+All statistics are written, by default, by each node to a "monitor" database within the InfluxDB system, allowing analysis of aggregated statistical data using the standard InfluxQL language. This allows users to track the performance of their system. Importantly, this allows cluster-level statistics to be viewed, since by querying the monitor database, statistics from all nodes may be queried. This can be a very powerful approach for troubleshooting your InfluxDB system and understanding its behaviour.
+
+## System Diagnostics
+`SHOW DIAGNOSTICS [FOR <module>]` displays various diagnostic information about the `influxd` process. This information is not stored persistently within the InfluxDB system. If _module_ is specified, it must be single-quoted. For example `SHOW STATS FOR 'build'`.
+
+## Standard expvar support
+All statistical information is available at HTTP API endpoint `/debug/vars`, in [expvar](https://golang.org/pkg/expvar/) format, allowing external systems to monitor an InfluxDB node. By default, the full path to this endpoint is `http://localhost:8086/debug/vars`.
+
+## Configuration
+The `monitor` module allows the following configuration:
+
+ * Whether to write statistical and diagnostic information to an InfluxDB system. This is enabled by default.
+ * The name of the database to where this information should be written. Defaults to `_internal`. The information is written to the default retention policy for the given database.
+ * The name of the retention policy, along with full configuration control of the retention policy, if the default retention policy is not suitable.
+ * The rate at which this information should be written. The default rate is once every 10 seconds.
+
+# Design and Implementation
+
+A new module named `monitor` supports all basic statistics and diagnostic functionality. This includes:
+
+ * Allowing other modules to register statistics and diagnostics information, allowing it to be accessed on demand by the `monitor` module.
+ * Serving the statistics and diagnostic information to the user, in response to commands such as `SHOW DIAGNOSTICS`.
+ * Expose standard Go runtime information such as garbage collection statistics.
+ * Make all collected expvar data via HTTP, for collection by 3rd-party tools.
+ * Writing the statistical information to the "monitor" database, for query purposes.
+
+## Registering statistics and diagnostics
+
+To export statistical information with the `monitor` system, code simply calls `influxdb.NewStatistics()` and receives an `expvar.Map` instance in response. This object can then be used to store statistics.
+
+For example, if you have a component called `Service`, you can statistics like so:
+
+```
+import (
+     "expvar"
+     "github.com/influxdb/influxdb"
+)
+.
+.
+.
+.
+type Service struct {
+  ....some other fields....
+  statMap *expvar.Map         /// Add a map of type *expvar.Map. Check GoDocs for how to use this.
+}
+
+
+func NewService() *Service {
+  s = &NewService{}
+  s.statMap = NewStatistics(key, name, tags)
+}
+```
+When calling `NewStatistics` `key` should be unique for the Service instance (if a network service, the protocol and binding port are good to include in the key). `name` will be the name of the Measurement used to store these statistics. Finally, when these statistics are written to the `monitor` database, all points will be tagged with `tags`. A value of nil for `tags` is legal. 
+
+To register diagnostic information, `monitor.RegisterDiagnosticsClient` is called, passing a `influxdb.monitor.DiagsClient` object to `monitor`. Implementing the `influxdb.monitor.DiagsClient` interface requires that your component have function returning diagnostic information in specific form, so that it can be displayed by the `monitor` system.
+
+## expvar
+Statistical information is gathered by each package using [expvar](https://golang.org/pkg/expvar). Each package registers a map using its package name.
+
+Due to the nature of `expvar`, statistical information is reset to its initial state when a server is restarted.

Godeps/_workspace/src/github.com/influxdb/influxdb/monitor/build_info.go

@@ -0,0 +1,20 @@
+package monitor
+
+// system captures build diagnostics
+type build struct {
+	Version string
+	Commit  string
+	Branch  string
+	Time    string
+}
+
+func (b *build) Diagnostics() (*Diagnostic, error) {
+	diagnostics := map[string]interface{}{
+		"Version":    b.Version,
+		"Commit":     b.Commit,
+		"Branch":     b.Branch,
+		"Build Time": b.Time,
+	}
+
+	return DiagnosticFromMap(diagnostics), nil
+}

Godeps/_workspace/src/github.com/influxdb/influxdb/monitor/config.go

@@ -0,0 +1,35 @@
+package monitor
+
+import (
+	"time"
+
+	"github.com/influxdb/influxdb/toml"
+)
+
+const (
+	// DefaultStoreEnabled is whether the system writes gathered information in
+	// an InfluxDB system for historical analysis.
+	DefaultStoreEnabled = true
+
+	// DefaultStoreDatabase is the name of the database where gathered information is written
+	DefaultStoreDatabase = "_internal"
+
+	// DefaultStoreInterval is the period between storing gathered information.
+	DefaultStoreInterval = 10 * time.Second
+)
+
+// Config represents the configuration for the monitor service.
+type Config struct {
+	StoreEnabled  bool          `toml:"store-enabled"`
+	StoreDatabase string        `toml:"store-database"`
+	StoreInterval toml.Duration `toml:"store-interval"`
+}
+
+// NewConfig returns an instance of Config with defaults.
+func NewConfig() Config {
+	return Config{
+		StoreEnabled:  true,
+		StoreDatabase: DefaultStoreDatabase,
+		StoreInterval: toml.Duration(DefaultStoreInterval),
+	}
+}

Godeps/_workspace/src/github.com/influxdb/influxdb/monitor/config_test.go

@@ -0,0 +1,30 @@
+package monitor_test
+
+import (
+	"testing"
+	"time"
+
+	"github.com/BurntSushi/toml"
+	"github.com/influxdb/influxdb/monitor"
+)
+
+func TestConfig_Parse(t *testing.T) {
+	// Parse configuration.
+	var c monitor.Config
+	if _, err := toml.Decode(`
+store-enabled=true
+store-database="the_db"
+store-interval="10m"
+`, &c); err != nil {
+		t.Fatal(err)
+	}
+
+	// Validate configuration.
+	if !c.StoreEnabled {
+		t.Fatalf("unexpected store-enabled: %v", c.StoreEnabled)
+	} else if c.StoreDatabase != "the_db" {
+		t.Fatalf("unexpected store-database: %s", c.StoreDatabase)
+	} else if time.Duration(c.StoreInterval) != 10*time.Minute {
+		t.Fatalf("unexpected store-interval:  %s", c.StoreInterval)
+	}
+}