Load external .so plugins

support for the Go 1.8 shared object feature of loading external
plugins.

this support relies on the developer defining a `Plugin` symbol in their
.so file that is a telegraf plugin interface.

So instead of the plugin developer "Adding" their own plugin to the
telegraf registry, telegraf loads the .so, looks up the Plugin symbol,
and then adds it if it finds it.

The name of the plugin is determined by telegraf, and is namespaced
based on the filename and path.

see #1717

CHANGELOG.md

@@ -56,6 +56,7 @@ be deprecated eventually.
 - [#2339](https://github.com/influxdata/telegraf/pull/2339): Increment gather_errors for all errors emitted by inputs.
 - [#2071](https://github.com/influxdata/telegraf/issues/2071): Use official docker SDK.
 - [#1678](https://github.com/influxdata/telegraf/pull/1678): Add AMQP consumer input plugin
+- [#2501](https://github.com/influxdata/telegraf/pull/2501): Support DEAD(X) state in system input plugin.
 
 ### Bugfixes
 

Makefile

@@ -15,8 +15,7 @@ windows: prepare-windows build-windows
 
 # Only run the build (no dependency grabbing)
 build:
-	go install -ldflags \
+	go install -ldflags "-X main.version=$(VERSION) -X main.commit=$(COMMIT) -X main.branch=$(BRANCH)" ./...
-		"-X main.version=$(VERSION) -X main.commit=$(COMMIT) -X main.branch=$(BRANCH)" ./...
 
 build-windows:
 	GOOS=windows GOARCH=amd64 go build -o telegraf.exe -ldflags \

cmd/telegraf/telegraf.go

@@ -6,19 +6,27 @@ import (
 	"log"
 	"os"
 	"os/signal"
+	"path"
+	"path/filepath"
+	"plugin"
 	"runtime"
 	"strings"
 	"syscall"
 
+	"github.com/influxdata/telegraf"
 	"github.com/influxdata/telegraf/agent"
 	"github.com/influxdata/telegraf/internal/config"
 	"github.com/influxdata/telegraf/logger"
-	_ "github.com/influxdata/telegraf/plugins/aggregators/all"
+	"github.com/influxdata/telegraf/plugins/aggregators"
 	"github.com/influxdata/telegraf/plugins/inputs"
-	_ "github.com/influxdata/telegraf/plugins/inputs/all"
 	"github.com/influxdata/telegraf/plugins/outputs"
+	"github.com/influxdata/telegraf/plugins/processors"
+
+	_ "github.com/influxdata/telegraf/plugins/aggregators/all"
+	_ "github.com/influxdata/telegraf/plugins/inputs/all"
 	_ "github.com/influxdata/telegraf/plugins/outputs/all"
 	_ "github.com/influxdata/telegraf/plugins/processors/all"
+
 	"github.com/kardianos/service"
 )
 
@@ -50,23 +58,29 @@ var fUsage = flag.String("usage", "",
 	"print usage for a plugin, ie, 'telegraf -usage mysql'")
 var fService = flag.String("service", "",
 	"operate on the service")
+var fPlugins = flag.String("external-plugins", "",
+	"path to directory containing external plugins")
 
 // Telegraf version, populated linker.
 //   ie, -ldflags "-X main.version=`git describe --always --tags`"
 var (
-	version string
+	version   string
-	commit  string
+	commit    string
-	branch  string
+	branch    string
+	goversion string
 )
 
 func init() {
-	// If commit or branch are not set, make that clear.
+	if version == "" {
+		version = "unknown"
+	}
 	if commit == "" {
 		commit = "unknown"
 	}
 	if branch == "" {
 		branch = "unknown"
 	}
+	goversion = runtime.Version() + " " + runtime.GOOS + "/" + runtime.GOARCH
 }
 
 const usage = `Telegraf, The plugin-driven server agent for collecting and reporting metrics.
@@ -83,6 +97,9 @@ The commands & flags are:
   --config <file>     configuration file to load
   --test              gather metrics once, print them to stdout, and exit
   --config-directory  directory containing additional *.conf files
+  --external-plugins  directory containing *.so files, this directory will be
+                        searched recursively. Any Plugin found will be loaded
+                        and namespaced.
   --input-filter      filter the input plugins to enable, separator is :
   --output-filter     filter the output plugins to enable, separator is :
   --usage             print usage for a plugin, ie, 'telegraf --usage mysql'
@@ -188,7 +205,8 @@ func reloadLoop(
 			}
 		}()
 
-		log.Printf("I! Starting Telegraf (version %s)\n", version)
+		log.Printf("I! Starting Telegraf (version %s), Go version: %s\n",
+			version, goversion)
 		log.Printf("I! Loaded outputs: %s", strings.Join(c.OutputNames(), " "))
 		log.Printf("I! Loaded inputs: %s", strings.Join(c.InputNames(), " "))
 		log.Printf("I! Tags enabled: %s", c.ListTags())
@@ -246,10 +264,90 @@ func (p *program) Stop(s service.Service) error {
 	return nil
 }
 
+// loadExternalPlugins loads external plugins from shared libraries (.so, .dll, etc.)
+// in the specified directory.
+func loadExternalPlugins(rootDir string) error {
+	return filepath.Walk(rootDir, func(pth string, info os.FileInfo, err error) error {
+		// Stop if there was an error.
+		if err != nil {
+			return err
+		}
+
+		// Ignore directories.
+		if info.IsDir() {
+			return nil
+		}
+
+		// Ignore files that aren't shared libraries.
+		ext := strings.ToLower(path.Ext(pth))
+		if ext != ".so" && ext != ".dll" {
+			return nil
+		}
+
+		// name will be the path to the plugin file beginning at the root
+		// directory, minus the extension.
+		// ie, if the plugin file is /opt/telegraf-plugins/group1/foo.so, name
+		//     will be "group1/foo"
+		name := strings.TrimPrefix(strings.TrimPrefix(pth, rootDir), string(os.PathSeparator))
+		name = strings.TrimSuffix(name, filepath.Ext(pth))
+		name = "external" + string(os.PathSeparator) + name
+
+		// Load plugin.
+		p, err := plugin.Open(pth)
+		if err != nil {
+			return fmt.Errorf("error loading [%s]: %s", pth, err)
+		}
+
+		s, err := p.Lookup("Plugin")
+		if err != nil {
+			fmt.Printf("ERROR Could not find 'Plugin' symbol in [%s]\n", pth)
+			return nil
+		}
+
+		switch tplugin := s.(type) {
+		case *telegraf.Input:
+			fmt.Printf("Adding external input plugin: %s\n", name)
+			inputs.Add(name, func() telegraf.Input { return *tplugin })
+		case *telegraf.Output:
+			fmt.Printf("Adding external output plugin: %s\n", name)
+			outputs.Add(name, func() telegraf.Output { return *tplugin })
+		case *telegraf.Processor:
+			fmt.Printf("Adding external processor plugin: %s\n", name)
+			processors.Add(name, func() telegraf.Processor { return *tplugin })
+		case *telegraf.Aggregator:
+			fmt.Printf("Adding external aggregator plugin: %s\n", name)
+			aggregators.Add(name, func() telegraf.Aggregator { return *tplugin })
+		default:
+			fmt.Printf("ERROR: 'Plugin' symbol from [%s] is not a telegraf interface, it has type: %T\n", pth, tplugin)
+		}
+
+		return nil
+	})
+}
+
+func printVersion() {
+	fmt.Printf(`Telegraf %s
+  branch:     %s
+  commit:     %s
+  go version: %s
+`, version, branch, commit, goversion)
+}
+
 func main() {
 	flag.Usage = func() { usageExit(0) }
 	flag.Parse()
 	args := flag.Args()
+	// Load external plugins, if requested.
+	if *fPlugins != "" {
+		pluginsDir, err := filepath.Abs(*fPlugins)
+		if err != nil {
+			log.Fatal(err.Error())
+		}
+		fmt.Printf("Loading external plugins from: %s\n", pluginsDir)
+		if err := loadExternalPlugins(*fPlugins); err != nil {
+			log.Fatal(err.Error())
+		}
+	}
 
 	inputFilters, outputFilters := []string{}, []string{}
 	if *fInputFilters != "" {
@@ -270,7 +368,7 @@ func main() {
 	if len(args) > 0 {
 		switch args[0] {
 		case "version":
-			fmt.Printf("Telegraf v%s (git: %s %s)\n", version, branch, commit)
+			printVersion()
 			return
 		case "config":
 			config.PrintSampleConfig(
@@ -298,7 +396,7 @@ func main() {
 		}
 		return
 	case *fVersion:
-		fmt.Printf("Telegraf v%s (git: %s %s)\n", version, branch, commit)
+		printVersion()
 		return
 	case *fSampleConfig:
 		config.PrintSampleConfig(

internal/config/config.go

@@ -39,6 +39,14 @@ var (
 
 	// envVarRe is a regex to find environment variables in the config file
 	envVarRe = regexp.MustCompile(`\$\w+`)
+
+	// addQuoteRe is a regex for finding and adding quotes around / characters
+	// when they are used for distinguishing external plugins.
+	// ie, a ReplaceAll() with this pattern will be used to turn this:
+	//     [[inputs.external/test/example]]
+	//  to
+	//     [[inputs."external/test/example"]]
+	addQuoteRe = regexp.MustCompile(`(\[?\[?inputs|outputs|processors|aggregators)\.(external\/[^.\]]+)`)
 )
 
 // Config specifies the URL/user/password for the database that telegraf
@@ -704,6 +712,9 @@ func parseFile(fpath string) (*ast.Table, error) {
 		}
 	}
 
+	// add quotes around external plugin paths.
+	contents = addQuoteRe.ReplaceAll(contents, []byte(`$1."$2"`))
+
 	return toml.Parse(contents)
 }
 

plugins/inputs/postgresql/README.md

@@ -30,21 +30,21 @@ _* value ignored and therefore not recorded._
 
 More information about the meaning of these metrics can be found in the [PostgreSQL Documentation](http://www.postgresql.org/docs/9.2/static/monitoring-stats.html#PG-STAT-DATABASE-VIEW)
 
-## Configruation
+## Configuration
 Specify address via a url matching:
 
   `postgres://[pqgotest[:password]]@localhost[/dbname]?sslmode=[disable|verify-ca|verify-full]`
 
 All connection parameters are optional. Without the dbname parameter, the driver will default to a database with the same name as the user. This dbname is just for instantiating a connection with the server and doesn't restrict the databases we are trying to grab metrics for.
-  
+
 A  list of databases to explicitly ignore.  If not specified, metrics for all databases are gathered.  Do NOT use with the 'databases' option.
 
   `ignored_databases = ["postgres", "template0", "template1"]`
-  
+
 A list of databases to pull metrics about. If not specified, metrics for all databases are gathered.  Do NOT use with the 'ignored_databases' option.
 
   `databases = ["app_production", "testing"]`
-  
+
 ### Configuration example
 ```
 [[inputs.postgresql]]

plugins/inputs/system/PROCESSES_README.md

@@ -23,6 +23,7 @@ it requires access to execute `ps`.
     - stopped
     - total
     - zombie
+    - dead
     - wait (freebsd only)
     - idle (bsd only)
     - paging (linux only)
@@ -39,6 +40,7 @@ Linux  FreeBSD  Darwin  meaning
   R       R       R     running
   S       S       S     sleeping
   Z       Z       Z     zombie
+  X      none    none   dead
   T       T       T     stopped
  none     I       I     idle (sleeping for longer than about 20 seconds)
   D      D,L      U     blocked (waiting in uninterruptible sleep, or locked)
@@ -54,5 +56,5 @@ None
 ```
 $ telegraf -config ~/ws/telegraf.conf -input-filter processes -test
 * Plugin: processes, Collection 1
-> processes blocked=8i,running=1i,sleeping=265i,stopped=0i,total=274i,zombie=0i,paging=0i,total_threads=687i 1457478636980905042
+> processes blocked=8i,running=1i,sleeping=265i,stopped=0i,total=274i,zombie=0i,dead=0i,paging=0i,total_threads=687i 1457478636980905042
 ```

plugins/inputs/system/processes.go

@@ -81,6 +81,7 @@ func getEmptyFields() map[string]interface{} {
 	case "openbsd":
 		fields["idle"] = int64(0)
 	case "linux":
+		fields["dead"] = int64(0)
 		fields["paging"] = int64(0)
 		fields["total_threads"] = int64(0)
 	}
@@ -107,6 +108,8 @@ func (p *Processes) gatherFromPS(fields map[string]interface{}) error {
 			fields["blocked"] = fields["blocked"].(int64) + int64(1)
 		case 'Z':
 			fields["zombies"] = fields["zombies"].(int64) + int64(1)
+		case 'X':
+			fields["dead"] = fields["dead"].(int64) + int64(1)
 		case 'T':
 			fields["stopped"] = fields["stopped"].(int64) + int64(1)
 		case 'R':
@@ -164,6 +167,8 @@ func (p *Processes) gatherFromProc(fields map[string]interface{}) error {
 			fields["blocked"] = fields["blocked"].(int64) + int64(1)
 		case 'Z':
 			fields["zombies"] = fields["zombies"].(int64) + int64(1)
+		case 'X':
+			fields["dead"] = fields["dead"].(int64) + int64(1)
 		case 'T', 't':
 			fields["stopped"] = fields["stopped"].(int64) + int64(1)
 		case 'W':