Support Go execd plugins with shim (#7283)

plugins/inputs/execd/shim/README.md

@@ -0,0 +1,48 @@
+# Telegraf Execd Go Shim
+
+The goal of this _shim_ is to make it trivial to extract an internal input plugin
+out to a stand-alone repo for the purpose of compiling it as a separate app and
+running it from the inputs.execd plugin.
+
+The execd-shim is still experimental and the interface may change in the future.
+Especially as the concept expands to prcoessors, aggregators, and outputs.
+
+## Steps to externalize a plugin
+
+1. Move the project to an external repo, optionally preserving the
+  _plugins/inputs/plugin_name_ folder structure. For an example of what this might
+  look at, take a look at [ssoroka/rand](https://github.com/ssoroka/rand) or
+  [danielnelson/telegraf-plugins](https://github.com/danielnelson/telegraf-plugins)
+1. Copy [main.go](./example/cmd/main.go) into your project under the cmd folder.
+  This will be the entrypoint to the plugin when run as a stand-alone program, and
+  it will call the shim code for you to make that happen.
+1. Edit the main.go file to import your plugin. Within Telegraf this would have
+  been done in an all.go file, but here we don't split the two apart, and the change
+  just goes in the top of main.go. If you skip this step, your plugin will do nothing.
+1. Optionally add a [plugin.conf](./example/cmd/plugin.conf) for configuration
+  specific to your plugin. Note that this config file **must be separate from the
+  rest of the config for Telegraf, and must not be in a shared directory where
+  Telegraf is expecting to load all configs**. If Telegraf reads this config file
+  it will not know which plugin it relates to.
+
+## Steps to build and run your plugin
+
+1. Build the cmd/main.go. For my rand project this looks like `go build -o rand cmd/main.go`
+1. Test out the binary if you haven't done this yet. eg `./rand -config plugin.conf`
+  Depending on your polling settings and whether you implemented a service plugin or
+  an input gathering plugin, you may see data right away, or you may have to hit enter
+  first, or wait for your poll duration to elapse, but the metrics will be written to
+  STDOUT. Ctrl-C to end your test.
+1. Configure Telegraf to call your new plugin binary. eg:
+
+```
+[[inputs.execd]]
+  command = ["/path/to/rand", "-config", "/path/to/plugin.conf"]
+  signal = "none"
+```
+
+## Congratulations!
+
+You've done it! Consider publishing your plugin to github and open a Pull Request
+back to the Telegraf repo letting us know about the availability of your
+[external plugin](https://github.com/influxdata/telegraf/blob/master/EXTERNAL_PLUGINS.md).

plugins/inputs/execd/shim/example/cmd/main.go

@@ -0,0 +1,60 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"os"
+	"time"
+
+	// TODO: import your plugins
+	// _ "github.com/my_github_user/my_plugin_repo/plugins/inputs/mypluginname"
+
+	"github.com/influxdata/telegraf/plugins/inputs/execd/shim"
+)
+
+var pollInterval = flag.Duration("poll_interval", 1*time.Second, "how often to send metrics")
+var pollIntervalDisabled = flag.Bool("poll_interval_disabled", false, "how often to send metrics")
+var configFile = flag.String("config", "", "path to the config file for this plugin")
+var err error
+
+// This is designed to be simple; Just change the import above and you're good.
+//
+// However, if you want to do all your config in code, you can like so:
+//
+// // initialize your plugin with any settngs you want
+// myInput := &mypluginname.MyPlugin{
+// 	DefaultSettingHere: 3,
+// }
+//
+// shim := shim.New()
+//
+// shim.AddInput(myInput)
+//
+// // now the shim.Run() call as below.
+//
+func main() {
+	// parse command line options
+	flag.Parse()
+	if *pollIntervalDisabled {
+		*pollInterval = shim.PollIntervalDisabled
+	}
+
+	// create the shim. This is what will run your plugins.
+	shim := shim.New()
+
+	// If no config is specified, all imported plugins are loaded.
+	// otherwise follow what the config asks for.
+	// Check for settings from a config toml file,
+	// (or just use whatever plugins were imported above)
+	err = shim.LoadConfig(configFile)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Err loading input: %s\n", err)
+		os.Exit(1)
+	}
+
+	// run the input plugin(s) until stdin closes or we receive a termination signal
+	if err := shim.Run(*pollInterval); err != nil {
+		fmt.Fprintf(os.Stderr, "Err: %s\n", err)
+		os.Exit(1)
+	}
+}

plugins/inputs/execd/shim/example/cmd/plugin.conf

@@ -0,0 +1,2 @@
+[[inputs.my_plugin_name]]
+	value_name = "value"

plugins/inputs/execd/shim/goshim.go

@@ -0,0 +1,278 @@
+package shim
+
+import (
+	"bufio"
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"os/signal"
+	"sync"
+	"syscall"
+	"time"
+
+	"github.com/BurntSushi/toml"
+	"github.com/influxdata/telegraf"
+	"github.com/influxdata/telegraf/agent"
+	"github.com/influxdata/telegraf/plugins/inputs"
+	"github.com/influxdata/telegraf/plugins/serializers/influx"
+)
+
+type empty struct{}
+
+var (
+	gatherPromptChans []chan empty
+	stdout            io.Writer = os.Stdout
+	stdin             io.Reader = os.Stdin
+)
+
+const (
+	// PollIntervalDisabled is used to indicate that you want to disable polling,
+	// as opposed to duration 0 meaning poll constantly.
+	PollIntervalDisabled = time.Duration(0)
+)
+
+type Shim struct {
+	Inputs []telegraf.Input
+}
+
+func New() *Shim {
+	return &Shim{}
+}
+
+// AddInput adds the input to the shim. Later calls to Run() will run this input.
+func (s *Shim) AddInput(input telegraf.Input) error {
+	if p, ok := input.(telegraf.Initializer); ok {
+		err := p.Init()
+		if err != nil {
+			return fmt.Errorf("failed to init input: %s", err)
+		}
+	}
+
+	s.Inputs = append(s.Inputs, input)
+	return nil
+}
+
+// AddInputs adds multiple inputs to the shim. Later calls to Run() will run these.
+func (s *Shim) AddInputs(newInputs []telegraf.Input) error {
+	for _, inp := range newInputs {
+		if err := s.AddInput(inp); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// Run the input plugins..
+func (s *Shim) Run(pollInterval time.Duration) error {
+	wg := sync.WaitGroup{}
+	quit := make(chan os.Signal, 1)
+	signal.Notify(quit, syscall.SIGINT, syscall.SIGTERM)
+
+	collectMetricsPrompt := make(chan os.Signal, 1)
+	listenForCollectMetricsSignals(collectMetricsPrompt)
+
+	wg.Add(1) // wait for the metric channel to close
+	metricCh := make(chan telegraf.Metric, 1)
+
+	serializer := influx.NewSerializer()
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	for _, input := range s.Inputs {
+		wrappedInput := inputShim{Input: input}
+
+		acc := agent.NewAccumulator(wrappedInput, metricCh)
+		acc.SetPrecision(time.Nanosecond)
+
+		if serviceInput, ok := input.(telegraf.ServiceInput); ok {
+			if err := serviceInput.Start(acc); err != nil {
+				return fmt.Errorf("failed to start input: %s", err)
+			}
+		}
+		gatherPromptCh := make(chan empty, 1)
+		gatherPromptChans = append(gatherPromptChans, gatherPromptCh)
+		wg.Add(1)
+		go func(input telegraf.Input) {
+			startGathering(ctx, input, acc, gatherPromptCh, pollInterval)
+			if serviceInput, ok := input.(telegraf.ServiceInput); ok {
+				serviceInput.Stop()
+			}
+			wg.Done()
+		}(input)
+	}
+
+	go stdinCollectMetricsPrompt(ctx, collectMetricsPrompt)
+
+loop:
+	for {
+		select {
+		case <-quit:
+			// cancel, but keep looping until the metric channel closes.
+			cancel()
+		case <-collectMetricsPrompt:
+			collectMetrics(ctx)
+		case m, open := <-metricCh:
+			if !open {
+				wg.Done()
+				break loop
+			}
+			b, err := serializer.Serialize(m)
+			if err != nil {
+				return fmt.Errorf("failed to serialize metric: %s", err)
+			}
+			// Write this to stdout
+			fmt.Fprint(stdout, string(b))
+		}
+	}
+
+	wg.Wait()
+	return nil
+}
+
+func hasQuit(ctx context.Context) bool {
+	select {
+	case <-ctx.Done():
+		return true
+	default:
+		return false
+	}
+}
+
+func stdinCollectMetricsPrompt(ctx context.Context, collectMetricsPrompt chan<- os.Signal) {
+	s := bufio.NewScanner(stdin)
+	// for every line read from stdin, make sure we're not supposed to quit,
+	// then push a message on to the collectMetricsPrompt
+	for s.Scan() {
+		// first check if we should quit
+		if hasQuit(ctx) {
+			return
+		}
+
+		// now push a non-blocking message to trigger metric collection.
+		pushCollectMetricsRequest(collectMetricsPrompt)
+	}
+}
+
+// pushCollectMetricsRequest pushes a non-blocking (nil) message to the
+// collectMetricsPrompt channel to trigger metric collection.
+// The channel is defined with a buffer of 1, so if it's full, duplicated
+// requests are discarded.
+func pushCollectMetricsRequest(collectMetricsPrompt chan<- os.Signal) {
+	select {
+	case collectMetricsPrompt <- nil:
+	default:
+	}
+}
+
+func collectMetrics(ctx context.Context) {
+	if hasQuit(ctx) {
+		return
+	}
+	for i := 0; i < len(gatherPromptChans); i++ {
+		// push a message out to each channel to collect metrics. don't block.
+		select {
+		case gatherPromptChans[i] <- empty{}:
+		default:
+		}
+	}
+}
+
+func startGathering(ctx context.Context, input telegraf.Input, acc telegraf.Accumulator, gatherPromptCh <-chan empty, pollInterval time.Duration) {
+	if pollInterval == PollIntervalDisabled {
+		return // don't poll
+	}
+	t := time.NewTicker(pollInterval)
+	defer t.Stop()
+	for {
+		// give priority to stopping.
+		if hasQuit(ctx) {
+			return
+		}
+		// see what's up
+		select {
+		case <-ctx.Done():
+			return
+		case <-gatherPromptCh:
+			if err := input.Gather(acc); err != nil {
+				fmt.Fprintf(os.Stderr, "failed to gather metrics: %s", err)
+			}
+		case <-t.C:
+			if err := input.Gather(acc); err != nil {
+				fmt.Fprintf(os.Stderr, "failed to gather metrics: %s", err)
+			}
+		}
+	}
+}
+
+// LoadConfig loads and adds the inputs to the shim
+func (s *Shim) LoadConfig(filePath *string) error {
+	loadedInputs, err := LoadConfig(filePath)
+	if err != nil {
+		return err
+	}
+	return s.AddInputs(loadedInputs)
+}
+
+// DefaultImportedPlugins defaults to whatever plugins happen to be loaded and
+// have registered themselves with the registry. This makes loading plugins
+// without having to define a config dead easy.
+func DefaultImportedPlugins() (i []telegraf.Input, e error) {
+	for _, inputCreatorFunc := range inputs.Inputs {
+		i = append(i, inputCreatorFunc())
+	}
+	return i, nil
+}
+
+// LoadConfig loads the config and returns inputs that later need to be loaded.
+func LoadConfig(filePath *string) ([]telegraf.Input, error) {
+	if filePath == nil {
+		return DefaultImportedPlugins()
+	}
+
+	b, err := ioutil.ReadFile(*filePath)
+	if err != nil {
+		return nil, err
+	}
+
+	conf := struct {
+		Inputs map[string][]toml.Primitive
+	}{}
+
+	md, err := toml.Decode(string(b), &conf)
+	if err != nil {
+		return nil, err
+	}
+
+	loadedInputs, err := loadConfigIntoInputs(md, conf.Inputs)
+
+	if len(md.Undecoded()) > 0 {
+		fmt.Fprintf(stdout, "Some plugins were loaded but not used: %q\n", md.Undecoded())
+	}
+	return loadedInputs, err
+}
+
+func loadConfigIntoInputs(md toml.MetaData, inputConfigs map[string][]toml.Primitive) ([]telegraf.Input, error) {
+	renderedInputs := []telegraf.Input{}
+
+	for name, primitives := range inputConfigs {
+		inputCreator, ok := inputs.Inputs[name]
+		if !ok {
+			return nil, errors.New("unknown input " + name)
+		}
+
+		for _, primitive := range primitives {
+			inp := inputCreator()
+			// Parse specific configuration
+			if err := md.PrimitiveDecode(primitive, inp); err != nil {
+				return nil, err
+			}
+
+			renderedInputs = append(renderedInputs, inp)
+		}
+	}
+	return renderedInputs, nil
+}

plugins/inputs/execd/shim/goshim_notwindows.go

@@ -0,0 +1,14 @@
+// +build !windows
+
+package shim
+
+import (
+	"os"
+	"os/signal"
+	"syscall"
+)
+
+func listenForCollectMetricsSignals(collectMetricsPrompt chan os.Signal) {
+	// just listen to all the signals.
+	signal.Notify(collectMetricsPrompt, syscall.SIGHUP, syscall.SIGUSR1, syscall.SIGUSR2)
+}

plugins/inputs/execd/shim/goshim_windows.go

@@ -0,0 +1,13 @@
+// +build windows
+
+package shim
+
+import (
+	"os"
+	"os/signal"
+	"syscall"
+)
+
+func listenForCollectMetricsSignals(collectMetricsPrompt chan os.Signal) {
+	signal.Notify(collectMetricsPrompt, syscall.SIGHUP)
+}

plugins/inputs/execd/shim/input.go

@@ -0,0 +1,20 @@
+package shim
+
+import "github.com/influxdata/telegraf"
+
+// inputShim implements the MetricMaker interface.
+type inputShim struct {
+	Input telegraf.Input
+}
+
+func (i inputShim) LogName() string {
+	return ""
+}
+
+func (i inputShim) MakeMetric(m telegraf.Metric) telegraf.Metric {
+	return m // don't need to do anything to it.
+}
+
+func (i inputShim) Log() telegraf.Logger {
+	return nil
+}

plugins/inputs/execd/shim/shim_posix_test.go

@@ -0,0 +1,76 @@
+// +build !windows
+
+package shim
+
+import (
+	"bytes"
+	"context"
+	"os"
+	"runtime"
+	"strings"
+	"syscall"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestShimUSR1SignalingWorks(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		t.Skip()
+		return
+	}
+	stdoutBytes := bytes.NewBufferString("")
+	stdout = stdoutBytes
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	wait := runInputPlugin(t, 40*time.Second)
+
+	// sleep a bit to avoid a race condition where the input hasn't loaded yet.
+	time.Sleep(10 * time.Millisecond)
+
+	// signal USR1 to yourself.
+	pid := os.Getpid()
+	process, err := os.FindProcess(pid)
+	require.NoError(t, err)
+
+	go func() {
+		// On slow machines this signal can fire before the service comes up.
+		// rather than depend on accurate sleep times, we'll just retry sending
+		// the signal every so often until it goes through.
+		for {
+			select {
+			case <-ctx.Done():
+				return // test is done
+			default:
+				// test isn't done, keep going.
+				process.Signal(syscall.SIGUSR1)
+				time.Sleep(200 * time.Millisecond)
+			}
+		}
+	}()
+
+	timeout := time.NewTimer(10 * time.Second)
+
+	select {
+	case <-wait:
+	case <-timeout.C:
+		require.Fail(t, "Timeout waiting for metric to arrive")
+	}
+
+	for stdoutBytes.Len() == 0 {
+		select {
+		case <-timeout.C:
+			require.Fail(t, "Timeout waiting to read metric from stdout")
+			return
+		default:
+			time.Sleep(10 * time.Millisecond)
+		}
+	}
+
+	out := string(stdoutBytes.Bytes())
+	require.Contains(t, out, "\n")
+	metricLine := strings.Split(out, "\n")[0]
+	require.Equal(t, "measurement,tag=tag field=1i 1234000005678", metricLine)
+}

plugins/inputs/execd/shim/shim_test.go

@@ -0,0 +1,119 @@
+package shim
+
+import (
+	"bytes"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/influxdata/telegraf"
+)
+
+func TestShimWorks(t *testing.T) {
+	stdoutBytes := bytes.NewBufferString("")
+	stdout = stdoutBytes
+
+	timeout := time.NewTimer(10 * time.Second)
+	wait := runInputPlugin(t, 10*time.Millisecond)
+
+	select {
+	case <-wait:
+	case <-timeout.C:
+		require.Fail(t, "Timeout waiting for metric to arrive")
+	}
+	for stdoutBytes.Len() == 0 {
+		select {
+		case <-timeout.C:
+			require.Fail(t, "Timeout waiting to read metric from stdout")
+			return
+		default:
+			time.Sleep(10 * time.Millisecond)
+		}
+	}
+
+	out := string(stdoutBytes.Bytes())
+	require.Contains(t, out, "\n")
+	metricLine := strings.Split(out, "\n")[0]
+	require.Equal(t, "measurement,tag=tag field=1i 1234000005678", metricLine)
+}
+
+func TestShimStdinSignalingWorks(t *testing.T) {
+	stdoutBytes := bytes.NewBufferString("")
+	stdout = stdoutBytes
+	stdinBytes := bytes.NewBufferString("")
+	stdin = stdinBytes
+
+	timeout := time.NewTimer(10 * time.Second)
+	wait := runInputPlugin(t, 40*time.Second)
+
+	stdinBytes.WriteString("\n")
+
+	select {
+	case <-wait:
+	case <-timeout.C:
+		require.Fail(t, "Timeout waiting for metric to arrive")
+	}
+
+	for stdoutBytes.Len() == 0 {
+		select {
+		case <-timeout.C:
+			require.Fail(t, "Timeout waiting to read metric from stdout")
+			return
+		default:
+			time.Sleep(10 * time.Millisecond)
+		}
+	}
+
+	out := string(stdoutBytes.Bytes())
+	require.Contains(t, out, "\n")
+	metricLine := strings.Split(out, "\n")[0]
+	require.Equal(t, "measurement,tag=tag field=1i 1234000005678", metricLine)
+}
+
+func runInputPlugin(t *testing.T, timeout time.Duration) chan bool {
+	wait := make(chan bool)
+	inp := &testInput{
+		wait: wait,
+	}
+
+	shim := New()
+	shim.AddInput(inp)
+	go func() {
+		err := shim.Run(timeout) // we aren't using the timer here
+		require.NoError(t, err)
+	}()
+	return wait
+}
+
+type testInput struct {
+	wait chan bool
+}
+
+func (i *testInput) SampleConfig() string {
+	return ""
+}
+
+func (i *testInput) Description() string {
+	return ""
+}
+
+func (i *testInput) Gather(acc telegraf.Accumulator) error {
+	acc.AddFields("measurement",
+		map[string]interface{}{
+			"field": 1,
+		},
+		map[string]string{
+			"tag": "tag",
+		}, time.Unix(1234, 5678))
+	i.wait <- true
+	return nil
+}
+
+func (i *testInput) Start(acc telegraf.Accumulator) error {
+	return nil
+}
+
+func (i *testInput) Stop() {
+}