Add github input plugin (#5587)

Gopkg.lock

@@ -527,6 +527,22 @@
   revision = "3af367b6b30c263d47e8895973edcca9a49cf029"
   version = "v0.2.0"
 
+[[projects]]
+  digest = "1:e38ad2825940d58bd8425be40bcd4211099d0c1988c158c35828197413b3cf85"
+  name = "github.com/google/go-github"
+  packages = ["github"]
+  pruneopts = ""
+  revision = "7462feb2032c2da9e3b85e9b04e6853a6e9e14ca"
+  version = "v24.0.1"
+
+[[projects]]
+  digest = "1:cea4aa2038169ee558bf507d5ea02c94ca85bcca28a4c7bb99fd59b31e43a686"
+  name = "github.com/google/go-querystring"
+  packages = ["query"]
+  pruneopts = ""
+  revision = "44c6ddd0a2342c386950e880b658017258da92fc"
+  version = "v1.0.0"
+
 [[projects]]
   digest = "1:c1d7e883c50a26ea34019320d8ae40fad86c9e5d56e63a1ba2cb618cef43e986"
   name = "github.com/google/uuid"
@@ -1565,6 +1581,7 @@
     "github.com/golang/protobuf/ptypes/empty",
     "github.com/golang/protobuf/ptypes/timestamp",
     "github.com/google/go-cmp/cmp",
+    "github.com/google/go-github/github",
     "github.com/gorilla/mux",
     "github.com/harlow/kinesis-consumer",
     "github.com/harlow/kinesis-consumer/checkpoint/ddb",

Gopkg.toml

@@ -284,3 +284,7 @@
 [[override]]
   name = "golang.org/x/text"
   source = "https://github.com/golang/text.git"
+
+[[constraint]]
+  name = "github.com/google/go-github"
+  version = "24.0.1"

plugins/inputs/all/all.go

@@ -38,6 +38,7 @@ import (
 	_ "github.com/influxdata/telegraf/plugins/inputs/filecount"
 	_ "github.com/influxdata/telegraf/plugins/inputs/filestat"
 	_ "github.com/influxdata/telegraf/plugins/inputs/fluentd"
+	_ "github.com/influxdata/telegraf/plugins/inputs/github"
 	_ "github.com/influxdata/telegraf/plugins/inputs/graylog"
 	_ "github.com/influxdata/telegraf/plugins/inputs/haproxy"
 	_ "github.com/influxdata/telegraf/plugins/inputs/hddtemp"

plugins/inputs/github/README.md

@@ -0,0 +1,47 @@
+# GitHub Input Plugin
+
+The [GitHub](https://www.github.com) input plugin gathers statistics from GitHub repositories.
+
+### Configuration:
+
+```toml
+[[inputs.github]]
+  ## List of repositories to monitor
+  ## ex: repositories = ["influxdata/telegraf"]
+  # repositories = []
+
+  ## Optional: Unauthenticated requests are limited to 60 per hour.
+  # access_token = ""
+
+  ## Optional: Default 5s.
+  # http_timeout = "5s"
+```
+
+### Metrics:
+
+- github_repository
+  - tags:
+    - `name` - The repository name
+    - `owner` - The owner of the repository
+    - `language` - The primary language of the repository
+    - `license` - The license set for the repository
+  - fields:
+    - `stars` (int)
+    - `forks` (int)
+    - `open_issues` (int)
+    - `size` (int)
+
+* github_rate_limit
+  - tags:
+    - `access_token` - An obfusticated reference to the configured access token or "Unauthenticated"
+  - fields:
+    - `limit` - How many requests you are limited to (per hour)
+    - `remaining` - How many requests you have remaining (per hour)
+    - `blocks` - How many requests have been blocked due to rate limit
+
+### Example Output:
+
+```
+github,full_name=influxdata/telegraf,name=telegraf,owner=influxdata,language=Go,license=MIT\ License stars=6401i,forks=2421i,open_issues=722i,size=22611i 1552651811000000000
+internal_github,access_token=Unauthenticated rate_limit_remaining=59i,rate_limit_limit=60i,rate_limit_blocks=0i 1552653551000000000
+```

plugins/inputs/github/github.go

@@ -0,0 +1,184 @@
+package github
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/google/go-github/github"
+	"github.com/influxdata/telegraf"
+	"github.com/influxdata/telegraf/internal"
+	"github.com/influxdata/telegraf/plugins/inputs"
+	"github.com/influxdata/telegraf/selfstat"
+	"golang.org/x/oauth2"
+)
+
+// GitHub - plugin main structure
+type GitHub struct {
+	Repositories []string          `toml:"repositories"`
+	AccessToken  string            `toml:"access_token"`
+	HTTPTimeout  internal.Duration `toml:"http_timeout"`
+	githubClient *github.Client
+
+	obfusticatedToken string
+
+	RateLimit       selfstat.Stat
+	RateLimitErrors selfstat.Stat
+	RateRemaining   selfstat.Stat
+}
+
+const sampleConfig = `
+  ## List of repositories to monitor
+  ## ex: repositories = ["influxdata/telegraf"]
+  # repositories = []
+
+  ## Optional: Unauthenticated requests are limited to 60 per hour.
+  # access_token = ""
+
+  ## Optional: Default 5s.
+  # http_timeout = "5s"
+`
+
+// SampleConfig returns sample configuration for this plugin.
+func (g *GitHub) SampleConfig() string {
+	return sampleConfig
+}
+
+// Description returns the plugin description.
+func (g *GitHub) Description() string {
+	return "Read repository information from GitHub, including forks, stars, and more."
+}
+
+// Create GitHub Client
+func (g *GitHub) createGitHubClient(ctx context.Context) (*github.Client, error) {
+	httpClient := &http.Client{
+		Transport: &http.Transport{
+			Proxy: http.ProxyFromEnvironment,
+		},
+		Timeout: g.HTTPTimeout.Duration,
+	}
+
+	g.obfusticatedToken = "Unauthenticated"
+
+	if g.AccessToken != "" {
+		tokenSource := oauth2.StaticTokenSource(
+			&oauth2.Token{AccessToken: g.AccessToken},
+		)
+		oauthClient := oauth2.NewClient(ctx, tokenSource)
+		ctx = context.WithValue(ctx, oauth2.HTTPClient, oauthClient)
+
+		g.obfusticatedToken = g.AccessToken[0:4] + "..." + g.AccessToken[len(g.AccessToken)-3:]
+
+		return github.NewClient(oauthClient), nil
+	}
+
+	return github.NewClient(httpClient), nil
+}
+
+// Gather GitHub Metrics
+func (g *GitHub) Gather(acc telegraf.Accumulator) error {
+	ctx := context.Background()
+
+	if g.githubClient == nil {
+		githubClient, err := g.createGitHubClient(ctx)
+
+		if err != nil {
+			return err
+		}
+
+		g.githubClient = githubClient
+
+		tokenTags := map[string]string{
+			"access_token": g.obfusticatedToken,
+		}
+
+		g.RateLimitErrors = selfstat.Register("github", "rate_limit_blocks", tokenTags)
+		g.RateLimit = selfstat.Register("github", "rate_limit_limit", tokenTags)
+		g.RateRemaining = selfstat.Register("github", "rate_limit_remaining", tokenTags)
+	}
+
+	var wg sync.WaitGroup
+	wg.Add(len(g.Repositories))
+
+	for _, repository := range g.Repositories {
+		go func(repositoryName string, acc telegraf.Accumulator) {
+			defer wg.Done()
+
+			owner, repository, err := splitRepositoryName(repositoryName)
+			if err != nil {
+				acc.AddError(err)
+				return
+			}
+
+			repositoryInfo, response, err := g.githubClient.Repositories.Get(ctx, owner, repository)
+
+			if _, ok := err.(*github.RateLimitError); ok {
+				g.RateLimitErrors.Incr(1)
+			}
+
+			if err != nil {
+				acc.AddError(err)
+				return
+			}
+
+			g.RateLimit.Set(int64(response.Rate.Limit))
+			g.RateRemaining.Set(int64(response.Rate.Remaining))
+
+			now := time.Now()
+			tags := getTags(repositoryInfo)
+			fields := getFields(repositoryInfo)
+
+			acc.AddFields("github_repository", fields, tags, now)
+		}(repository, acc)
+	}
+
+	wg.Wait()
+	return nil
+}
+
+func splitRepositoryName(repositoryName string) (string, string, error) {
+	splits := strings.SplitN(repositoryName, "/", 2)
+
+	if len(splits) != 2 {
+		return "", "", fmt.Errorf("%v is not of format 'owner/repository'", repositoryName)
+	}
+
+	return splits[0], splits[1], nil
+}
+
+func getLicense(repositoryInfo *github.Repository) string {
+	if repositoryInfo.GetLicense() != nil {
+		return *repositoryInfo.License.Name
+	}
+
+	return "None"
+}
+
+func getTags(repositoryInfo *github.Repository) map[string]string {
+	return map[string]string{
+		"owner":    *repositoryInfo.Owner.Login,
+		"name":     *repositoryInfo.Name,
+		"language": *repositoryInfo.Language,
+		"license":  getLicense(repositoryInfo),
+	}
+}
+
+func getFields(repositoryInfo *github.Repository) map[string]interface{} {
+	return map[string]interface{}{
+		"stars":       *repositoryInfo.StargazersCount,
+		"forks":       *repositoryInfo.ForksCount,
+		"open_issues": *repositoryInfo.OpenIssuesCount,
+		"size":        *repositoryInfo.Size,
+	}
+}
+
+func init() {
+	inputs.Add("github", func() telegraf.Input {
+		return &GitHub{
+			HTTPTimeout: internal.Duration{Duration: time.Second * 5},
+		}
+	})
+}

plugins/inputs/github/github_test.go

@@ -0,0 +1,119 @@
+package github
+
+import (
+	"reflect"
+	"testing"
+
+	gh "github.com/google/go-github/github"
+	"github.com/stretchr/testify/require"
+)
+
+func TestSplitRepositoryNameWithWorkingExample(t *testing.T) {
+	var validRepositoryNames = []struct {
+		fullName   string
+		owner      string
+		repository string
+	}{
+		{"influxdata/telegraf", "influxdata", "telegraf"},
+		{"influxdata/influxdb", "influxdata", "influxdb"},
+		{"rawkode/saltstack-dotfiles", "rawkode", "saltstack-dotfiles"},
+	}
+
+	for _, tt := range validRepositoryNames {
+		t.Run(tt.fullName, func(t *testing.T) {
+			owner, repository, _ := splitRepositoryName(tt.fullName)
+
+			require.Equal(t, tt.owner, owner)
+			require.Equal(t, tt.repository, repository)
+		})
+	}
+}
+
+func TestSplitRepositoryNameWithNoSlash(t *testing.T) {
+	var invalidRepositoryNames = []string{
+		"influxdata-influxdb",
+	}
+
+	for _, tt := range invalidRepositoryNames {
+		t.Run(tt, func(t *testing.T) {
+			_, _, err := splitRepositoryName(tt)
+
+			require.NotNil(t, err)
+		})
+	}
+}
+
+func TestGetLicenseWhenExists(t *testing.T) {
+	licenseName := "MIT"
+	license := gh.License{Name: &licenseName}
+	repository := gh.Repository{License: &license}
+
+	getLicenseReturn := getLicense(&repository)
+
+	require.Equal(t, "MIT", getLicenseReturn)
+}
+
+func TestGetLicenseWhenMissing(t *testing.T) {
+	repository := gh.Repository{}
+
+	getLicenseReturn := getLicense(&repository)
+
+	require.Equal(t, "None", getLicenseReturn)
+}
+
+func TestGetTags(t *testing.T) {
+	licenseName := "MIT"
+	license := gh.License{Name: &licenseName}
+
+	ownerName := "influxdata"
+	owner := gh.User{Login: &ownerName}
+
+	fullName := "influxdata/influxdb"
+	repositoryName := "influxdb"
+
+	language := "Go"
+
+	repository := gh.Repository{
+		FullName: &fullName,
+		Name:     &repositoryName,
+		License:  &license,
+		Owner:    &owner,
+		Language: &language,
+	}
+
+	getTagsReturn := getTags(&repository)
+
+	correctTagsReturn := map[string]string{
+		"owner":    ownerName,
+		"name":     repositoryName,
+		"language": language,
+		"license":  licenseName,
+	}
+
+	require.Equal(t, true, reflect.DeepEqual(getTagsReturn, correctTagsReturn))
+}
+
+func TestGetFields(t *testing.T) {
+	stars := 1
+	forks := 2
+	openIssues := 3
+	size := 4
+
+	repository := gh.Repository{
+		StargazersCount: &stars,
+		ForksCount:      &forks,
+		OpenIssuesCount: &openIssues,
+		Size:            &size,
+	}
+
+	getFieldsReturn := getFields(&repository)
+
+	correctFieldReturn := make(map[string]interface{})
+
+	correctFieldReturn["stars"] = 1
+	correctFieldReturn["forks"] = 2
+	correctFieldReturn["open_issues"] = 3
+	correctFieldReturn["size"] = 4
+
+	require.Equal(t, true, reflect.DeepEqual(getFieldsReturn, correctFieldReturn))
+}