Add github input plugin (#5587)

This commit is contained in:
David McKay 2019-04-03 00:06:15 +01:00 committed by Daniel Nelson
parent 0e50a3977d
commit 0a68c8468b
6 changed files with 372 additions and 0 deletions

17
Gopkg.lock generated
View File

@ -527,6 +527,22 @@
revision = "3af367b6b30c263d47e8895973edcca9a49cf029" revision = "3af367b6b30c263d47e8895973edcca9a49cf029"
version = "v0.2.0" version = "v0.2.0"
[[projects]]
digest = "1:e38ad2825940d58bd8425be40bcd4211099d0c1988c158c35828197413b3cf85"
name = "github.com/google/go-github"
packages = ["github"]
pruneopts = ""
revision = "7462feb2032c2da9e3b85e9b04e6853a6e9e14ca"
version = "v24.0.1"
[[projects]]
digest = "1:cea4aa2038169ee558bf507d5ea02c94ca85bcca28a4c7bb99fd59b31e43a686"
name = "github.com/google/go-querystring"
packages = ["query"]
pruneopts = ""
revision = "44c6ddd0a2342c386950e880b658017258da92fc"
version = "v1.0.0"
[[projects]] [[projects]]
digest = "1:c1d7e883c50a26ea34019320d8ae40fad86c9e5d56e63a1ba2cb618cef43e986" digest = "1:c1d7e883c50a26ea34019320d8ae40fad86c9e5d56e63a1ba2cb618cef43e986"
name = "github.com/google/uuid" name = "github.com/google/uuid"
@ -1565,6 +1581,7 @@
"github.com/golang/protobuf/ptypes/empty", "github.com/golang/protobuf/ptypes/empty",
"github.com/golang/protobuf/ptypes/timestamp", "github.com/golang/protobuf/ptypes/timestamp",
"github.com/google/go-cmp/cmp", "github.com/google/go-cmp/cmp",
"github.com/google/go-github/github",
"github.com/gorilla/mux", "github.com/gorilla/mux",
"github.com/harlow/kinesis-consumer", "github.com/harlow/kinesis-consumer",
"github.com/harlow/kinesis-consumer/checkpoint/ddb", "github.com/harlow/kinesis-consumer/checkpoint/ddb",

View File

@ -284,3 +284,7 @@
[[override]] [[override]]
name = "golang.org/x/text" name = "golang.org/x/text"
source = "https://github.com/golang/text.git" source = "https://github.com/golang/text.git"
[[constraint]]
name = "github.com/google/go-github"
version = "24.0.1"

View File

@ -38,6 +38,7 @@ import (
_ "github.com/influxdata/telegraf/plugins/inputs/filecount" _ "github.com/influxdata/telegraf/plugins/inputs/filecount"
_ "github.com/influxdata/telegraf/plugins/inputs/filestat" _ "github.com/influxdata/telegraf/plugins/inputs/filestat"
_ "github.com/influxdata/telegraf/plugins/inputs/fluentd" _ "github.com/influxdata/telegraf/plugins/inputs/fluentd"
_ "github.com/influxdata/telegraf/plugins/inputs/github"
_ "github.com/influxdata/telegraf/plugins/inputs/graylog" _ "github.com/influxdata/telegraf/plugins/inputs/graylog"
_ "github.com/influxdata/telegraf/plugins/inputs/haproxy" _ "github.com/influxdata/telegraf/plugins/inputs/haproxy"
_ "github.com/influxdata/telegraf/plugins/inputs/hddtemp" _ "github.com/influxdata/telegraf/plugins/inputs/hddtemp"

View File

@ -0,0 +1,47 @@
# GitHub Input Plugin
The [GitHub](https://www.github.com) input plugin gathers statistics from GitHub repositories.
### Configuration:
```toml
[[inputs.github]]
## List of repositories to monitor
## ex: repositories = ["influxdata/telegraf"]
# repositories = []
## Optional: Unauthenticated requests are limited to 60 per hour.
# access_token = ""
## Optional: Default 5s.
# http_timeout = "5s"
```
### Metrics:
- github_repository
- tags:
- `name` - The repository name
- `owner` - The owner of the repository
- `language` - The primary language of the repository
- `license` - The license set for the repository
- fields:
- `stars` (int)
- `forks` (int)
- `open_issues` (int)
- `size` (int)
* github_rate_limit
- tags:
- `access_token` - An obfusticated reference to the configured access token or "Unauthenticated"
- fields:
- `limit` - How many requests you are limited to (per hour)
- `remaining` - How many requests you have remaining (per hour)
- `blocks` - How many requests have been blocked due to rate limit
### Example Output:
```
github,full_name=influxdata/telegraf,name=telegraf,owner=influxdata,language=Go,license=MIT\ License stars=6401i,forks=2421i,open_issues=722i,size=22611i 1552651811000000000
internal_github,access_token=Unauthenticated rate_limit_remaining=59i,rate_limit_limit=60i,rate_limit_blocks=0i 1552653551000000000
```

View File

@ -0,0 +1,184 @@
package github
import (
"context"
"fmt"
"net/http"
"strings"
"sync"
"time"
"github.com/google/go-github/github"
"github.com/influxdata/telegraf"
"github.com/influxdata/telegraf/internal"
"github.com/influxdata/telegraf/plugins/inputs"
"github.com/influxdata/telegraf/selfstat"
"golang.org/x/oauth2"
)
// GitHub - plugin main structure
type GitHub struct {
Repositories []string `toml:"repositories"`
AccessToken string `toml:"access_token"`
HTTPTimeout internal.Duration `toml:"http_timeout"`
githubClient *github.Client
obfusticatedToken string
RateLimit selfstat.Stat
RateLimitErrors selfstat.Stat
RateRemaining selfstat.Stat
}
const sampleConfig = `
## List of repositories to monitor
## ex: repositories = ["influxdata/telegraf"]
# repositories = []
## Optional: Unauthenticated requests are limited to 60 per hour.
# access_token = ""
## Optional: Default 5s.
# http_timeout = "5s"
`
// SampleConfig returns sample configuration for this plugin.
func (g *GitHub) SampleConfig() string {
return sampleConfig
}
// Description returns the plugin description.
func (g *GitHub) Description() string {
return "Read repository information from GitHub, including forks, stars, and more."
}
// Create GitHub Client
func (g *GitHub) createGitHubClient(ctx context.Context) (*github.Client, error) {
httpClient := &http.Client{
Transport: &http.Transport{
Proxy: http.ProxyFromEnvironment,
},
Timeout: g.HTTPTimeout.Duration,
}
g.obfusticatedToken = "Unauthenticated"
if g.AccessToken != "" {
tokenSource := oauth2.StaticTokenSource(
&oauth2.Token{AccessToken: g.AccessToken},
)
oauthClient := oauth2.NewClient(ctx, tokenSource)
ctx = context.WithValue(ctx, oauth2.HTTPClient, oauthClient)
g.obfusticatedToken = g.AccessToken[0:4] + "..." + g.AccessToken[len(g.AccessToken)-3:]
return github.NewClient(oauthClient), nil
}
return github.NewClient(httpClient), nil
}
// Gather GitHub Metrics
func (g *GitHub) Gather(acc telegraf.Accumulator) error {
ctx := context.Background()
if g.githubClient == nil {
githubClient, err := g.createGitHubClient(ctx)
if err != nil {
return err
}
g.githubClient = githubClient
tokenTags := map[string]string{
"access_token": g.obfusticatedToken,
}
g.RateLimitErrors = selfstat.Register("github", "rate_limit_blocks", tokenTags)
g.RateLimit = selfstat.Register("github", "rate_limit_limit", tokenTags)
g.RateRemaining = selfstat.Register("github", "rate_limit_remaining", tokenTags)
}
var wg sync.WaitGroup
wg.Add(len(g.Repositories))
for _, repository := range g.Repositories {
go func(repositoryName string, acc telegraf.Accumulator) {
defer wg.Done()
owner, repository, err := splitRepositoryName(repositoryName)
if err != nil {
acc.AddError(err)
return
}
repositoryInfo, response, err := g.githubClient.Repositories.Get(ctx, owner, repository)
if _, ok := err.(*github.RateLimitError); ok {
g.RateLimitErrors.Incr(1)
}
if err != nil {
acc.AddError(err)
return
}
g.RateLimit.Set(int64(response.Rate.Limit))
g.RateRemaining.Set(int64(response.Rate.Remaining))
now := time.Now()
tags := getTags(repositoryInfo)
fields := getFields(repositoryInfo)
acc.AddFields("github_repository", fields, tags, now)
}(repository, acc)
}
wg.Wait()
return nil
}
func splitRepositoryName(repositoryName string) (string, string, error) {
splits := strings.SplitN(repositoryName, "/", 2)
if len(splits) != 2 {
return "", "", fmt.Errorf("%v is not of format 'owner/repository'", repositoryName)
}
return splits[0], splits[1], nil
}
func getLicense(repositoryInfo *github.Repository) string {
if repositoryInfo.GetLicense() != nil {
return *repositoryInfo.License.Name
}
return "None"
}
func getTags(repositoryInfo *github.Repository) map[string]string {
return map[string]string{
"owner": *repositoryInfo.Owner.Login,
"name": *repositoryInfo.Name,
"language": *repositoryInfo.Language,
"license": getLicense(repositoryInfo),
}
}
func getFields(repositoryInfo *github.Repository) map[string]interface{} {
return map[string]interface{}{
"stars": *repositoryInfo.StargazersCount,
"forks": *repositoryInfo.ForksCount,
"open_issues": *repositoryInfo.OpenIssuesCount,
"size": *repositoryInfo.Size,
}
}
func init() {
inputs.Add("github", func() telegraf.Input {
return &GitHub{
HTTPTimeout: internal.Duration{Duration: time.Second * 5},
}
})
}

View File

@ -0,0 +1,119 @@
package github
import (
"reflect"
"testing"
gh "github.com/google/go-github/github"
"github.com/stretchr/testify/require"
)
func TestSplitRepositoryNameWithWorkingExample(t *testing.T) {
var validRepositoryNames = []struct {
fullName string
owner string
repository string
}{
{"influxdata/telegraf", "influxdata", "telegraf"},
{"influxdata/influxdb", "influxdata", "influxdb"},
{"rawkode/saltstack-dotfiles", "rawkode", "saltstack-dotfiles"},
}
for _, tt := range validRepositoryNames {
t.Run(tt.fullName, func(t *testing.T) {
owner, repository, _ := splitRepositoryName(tt.fullName)
require.Equal(t, tt.owner, owner)
require.Equal(t, tt.repository, repository)
})
}
}
func TestSplitRepositoryNameWithNoSlash(t *testing.T) {
var invalidRepositoryNames = []string{
"influxdata-influxdb",
}
for _, tt := range invalidRepositoryNames {
t.Run(tt, func(t *testing.T) {
_, _, err := splitRepositoryName(tt)
require.NotNil(t, err)
})
}
}
func TestGetLicenseWhenExists(t *testing.T) {
licenseName := "MIT"
license := gh.License{Name: &licenseName}
repository := gh.Repository{License: &license}
getLicenseReturn := getLicense(&repository)
require.Equal(t, "MIT", getLicenseReturn)
}
func TestGetLicenseWhenMissing(t *testing.T) {
repository := gh.Repository{}
getLicenseReturn := getLicense(&repository)
require.Equal(t, "None", getLicenseReturn)
}
func TestGetTags(t *testing.T) {
licenseName := "MIT"
license := gh.License{Name: &licenseName}
ownerName := "influxdata"
owner := gh.User{Login: &ownerName}
fullName := "influxdata/influxdb"
repositoryName := "influxdb"
language := "Go"
repository := gh.Repository{
FullName: &fullName,
Name: &repositoryName,
License: &license,
Owner: &owner,
Language: &language,
}
getTagsReturn := getTags(&repository)
correctTagsReturn := map[string]string{
"owner": ownerName,
"name": repositoryName,
"language": language,
"license": licenseName,
}
require.Equal(t, true, reflect.DeepEqual(getTagsReturn, correctTagsReturn))
}
func TestGetFields(t *testing.T) {
stars := 1
forks := 2
openIssues := 3
size := 4
repository := gh.Repository{
StargazersCount: &stars,
ForksCount: &forks,
OpenIssuesCount: &openIssues,
Size: &size,
}
getFieldsReturn := getFields(&repository)
correctFieldReturn := make(map[string]interface{})
correctFieldReturn["stars"] = 1
correctFieldReturn["forks"] = 2
correctFieldReturn["open_issues"] = 3
correctFieldReturn["size"] = 4
require.Equal(t, true, reflect.DeepEqual(getFieldsReturn, correctFieldReturn))
}