diff --git a/Gopkg.lock b/Gopkg.lock index 67654d523..1521eb2cd 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -527,6 +527,22 @@ revision = "3af367b6b30c263d47e8895973edcca9a49cf029" version = "v0.2.0" +[[projects]] + digest = "1:e38ad2825940d58bd8425be40bcd4211099d0c1988c158c35828197413b3cf85" + name = "github.com/google/go-github" + packages = ["github"] + pruneopts = "" + revision = "7462feb2032c2da9e3b85e9b04e6853a6e9e14ca" + version = "v24.0.1" + +[[projects]] + digest = "1:cea4aa2038169ee558bf507d5ea02c94ca85bcca28a4c7bb99fd59b31e43a686" + name = "github.com/google/go-querystring" + packages = ["query"] + pruneopts = "" + revision = "44c6ddd0a2342c386950e880b658017258da92fc" + version = "v1.0.0" + [[projects]] digest = "1:c1d7e883c50a26ea34019320d8ae40fad86c9e5d56e63a1ba2cb618cef43e986" name = "github.com/google/uuid" @@ -1565,6 +1581,7 @@ "github.com/golang/protobuf/ptypes/empty", "github.com/golang/protobuf/ptypes/timestamp", "github.com/google/go-cmp/cmp", + "github.com/google/go-github/github", "github.com/gorilla/mux", "github.com/harlow/kinesis-consumer", "github.com/harlow/kinesis-consumer/checkpoint/ddb", diff --git a/Gopkg.toml b/Gopkg.toml index bfc854450..057af5e3b 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -284,3 +284,7 @@ [[override]] name = "golang.org/x/text" source = "https://github.com/golang/text.git" + +[[constraint]] + name = "github.com/google/go-github" + version = "24.0.1" diff --git a/plugins/inputs/all/all.go b/plugins/inputs/all/all.go index 765505c3e..5f1ba4759 100644 --- a/plugins/inputs/all/all.go +++ b/plugins/inputs/all/all.go @@ -38,6 +38,7 @@ import ( _ "github.com/influxdata/telegraf/plugins/inputs/filecount" _ "github.com/influxdata/telegraf/plugins/inputs/filestat" _ "github.com/influxdata/telegraf/plugins/inputs/fluentd" + _ "github.com/influxdata/telegraf/plugins/inputs/github" _ "github.com/influxdata/telegraf/plugins/inputs/graylog" _ "github.com/influxdata/telegraf/plugins/inputs/haproxy" _ "github.com/influxdata/telegraf/plugins/inputs/hddtemp" diff --git a/plugins/inputs/github/README.md b/plugins/inputs/github/README.md new file mode 100644 index 000000000..dc5a161cd --- /dev/null +++ b/plugins/inputs/github/README.md @@ -0,0 +1,47 @@ +# GitHub Input Plugin + +The [GitHub](https://www.github.com) input plugin gathers statistics from GitHub repositories. + +### Configuration: + +```toml +[[inputs.github]] + ## List of repositories to monitor + ## ex: repositories = ["influxdata/telegraf"] + # repositories = [] + + ## Optional: Unauthenticated requests are limited to 60 per hour. + # access_token = "" + + ## Optional: Default 5s. + # http_timeout = "5s" +``` + +### Metrics: + +- github_repository + - tags: + - `name` - The repository name + - `owner` - The owner of the repository + - `language` - The primary language of the repository + - `license` - The license set for the repository + - fields: + - `stars` (int) + - `forks` (int) + - `open_issues` (int) + - `size` (int) + +* github_rate_limit + - tags: + - `access_token` - An obfusticated reference to the configured access token or "Unauthenticated" + - fields: + - `limit` - How many requests you are limited to (per hour) + - `remaining` - How many requests you have remaining (per hour) + - `blocks` - How many requests have been blocked due to rate limit + +### Example Output: + +``` +github,full_name=influxdata/telegraf,name=telegraf,owner=influxdata,language=Go,license=MIT\ License stars=6401i,forks=2421i,open_issues=722i,size=22611i 1552651811000000000 +internal_github,access_token=Unauthenticated rate_limit_remaining=59i,rate_limit_limit=60i,rate_limit_blocks=0i 1552653551000000000 +``` diff --git a/plugins/inputs/github/github.go b/plugins/inputs/github/github.go new file mode 100644 index 000000000..cf709e69a --- /dev/null +++ b/plugins/inputs/github/github.go @@ -0,0 +1,184 @@ +package github + +import ( + "context" + "fmt" + "net/http" + "strings" + "sync" + "time" + + "github.com/google/go-github/github" + "github.com/influxdata/telegraf" + "github.com/influxdata/telegraf/internal" + "github.com/influxdata/telegraf/plugins/inputs" + "github.com/influxdata/telegraf/selfstat" + "golang.org/x/oauth2" +) + +// GitHub - plugin main structure +type GitHub struct { + Repositories []string `toml:"repositories"` + AccessToken string `toml:"access_token"` + HTTPTimeout internal.Duration `toml:"http_timeout"` + githubClient *github.Client + + obfusticatedToken string + + RateLimit selfstat.Stat + RateLimitErrors selfstat.Stat + RateRemaining selfstat.Stat +} + +const sampleConfig = ` + ## List of repositories to monitor + ## ex: repositories = ["influxdata/telegraf"] + # repositories = [] + + ## Optional: Unauthenticated requests are limited to 60 per hour. + # access_token = "" + + ## Optional: Default 5s. + # http_timeout = "5s" +` + +// SampleConfig returns sample configuration for this plugin. +func (g *GitHub) SampleConfig() string { + return sampleConfig +} + +// Description returns the plugin description. +func (g *GitHub) Description() string { + return "Read repository information from GitHub, including forks, stars, and more." +} + +// Create GitHub Client +func (g *GitHub) createGitHubClient(ctx context.Context) (*github.Client, error) { + httpClient := &http.Client{ + Transport: &http.Transport{ + Proxy: http.ProxyFromEnvironment, + }, + Timeout: g.HTTPTimeout.Duration, + } + + g.obfusticatedToken = "Unauthenticated" + + if g.AccessToken != "" { + tokenSource := oauth2.StaticTokenSource( + &oauth2.Token{AccessToken: g.AccessToken}, + ) + oauthClient := oauth2.NewClient(ctx, tokenSource) + ctx = context.WithValue(ctx, oauth2.HTTPClient, oauthClient) + + g.obfusticatedToken = g.AccessToken[0:4] + "..." + g.AccessToken[len(g.AccessToken)-3:] + + return github.NewClient(oauthClient), nil + } + + return github.NewClient(httpClient), nil +} + +// Gather GitHub Metrics +func (g *GitHub) Gather(acc telegraf.Accumulator) error { + ctx := context.Background() + + if g.githubClient == nil { + githubClient, err := g.createGitHubClient(ctx) + + if err != nil { + return err + } + + g.githubClient = githubClient + + tokenTags := map[string]string{ + "access_token": g.obfusticatedToken, + } + + g.RateLimitErrors = selfstat.Register("github", "rate_limit_blocks", tokenTags) + g.RateLimit = selfstat.Register("github", "rate_limit_limit", tokenTags) + g.RateRemaining = selfstat.Register("github", "rate_limit_remaining", tokenTags) + } + + var wg sync.WaitGroup + wg.Add(len(g.Repositories)) + + for _, repository := range g.Repositories { + go func(repositoryName string, acc telegraf.Accumulator) { + defer wg.Done() + + owner, repository, err := splitRepositoryName(repositoryName) + if err != nil { + acc.AddError(err) + return + } + + repositoryInfo, response, err := g.githubClient.Repositories.Get(ctx, owner, repository) + + if _, ok := err.(*github.RateLimitError); ok { + g.RateLimitErrors.Incr(1) + } + + if err != nil { + acc.AddError(err) + return + } + + g.RateLimit.Set(int64(response.Rate.Limit)) + g.RateRemaining.Set(int64(response.Rate.Remaining)) + + now := time.Now() + tags := getTags(repositoryInfo) + fields := getFields(repositoryInfo) + + acc.AddFields("github_repository", fields, tags, now) + }(repository, acc) + } + + wg.Wait() + return nil +} + +func splitRepositoryName(repositoryName string) (string, string, error) { + splits := strings.SplitN(repositoryName, "/", 2) + + if len(splits) != 2 { + return "", "", fmt.Errorf("%v is not of format 'owner/repository'", repositoryName) + } + + return splits[0], splits[1], nil +} + +func getLicense(repositoryInfo *github.Repository) string { + if repositoryInfo.GetLicense() != nil { + return *repositoryInfo.License.Name + } + + return "None" +} + +func getTags(repositoryInfo *github.Repository) map[string]string { + return map[string]string{ + "owner": *repositoryInfo.Owner.Login, + "name": *repositoryInfo.Name, + "language": *repositoryInfo.Language, + "license": getLicense(repositoryInfo), + } +} + +func getFields(repositoryInfo *github.Repository) map[string]interface{} { + return map[string]interface{}{ + "stars": *repositoryInfo.StargazersCount, + "forks": *repositoryInfo.ForksCount, + "open_issues": *repositoryInfo.OpenIssuesCount, + "size": *repositoryInfo.Size, + } +} + +func init() { + inputs.Add("github", func() telegraf.Input { + return &GitHub{ + HTTPTimeout: internal.Duration{Duration: time.Second * 5}, + } + }) +} diff --git a/plugins/inputs/github/github_test.go b/plugins/inputs/github/github_test.go new file mode 100644 index 000000000..0ebae3a67 --- /dev/null +++ b/plugins/inputs/github/github_test.go @@ -0,0 +1,119 @@ +package github + +import ( + "reflect" + "testing" + + gh "github.com/google/go-github/github" + "github.com/stretchr/testify/require" +) + +func TestSplitRepositoryNameWithWorkingExample(t *testing.T) { + var validRepositoryNames = []struct { + fullName string + owner string + repository string + }{ + {"influxdata/telegraf", "influxdata", "telegraf"}, + {"influxdata/influxdb", "influxdata", "influxdb"}, + {"rawkode/saltstack-dotfiles", "rawkode", "saltstack-dotfiles"}, + } + + for _, tt := range validRepositoryNames { + t.Run(tt.fullName, func(t *testing.T) { + owner, repository, _ := splitRepositoryName(tt.fullName) + + require.Equal(t, tt.owner, owner) + require.Equal(t, tt.repository, repository) + }) + } +} + +func TestSplitRepositoryNameWithNoSlash(t *testing.T) { + var invalidRepositoryNames = []string{ + "influxdata-influxdb", + } + + for _, tt := range invalidRepositoryNames { + t.Run(tt, func(t *testing.T) { + _, _, err := splitRepositoryName(tt) + + require.NotNil(t, err) + }) + } +} + +func TestGetLicenseWhenExists(t *testing.T) { + licenseName := "MIT" + license := gh.License{Name: &licenseName} + repository := gh.Repository{License: &license} + + getLicenseReturn := getLicense(&repository) + + require.Equal(t, "MIT", getLicenseReturn) +} + +func TestGetLicenseWhenMissing(t *testing.T) { + repository := gh.Repository{} + + getLicenseReturn := getLicense(&repository) + + require.Equal(t, "None", getLicenseReturn) +} + +func TestGetTags(t *testing.T) { + licenseName := "MIT" + license := gh.License{Name: &licenseName} + + ownerName := "influxdata" + owner := gh.User{Login: &ownerName} + + fullName := "influxdata/influxdb" + repositoryName := "influxdb" + + language := "Go" + + repository := gh.Repository{ + FullName: &fullName, + Name: &repositoryName, + License: &license, + Owner: &owner, + Language: &language, + } + + getTagsReturn := getTags(&repository) + + correctTagsReturn := map[string]string{ + "owner": ownerName, + "name": repositoryName, + "language": language, + "license": licenseName, + } + + require.Equal(t, true, reflect.DeepEqual(getTagsReturn, correctTagsReturn)) +} + +func TestGetFields(t *testing.T) { + stars := 1 + forks := 2 + openIssues := 3 + size := 4 + + repository := gh.Repository{ + StargazersCount: &stars, + ForksCount: &forks, + OpenIssuesCount: &openIssues, + Size: &size, + } + + getFieldsReturn := getFields(&repository) + + correctFieldReturn := make(map[string]interface{}) + + correctFieldReturn["stars"] = 1 + correctFieldReturn["forks"] = 2 + correctFieldReturn["open_issues"] = 3 + correctFieldReturn["size"] = 4 + + require.Equal(t, true, reflect.DeepEqual(getFieldsReturn, correctFieldReturn)) +}