Add github input plugin (#5587)
This commit is contained in:
David McKay
Daniel Nelson
parent
0e50a3977d
commit
0a68c8468b
|
|
@ -527,6 +527,22 @@
|
|||
revision = "3af367b6b30c263d47e8895973edcca9a49cf029"
|
||||
version = "v0.2.0"
|
||||
|
||||
[[projects]]
|
||||
digest = "1:e38ad2825940d58bd8425be40bcd4211099d0c1988c158c35828197413b3cf85"
|
||||
name = "github.com/google/go-github"
|
||||
packages = ["github"]
|
||||
pruneopts = ""
|
||||
revision = "7462feb2032c2da9e3b85e9b04e6853a6e9e14ca"
|
||||
version = "v24.0.1"
|
||||
|
||||
[[projects]]
|
||||
digest = "1:cea4aa2038169ee558bf507d5ea02c94ca85bcca28a4c7bb99fd59b31e43a686"
|
||||
name = "github.com/google/go-querystring"
|
||||
packages = ["query"]
|
||||
pruneopts = ""
|
||||
revision = "44c6ddd0a2342c386950e880b658017258da92fc"
|
||||
version = "v1.0.0"
|
||||
|
||||
[[projects]]
|
||||
digest = "1:c1d7e883c50a26ea34019320d8ae40fad86c9e5d56e63a1ba2cb618cef43e986"
|
||||
name = "github.com/google/uuid"
|
||||
|
|
@ -1565,6 +1581,7 @@
|
|||
"github.com/golang/protobuf/ptypes/empty",
|
||||
"github.com/golang/protobuf/ptypes/timestamp",
|
||||
"github.com/google/go-cmp/cmp",
|
||||
"github.com/google/go-github/github",
|
||||
"github.com/gorilla/mux",
|
||||
"github.com/harlow/kinesis-consumer",
|
||||
"github.com/harlow/kinesis-consumer/checkpoint/ddb",
|
||||
|
|
|
|||
|
|
@ -284,3 +284,7 @@
|
|||
[[override]]
|
||||
name = "golang.org/x/text"
|
||||
source = "https://github.com/golang/text.git"
|
||||
|
||||
[[constraint]]
|
||||
name = "github.com/google/go-github"
|
||||
version = "24.0.1"
|
||||
|
|
|
|||
|
|
@ -38,6 +38,7 @@ import (
|
|||
_ "github.com/influxdata/telegraf/plugins/inputs/filecount"
|
||||
_ "github.com/influxdata/telegraf/plugins/inputs/filestat"
|
||||
_ "github.com/influxdata/telegraf/plugins/inputs/fluentd"
|
||||
_ "github.com/influxdata/telegraf/plugins/inputs/github"
|
||||
_ "github.com/influxdata/telegraf/plugins/inputs/graylog"
|
||||
_ "github.com/influxdata/telegraf/plugins/inputs/haproxy"
|
||||
_ "github.com/influxdata/telegraf/plugins/inputs/hddtemp"
|
||||
|
|
|
|||
|
|
@ -0,0 +1,47 @@
|
|||
# GitHub Input Plugin
|
||||
|
||||
The [GitHub](https://www.github.com) input plugin gathers statistics from GitHub repositories.
|
||||
|
||||
### Configuration:
|
||||
|
||||
```toml
|
||||
[[inputs.github]]
|
||||
## List of repositories to monitor
|
||||
## ex: repositories = ["influxdata/telegraf"]
|
||||
# repositories = []
|
||||
|
||||
## Optional: Unauthenticated requests are limited to 60 per hour.
|
||||
# access_token = ""
|
||||
|
||||
## Optional: Default 5s.
|
||||
# http_timeout = "5s"
|
||||
```
|
||||
|
||||
### Metrics:
|
||||
|
||||
- github_repository
|
||||
- tags:
|
||||
- `name` - The repository name
|
||||
- `owner` - The owner of the repository
|
||||
- `language` - The primary language of the repository
|
||||
- `license` - The license set for the repository
|
||||
- fields:
|
||||
- `stars` (int)
|
||||
- `forks` (int)
|
||||
- `open_issues` (int)
|
||||
- `size` (int)
|
||||
|
||||
* github_rate_limit
|
||||
- tags:
|
||||
- `access_token` - An obfusticated reference to the configured access token or "Unauthenticated"
|
||||
- fields:
|
||||
- `limit` - How many requests you are limited to (per hour)
|
||||
- `remaining` - How many requests you have remaining (per hour)
|
||||
- `blocks` - How many requests have been blocked due to rate limit
|
||||
|
||||
### Example Output:
|
||||
|
||||
```
|
||||
github,full_name=influxdata/telegraf,name=telegraf,owner=influxdata,language=Go,license=MIT\ License stars=6401i,forks=2421i,open_issues=722i,size=22611i 1552651811000000000
|
||||
internal_github,access_token=Unauthenticated rate_limit_remaining=59i,rate_limit_limit=60i,rate_limit_blocks=0i 1552653551000000000
|
||||
```
|
||||
|
|
@ -0,0 +1,184 @@
|
|||
package github
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/google/go-github/github"
|
||||
"github.com/influxdata/telegraf"
|
||||
"github.com/influxdata/telegraf/internal"
|
||||
"github.com/influxdata/telegraf/plugins/inputs"
|
||||
"github.com/influxdata/telegraf/selfstat"
|
||||
"golang.org/x/oauth2"
|
||||
)
|
||||
|
||||
// GitHub - plugin main structure
|
||||
type GitHub struct {
|
||||
Repositories []string `toml:"repositories"`
|
||||
AccessToken string `toml:"access_token"`
|
||||
HTTPTimeout internal.Duration `toml:"http_timeout"`
|
||||
githubClient *github.Client
|
||||
|
||||
obfusticatedToken string
|
||||
|
||||
RateLimit selfstat.Stat
|
||||
RateLimitErrors selfstat.Stat
|
||||
RateRemaining selfstat.Stat
|
||||
}
|
||||
|
||||
const sampleConfig = `
|
||||
## List of repositories to monitor
|
||||
## ex: repositories = ["influxdata/telegraf"]
|
||||
# repositories = []
|
||||
|
||||
## Optional: Unauthenticated requests are limited to 60 per hour.
|
||||
# access_token = ""
|
||||
|
||||
## Optional: Default 5s.
|
||||
# http_timeout = "5s"
|
||||
`
|
||||
|
||||
// SampleConfig returns sample configuration for this plugin.
|
||||
func (g *GitHub) SampleConfig() string {
|
||||
return sampleConfig
|
||||
}
|
||||
|
||||
// Description returns the plugin description.
|
||||
func (g *GitHub) Description() string {
|
||||
return "Read repository information from GitHub, including forks, stars, and more."
|
||||
}
|
||||
|
||||
// Create GitHub Client
|
||||
func (g *GitHub) createGitHubClient(ctx context.Context) (*github.Client, error) {
|
||||
httpClient := &http.Client{
|
||||
Transport: &http.Transport{
|
||||
Proxy: http.ProxyFromEnvironment,
|
||||
},
|
||||
Timeout: g.HTTPTimeout.Duration,
|
||||
}
|
||||
|
||||
g.obfusticatedToken = "Unauthenticated"
|
||||
|
||||
if g.AccessToken != "" {
|
||||
tokenSource := oauth2.StaticTokenSource(
|
||||
&oauth2.Token{AccessToken: g.AccessToken},
|
||||
)
|
||||
oauthClient := oauth2.NewClient(ctx, tokenSource)
|
||||
ctx = context.WithValue(ctx, oauth2.HTTPClient, oauthClient)
|
||||
|
||||
g.obfusticatedToken = g.AccessToken[0:4] + "..." + g.AccessToken[len(g.AccessToken)-3:]
|
||||
|
||||
return github.NewClient(oauthClient), nil
|
||||
}
|
||||
|
||||
return github.NewClient(httpClient), nil
|
||||
}
|
||||
|
||||
// Gather GitHub Metrics
|
||||
func (g *GitHub) Gather(acc telegraf.Accumulator) error {
|
||||
ctx := context.Background()
|
||||
|
||||
if g.githubClient == nil {
|
||||
githubClient, err := g.createGitHubClient(ctx)
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
g.githubClient = githubClient
|
||||
|
||||
tokenTags := map[string]string{
|
||||
"access_token": g.obfusticatedToken,
|
||||
}
|
||||
|
||||
g.RateLimitErrors = selfstat.Register("github", "rate_limit_blocks", tokenTags)
|
||||
g.RateLimit = selfstat.Register("github", "rate_limit_limit", tokenTags)
|
||||
g.RateRemaining = selfstat.Register("github", "rate_limit_remaining", tokenTags)
|
||||
}
|
||||
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(len(g.Repositories))
|
||||
|
||||
for _, repository := range g.Repositories {
|
||||
go func(repositoryName string, acc telegraf.Accumulator) {
|
||||
defer wg.Done()
|
||||
|
||||
owner, repository, err := splitRepositoryName(repositoryName)
|
||||
if err != nil {
|
||||
acc.AddError(err)
|
||||
return
|
||||
}
|
||||
|
||||
repositoryInfo, response, err := g.githubClient.Repositories.Get(ctx, owner, repository)
|
||||
|
||||
if _, ok := err.(*github.RateLimitError); ok {
|
||||
g.RateLimitErrors.Incr(1)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
acc.AddError(err)
|
||||
return
|
||||
}
|
||||
|
||||
g.RateLimit.Set(int64(response.Rate.Limit))
|
||||
g.RateRemaining.Set(int64(response.Rate.Remaining))
|
||||
|
||||
now := time.Now()
|
||||
tags := getTags(repositoryInfo)
|
||||
fields := getFields(repositoryInfo)
|
||||
|
||||
acc.AddFields("github_repository", fields, tags, now)
|
||||
}(repository, acc)
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
return nil
|
||||
}
|
||||
|
||||
func splitRepositoryName(repositoryName string) (string, string, error) {
|
||||
splits := strings.SplitN(repositoryName, "/", 2)
|
||||
|
||||
if len(splits) != 2 {
|
||||
return "", "", fmt.Errorf("%v is not of format 'owner/repository'", repositoryName)
|
||||
}
|
||||
|
||||
return splits[0], splits[1], nil
|
||||
}
|
||||
|
||||
func getLicense(repositoryInfo *github.Repository) string {
|
||||
if repositoryInfo.GetLicense() != nil {
|
||||
return *repositoryInfo.License.Name
|
||||
}
|
||||
|
||||
return "None"
|
||||
}
|
||||
|
||||
func getTags(repositoryInfo *github.Repository) map[string]string {
|
||||
return map[string]string{
|
||||
"owner": *repositoryInfo.Owner.Login,
|
||||
"name": *repositoryInfo.Name,
|
||||
"language": *repositoryInfo.Language,
|
||||
"license": getLicense(repositoryInfo),
|
||||
}
|
||||
}
|
||||
|
||||
func getFields(repositoryInfo *github.Repository) map[string]interface{} {
|
||||
return map[string]interface{}{
|
||||
"stars": *repositoryInfo.StargazersCount,
|
||||
"forks": *repositoryInfo.ForksCount,
|
||||
"open_issues": *repositoryInfo.OpenIssuesCount,
|
||||
"size": *repositoryInfo.Size,
|
||||
}
|
||||
}
|
||||
|
||||
func init() {
|
||||
inputs.Add("github", func() telegraf.Input {
|
||||
return &GitHub{
|
||||
HTTPTimeout: internal.Duration{Duration: time.Second * 5},
|
||||
}
|
||||
})
|
||||
}
|
||||
|
|
@ -0,0 +1,119 @@
|
|||
package github
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
gh "github.com/google/go-github/github"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestSplitRepositoryNameWithWorkingExample(t *testing.T) {
|
||||
var validRepositoryNames = []struct {
|
||||
fullName string
|
||||
owner string
|
||||
repository string
|
||||
}{
|
||||
{"influxdata/telegraf", "influxdata", "telegraf"},
|
||||
{"influxdata/influxdb", "influxdata", "influxdb"},
|
||||
{"rawkode/saltstack-dotfiles", "rawkode", "saltstack-dotfiles"},
|
||||
}
|
||||
|
||||
for _, tt := range validRepositoryNames {
|
||||
t.Run(tt.fullName, func(t *testing.T) {
|
||||
owner, repository, _ := splitRepositoryName(tt.fullName)
|
||||
|
||||
require.Equal(t, tt.owner, owner)
|
||||
require.Equal(t, tt.repository, repository)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSplitRepositoryNameWithNoSlash(t *testing.T) {
|
||||
var invalidRepositoryNames = []string{
|
||||
"influxdata-influxdb",
|
||||
}
|
||||
|
||||
for _, tt := range invalidRepositoryNames {
|
||||
t.Run(tt, func(t *testing.T) {
|
||||
_, _, err := splitRepositoryName(tt)
|
||||
|
||||
require.NotNil(t, err)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetLicenseWhenExists(t *testing.T) {
|
||||
licenseName := "MIT"
|
||||
license := gh.License{Name: &licenseName}
|
||||
repository := gh.Repository{License: &license}
|
||||
|
||||
getLicenseReturn := getLicense(&repository)
|
||||
|
||||
require.Equal(t, "MIT", getLicenseReturn)
|
||||
}
|
||||
|
||||
func TestGetLicenseWhenMissing(t *testing.T) {
|
||||
repository := gh.Repository{}
|
||||
|
||||
getLicenseReturn := getLicense(&repository)
|
||||
|
||||
require.Equal(t, "None", getLicenseReturn)
|
||||
}
|
||||
|
||||
func TestGetTags(t *testing.T) {
|
||||
licenseName := "MIT"
|
||||
license := gh.License{Name: &licenseName}
|
||||
|
||||
ownerName := "influxdata"
|
||||
owner := gh.User{Login: &ownerName}
|
||||
|
||||
fullName := "influxdata/influxdb"
|
||||
repositoryName := "influxdb"
|
||||
|
||||
language := "Go"
|
||||
|
||||
repository := gh.Repository{
|
||||
FullName: &fullName,
|
||||
Name: &repositoryName,
|
||||
License: &license,
|
||||
Owner: &owner,
|
||||
Language: &language,
|
||||
}
|
||||
|
||||
getTagsReturn := getTags(&repository)
|
||||
|
||||
correctTagsReturn := map[string]string{
|
||||
"owner": ownerName,
|
||||
"name": repositoryName,
|
||||
"language": language,
|
||||
"license": licenseName,
|
||||
}
|
||||
|
||||
require.Equal(t, true, reflect.DeepEqual(getTagsReturn, correctTagsReturn))
|
||||
}
|
||||
|
||||
func TestGetFields(t *testing.T) {
|
||||
stars := 1
|
||||
forks := 2
|
||||
openIssues := 3
|
||||
size := 4
|
||||
|
||||
repository := gh.Repository{
|
||||
StargazersCount: &stars,
|
||||
ForksCount: &forks,
|
||||
OpenIssuesCount: &openIssues,
|
||||
Size: &size,
|
||||
}
|
||||
|
||||
getFieldsReturn := getFields(&repository)
|
||||
|
||||
correctFieldReturn := make(map[string]interface{})
|
||||
|
||||
correctFieldReturn["stars"] = 1
|
||||
correctFieldReturn["forks"] = 2
|
||||
correctFieldReturn["open_issues"] = 3
|
||||
correctFieldReturn["size"] = 4
|
||||
|
||||
require.Equal(t, true, reflect.DeepEqual(getFieldsReturn, correctFieldReturn))
|
||||
}
|
||||
Loading…
Reference in New Issue