From 2457d95262ec4d3ccde6a6925a16eedf675500f4 Mon Sep 17 00:00:00 2001 From: Nathan Haneysmith Date: Thu, 25 Aug 2016 17:46:38 -0700 Subject: [PATCH 1/7] Move CloudWatch rate limit to config Reference #1670 --- plugins/inputs/cloudwatch/README.md | 5 +++++ plugins/inputs/cloudwatch/cloudwatch.go | 13 +++++++------ 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/plugins/inputs/cloudwatch/README.md b/plugins/inputs/cloudwatch/README.md index df62e62bc..28d2a295e 100644 --- a/plugins/inputs/cloudwatch/README.md +++ b/plugins/inputs/cloudwatch/README.md @@ -34,6 +34,11 @@ API endpoint. In the following order the plugin will attempt to authenticate. ## Metric Statistic Namespace (required) namespace = 'AWS/ELB' + ## Maximum requests per second. Note that the global default AWS rate limit is + ## 10 reqs/sec, so if you define multiple namespaces, these should add up to a + ## maximum of 10. + ratelimit = 10 + ## Metrics to Pull (optional) ## Defaults to all Metrics in Namespace if nothing is provided ## Refreshes Namespace available metrics every 1h diff --git a/plugins/inputs/cloudwatch/cloudwatch.go b/plugins/inputs/cloudwatch/cloudwatch.go index f3019eb4b..2b6642b27 100644 --- a/plugins/inputs/cloudwatch/cloudwatch.go +++ b/plugins/inputs/cloudwatch/cloudwatch.go @@ -28,11 +28,12 @@ type ( Filename string `toml:"shared_credential_file"` Token string `toml:"token"` - Period internal.Duration `toml:"period"` - Delay internal.Duration `toml:"delay"` - Namespace string `toml:"namespace"` - Metrics []*Metric `toml:"metrics"` - CacheTTL internal.Duration `toml:"cache_ttl"` + Period internal.Duration `toml:"period"` + Delay internal.Duration `toml:"delay"` + Namespace string `toml:"namespace"` + Metrics []*Metric `toml:"metrics"` + CacheTTL internal.Duration `toml:"cache_ttl"` + RateLimit internal.RateLimit `toml:"ratelimit"` client cloudwatchClient metricCache *MetricCache } @@ -175,7 +176,7 @@ func (c *CloudWatch) Gather(acc telegraf.Accumulator) error { // limit concurrency or we can easily exhaust user connection limit // see cloudwatch API request limits: // http://docs.aws.amazon.com/AmazonCloudWatch/latest/DeveloperGuide/cloudwatch_limits.html - lmtr := limiter.NewRateLimiter(10, time.Second) + lmtr := limiter.NewRateLimiter(c.RateLimit, time.Second) defer lmtr.Stop() var wg sync.WaitGroup wg.Add(len(metrics)) From e931706249c7077b6a374f62e036fd3583f674c2 Mon Sep 17 00:00:00 2001 From: Nathan Haneysmith Date: Thu, 25 Aug 2016 17:53:46 -0700 Subject: [PATCH 2/7] make that variable a string --- plugins/inputs/cloudwatch/cloudwatch.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/plugins/inputs/cloudwatch/cloudwatch.go b/plugins/inputs/cloudwatch/cloudwatch.go index 2b6642b27..1eb173d7c 100644 --- a/plugins/inputs/cloudwatch/cloudwatch.go +++ b/plugins/inputs/cloudwatch/cloudwatch.go @@ -28,12 +28,12 @@ type ( Filename string `toml:"shared_credential_file"` Token string `toml:"token"` - Period internal.Duration `toml:"period"` - Delay internal.Duration `toml:"delay"` - Namespace string `toml:"namespace"` - Metrics []*Metric `toml:"metrics"` - CacheTTL internal.Duration `toml:"cache_ttl"` - RateLimit internal.RateLimit `toml:"ratelimit"` + Period internal.Duration `toml:"period"` + Delay internal.Duration `toml:"delay"` + Namespace string `toml:"namespace"` + Metrics []*Metric `toml:"metrics"` + CacheTTL internal.Duration `toml:"cache_ttl"` + RateLimit string `toml:"ratelimit"` client cloudwatchClient metricCache *MetricCache } From a0e23d30fe2491ab01bb4d68187e15ed4941980f Mon Sep 17 00:00:00 2001 From: Nathan Haneysmith Date: Thu, 25 Aug 2016 17:56:33 -0700 Subject: [PATCH 3/7] ahem, apparently limiter wants an int --- plugins/inputs/cloudwatch/cloudwatch.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/inputs/cloudwatch/cloudwatch.go b/plugins/inputs/cloudwatch/cloudwatch.go index 1eb173d7c..effcf62d0 100644 --- a/plugins/inputs/cloudwatch/cloudwatch.go +++ b/plugins/inputs/cloudwatch/cloudwatch.go @@ -33,7 +33,7 @@ type ( Namespace string `toml:"namespace"` Metrics []*Metric `toml:"metrics"` CacheTTL internal.Duration `toml:"cache_ttl"` - RateLimit string `toml:"ratelimit"` + RateLimit int `toml:"ratelimit"` client cloudwatchClient metricCache *MetricCache } From 4e019a176db0f09d140923939cf35a70c656fefd Mon Sep 17 00:00:00 2001 From: Nathan Haneysmith Date: Thu, 25 Aug 2016 18:04:29 -0700 Subject: [PATCH 4/7] add the ratelimit to the sample config --- plugins/inputs/cloudwatch/cloudwatch.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/plugins/inputs/cloudwatch/cloudwatch.go b/plugins/inputs/cloudwatch/cloudwatch.go index effcf62d0..3e5f71b90 100644 --- a/plugins/inputs/cloudwatch/cloudwatch.go +++ b/plugins/inputs/cloudwatch/cloudwatch.go @@ -97,6 +97,11 @@ func (c *CloudWatch) SampleConfig() string { ## Metric Statistic Namespace (required) namespace = 'AWS/ELB' + ## Maximum requests per second. Note that the global default AWS rate limit is + ## 10 reqs/sec, so if you define multiple namespaces, these should add up to a + ## maximum of 10. + ratelimit = 10 + ## Metrics to Pull (optional) ## Defaults to all Metrics in Namespace if nothing is provided ## Refreshes Namespace available metrics every 1h From 0589a1d0a52e7c626dc9a3bc3857c33c0bfa9c95 Mon Sep 17 00:00:00 2001 From: Nathan Haneysmith Date: Thu, 25 Aug 2016 18:17:33 -0700 Subject: [PATCH 5/7] update the test to include the rate --- plugins/inputs/cloudwatch/cloudwatch_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/plugins/inputs/cloudwatch/cloudwatch_test.go b/plugins/inputs/cloudwatch/cloudwatch_test.go index 8f8a3ad0b..73fca9253 100644 --- a/plugins/inputs/cloudwatch/cloudwatch_test.go +++ b/plugins/inputs/cloudwatch/cloudwatch_test.go @@ -58,6 +58,7 @@ func TestGather(t *testing.T) { Namespace: "AWS/ELB", Delay: internalDuration, Period: internalDuration, + RateLimit: 10, } var acc testutil.Accumulator From ca49babf3abe2712fb68d0e3400ba041b8cadc78 Mon Sep 17 00:00:00 2001 From: Nathan Haneysmith Date: Mon, 29 Aug 2016 11:41:43 -0700 Subject: [PATCH 6/7] set a default value of 10 for ratelimit --- plugins/inputs/cloudwatch/README.md | 2 +- plugins/inputs/cloudwatch/cloudwatch.go | 10 +++++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/plugins/inputs/cloudwatch/README.md b/plugins/inputs/cloudwatch/README.md index 28d2a295e..4430e48fd 100644 --- a/plugins/inputs/cloudwatch/README.md +++ b/plugins/inputs/cloudwatch/README.md @@ -36,7 +36,7 @@ API endpoint. In the following order the plugin will attempt to authenticate. ## Maximum requests per second. Note that the global default AWS rate limit is ## 10 reqs/sec, so if you define multiple namespaces, these should add up to a - ## maximum of 10. + ## maximum of 10. Optional - default value is 10. ratelimit = 10 ## Metrics to Pull (optional) diff --git a/plugins/inputs/cloudwatch/cloudwatch.go b/plugins/inputs/cloudwatch/cloudwatch.go index 3e5f71b90..d334f82d1 100644 --- a/plugins/inputs/cloudwatch/cloudwatch.go +++ b/plugins/inputs/cloudwatch/cloudwatch.go @@ -99,7 +99,7 @@ func (c *CloudWatch) SampleConfig() string { ## Maximum requests per second. Note that the global default AWS rate limit is ## 10 reqs/sec, so if you define multiple namespaces, these should add up to a - ## maximum of 10. + ## maximum of 10. Optional - default value is 10. ratelimit = 10 ## Metrics to Pull (optional) @@ -120,6 +120,8 @@ func (c *CloudWatch) Description() string { } func (c *CloudWatch) Gather(acc telegraf.Accumulator) error { + c.setDefaultValues() + if c.client == nil { c.initializeCloudWatch() } @@ -197,6 +199,12 @@ func (c *CloudWatch) Gather(acc telegraf.Accumulator) error { return errChan.Error() } +func (c *CloudWatch) setDefaultValues() { + if c.RateLimit == 0 { + c.RateLimit = 10 + } +} + func init() { inputs.Add("cloudwatch", func() telegraf.Input { ttl, _ := time.ParseDuration("1hr") From 7d8de4b8e116da357e31cdeaa79ed82fd210ab34 Mon Sep 17 00:00:00 2001 From: Nathan Haneysmith Date: Tue, 30 Aug 2016 14:33:51 -0700 Subject: [PATCH 7/7] Move default ratelimit to init --- plugins/inputs/cloudwatch/cloudwatch.go | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/plugins/inputs/cloudwatch/cloudwatch.go b/plugins/inputs/cloudwatch/cloudwatch.go index d334f82d1..ebc4147d8 100644 --- a/plugins/inputs/cloudwatch/cloudwatch.go +++ b/plugins/inputs/cloudwatch/cloudwatch.go @@ -120,8 +120,6 @@ func (c *CloudWatch) Description() string { } func (c *CloudWatch) Gather(acc telegraf.Accumulator) error { - c.setDefaultValues() - if c.client == nil { c.initializeCloudWatch() } @@ -199,17 +197,12 @@ func (c *CloudWatch) Gather(acc telegraf.Accumulator) error { return errChan.Error() } -func (c *CloudWatch) setDefaultValues() { - if c.RateLimit == 0 { - c.RateLimit = 10 - } -} - func init() { inputs.Add("cloudwatch", func() telegraf.Input { ttl, _ := time.ParseDuration("1hr") return &CloudWatch{ - CacheTTL: internal.Duration{Duration: ttl}, + CacheTTL: internal.Duration{Duration: ttl}, + RateLimit: 10, } }) }