Consolidate AWS credentials (#1208)

* Use shared AWS credential configuration. * Cloudwatch dimension wilcards * Allow configuring cache_ttl for cloudwatch metrics. * Allow for wildcard in dimension values to select all available metrics. * Use internal.Duration for CacheTTL and go fmt * Refactor to not use embedded structs for config. * Update AWS plugin READMEs with credentials details, update Changelog. * Fix changelog after rebasing to master and 0.13.1 release. * Fix changelog after rebase.
2016-05-25 06:30:39 -05:00
parent 6351aa5167
commit 5f3a91bffd
8 changed files with 283 additions and 73 deletions
--- a/plugins/inputs/cloudwatch/README.md
+++ b/plugins/inputs/cloudwatch/README.md
@@ -6,9 +6,12 @@ This plugin will pull Metric Statistics from Amazon CloudWatch.

 This plugin uses a credential chain for Authentication with the CloudWatch
 API endpoint. In the following order the plugin will attempt to authenticate.
-1. [IAMS Role](http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html)
-2. [Environment Variables](https://github.com/aws/aws-sdk-go/wiki/configuring-sdk#environment-variables)
-3. [Shared Credentials](https://github.com/aws/aws-sdk-go/wiki/configuring-sdk#shared-credentials-file)
+1. Assumed credentials via STS if `role_arn` attribute is specified (source credentials are evaluated from subsequent rules)
+2. Explicit credentials from `access_key`, `secret_key`, and `token` attributes
+3. Shared profile from `profile` attribute
+4. [Environment Variables](https://github.com/aws/aws-sdk-go/wiki/configuring-sdk#environment-variables)
+5. [Shared Credentials](https://github.com/aws/aws-sdk-go/wiki/configuring-sdk#shared-credentials-file)
+6. [EC2 Instance Profile](http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html)

 ### Configuration:

@@ -24,7 +27,7 @@ API endpoint. In the following order the plugin will attempt to authenticate.
  delay = '1m'

  ## Override global run interval (optional - defaults to global interval)
-  ## Recomended: use metric 'interval' that is a multiple of 'period' to avoid 
+  ## Recomended: use metric 'interval' that is a multiple of 'period' to avoid
  ## gaps or overlap in pulled data
  interval = '1m'

@@ -36,11 +39,15 @@ API endpoint. In the following order the plugin will attempt to authenticate.
  ## Refreshes Namespace available metrics every 1h
  [[inputs.cloudwatch.metrics]]
    names = ['Latency', 'RequestCount']
-	
+
    ## Dimension filters for Metric (optional)
    [[inputs.cloudwatch.metrics.dimensions]]
      name = 'LoadBalancerName'
      value = 'p-example'
+
+    [[inputs.cloudwatch.metrics.dimensions]]
+      name = 'AvailabilityZone'
+      value = '*'
 ```
 #### Requirements and Terminology

@@ -52,6 +59,39 @@ Plugin Configuration utilizes [CloudWatch concepts](http://docs.aws.amazon.com/A
 - `names` must be valid CloudWatch [Metric](http://docs.aws.amazon.com/AmazonCloudWatch/latest/DeveloperGuide/cloudwatch_concepts.html#Metric) names
 - `dimensions` must be valid CloudWatch [Dimension](http://docs.aws.amazon.com/AmazonCloudWatch/latest/DeveloperGuide/cloudwatch_concepts.html#Dimension) name/value pairs

+Omitting or specifying a value of `'*'` for a dimension value configures all available metrics that contain a dimension with the specified name
+to be retrieved. If specifying >1 dimension, then the metric must contain *all* the configured dimensions where the the value of the
+wildcard dimension is ignored.
+
+Example:
+```
+[[inputs.cloudwatch.metrics]]
+  names = ['Latency']
+
+  ## Dimension filters for Metric (optional)
+  [[inputs.cloudwatch.metrics.dimensions]]
+    name = 'LoadBalancerName'
+    value = 'p-example'
+
+  [[inputs.cloudwatch.metrics.dimensions]]
+    name = 'AvailabilityZone'
+    value = '*'
+```
+
+If the following ELBs are available:
+- name: `p-example`, availabilityZone: `us-east-1a`
+- name: `p-example`, availabilityZone: `us-east-1b`
+- name: `q-example`, availabilityZone: `us-east-1a`
+- name: `q-example`, availabilityZone: `us-east-1b`
+
+
+Then 2 metrics will be output:
+- name: `p-example`, availabilityZone: `us-east-1a`
+- name: `p-example`, availabilityZone: `us-east-1b`
+
+If the `AvailabilityZone` wildcard dimension was omitted, then a single metric (name: `p-example`)
+would be exported containing the aggregate values of the ELB across availability zones.
+
 #### Restrictions and Limitations
 - CloudWatch metrics are not available instantly via the CloudWatch API. You should adjust your collection `delay` to account for this lag in metrics availability based on your [monitoring subscription level](http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-cloudwatch-new.html)
 - CloudWatch API usage incurs cost - see [GetMetricStatistics Pricing](https://aws.amazon.com/cloudwatch/pricing/)
--- a/plugins/inputs/cloudwatch/cloudwatch.go
+++ b/plugins/inputs/cloudwatch/cloudwatch.go
@@ -6,25 +6,30 @@ import (
 	"time"

 	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/aws/credentials"
-	"github.com/aws/aws-sdk-go/aws/session"

 	"github.com/aws/aws-sdk-go/service/cloudwatch"

 	"github.com/influxdata/telegraf"
 	"github.com/influxdata/telegraf/internal"
+	internalaws "github.com/influxdata/telegraf/internal/config/aws"
 	"github.com/influxdata/telegraf/plugins/inputs"
 )

 type (
 	CloudWatch struct {
-		Region      string            `toml:"region"`
-		AccessKey   string            `toml:"access_key"`
-		SecretKey   string            `toml:"secret_key"`
+		Region    string `toml:"region"`
+		AccessKey string `toml:"access_key"`
+		SecretKey string `toml:"secret_key"`
+		RoleARN   string `toml:"role_arn"`
+		Profile   string `toml:"profile"`
+		Filename  string `toml:"shared_credential_file"`
+		Token     string `toml:"token"`
+
 		Period      internal.Duration `toml:"period"`
 		Delay       internal.Duration `toml:"delay"`
 		Namespace   string            `toml:"namespace"`
 		Metrics     []*Metric         `toml:"metrics"`
+		CacheTTL    internal.Duration `toml:"cache_ttl"`
 		client      cloudwatchClient
 		metricCache *MetricCache
 	}
@@ -58,12 +63,18 @@ func (c *CloudWatch) SampleConfig() string {

  ## Amazon Credentials
  ## Credentials are loaded in the following order
-  ## 1) explicit credentials from 'access_key' and 'secret_key'
-  ## 2) environment variables
-  ## 3) shared credentials file
-  ## 4) EC2 Instance Profile
+  ## 1) Assumed credentials via STS if role_arn is specified
+  ## 2) explicit credentials from 'access_key' and 'secret_key'
+  ## 3) shared profile from 'profile'
+  ## 4) environment variables
+  ## 5) shared credentials file
+  ## 6) EC2 Instance Profile
  #access_key = ""
  #secret_key = ""
+  #token = ""
+  #role_arn = ""
+  #profile = ""
+  #shared_credential_file = ""

  ## Requested CloudWatch aggregation Period (required - must be a multiple of 60s)
  period = '1m'
@@ -75,6 +86,10 @@ func (c *CloudWatch) SampleConfig() string {
  ## gaps or overlap in pulled data
  interval = '1m'

+  ## Configure the TTL for the internal cache of metrics.
+  ## Defaults to 1 hr if not specified
+  #cache_ttl = '10m'
+
  ## Metric Statistic Namespace (required)
  namespace = 'AWS/ELB'

@@ -106,20 +121,40 @@ func (c *CloudWatch) Gather(acc telegraf.Accumulator) error {
 	if c.Metrics != nil {
 		metrics = []*cloudwatch.Metric{}
 		for _, m := range c.Metrics {
-			dimensions := make([]*cloudwatch.Dimension, len(m.Dimensions))
-			for k, d := range m.Dimensions {
-				dimensions[k] = &cloudwatch.Dimension{
-					Name:  aws.String(d.Name),
-					Value: aws.String(d.Value),
+			if !hasWilcard(m.Dimensions) {
+				dimensions := make([]*cloudwatch.Dimension, len(m.Dimensions))
+				for k, d := range m.Dimensions {
+					fmt.Printf("Dimension [%s]:[%s]\n", d.Name, d.Value)
+					dimensions[k] = &cloudwatch.Dimension{
+						Name:  aws.String(d.Name),
+						Value: aws.String(d.Value),
+					}
+				}
+				for _, name := range m.MetricNames {
+					metrics = append(metrics, &cloudwatch.Metric{
+						Namespace:  aws.String(c.Namespace),
+						MetricName: aws.String(name),
+						Dimensions: dimensions,
+					})
+				}
+			} else {
+				allMetrics, err := c.fetchNamespaceMetrics()
+				if err != nil {
+					return err
+				}
+				for _, name := range m.MetricNames {
+					for _, metric := range allMetrics {
+						if isSelected(metric, m.Dimensions) {
+							metrics = append(metrics, &cloudwatch.Metric{
+								Namespace:  aws.String(c.Namespace),
+								MetricName: aws.String(name),
+								Dimensions: metric.Dimensions,
+							})
+						}
+					}
 				}
 			}
-			for _, name := range m.MetricNames {
-				metrics = append(metrics, &cloudwatch.Metric{
-					Namespace:  aws.String(c.Namespace),
-					MetricName: aws.String(name),
-					Dimensions: dimensions,
-				})
-			}
+
 		}
 	} else {
 		var err error
@@ -153,7 +188,10 @@ func (c *CloudWatch) Gather(acc telegraf.Accumulator) error {

 func init() {
 	inputs.Add("cloudwatch", func() telegraf.Input {
-		return &CloudWatch{}
+		ttl, _ := time.ParseDuration("1hr")
+		return &CloudWatch{
+			CacheTTL: internal.Duration{Duration: ttl},
+		}
 	})
 }

@@ -161,14 +199,18 @@ func init() {
 * Initialize CloudWatch client
 */
 func (c *CloudWatch) initializeCloudWatch() error {
-	config := &aws.Config{
-		Region: aws.String(c.Region),
-	}
-	if c.AccessKey != "" || c.SecretKey != "" {
-		config.Credentials = credentials.NewStaticCredentials(c.AccessKey, c.SecretKey, "")
+	credentialConfig := &internalaws.CredentialConfig{
+		Region:    c.Region,
+		AccessKey: c.AccessKey,
+		SecretKey: c.SecretKey,
+		RoleARN:   c.RoleARN,
+		Profile:   c.Profile,
+		Filename:  c.Filename,
+		Token:     c.Token,
 	}
+	configProvider := credentialConfig.Credentials()

-	c.client = cloudwatch.New(session.New(config))
+	c.client = cloudwatch.New(configProvider)
 	return nil
 }

@@ -203,11 +245,10 @@ func (c *CloudWatch) fetchNamespaceMetrics() (metrics []*cloudwatch.Metric, err
 		more = token != nil
 	}

-	cacheTTL, _ := time.ParseDuration("1hr")
 	c.metricCache = &MetricCache{
 		Metrics: metrics,
 		Fetched: time.Now(),
-		TTL:     cacheTTL,
+		TTL:     c.CacheTTL.Duration,
 	}

 	return
@@ -309,3 +350,32 @@ func (c *CloudWatch) getStatisticsInput(metric *cloudwatch.Metric, now time.Time
 func (c *MetricCache) IsValid() bool {
 	return c.Metrics != nil && time.Since(c.Fetched) < c.TTL
 }
+
+func hasWilcard(dimensions []*Dimension) bool {
+	for _, d := range dimensions {
+		if d.Value == "" || d.Value == "*" {
+			return true
+		}
+	}
+	return false
+}
+
+func isSelected(metric *cloudwatch.Metric, dimensions []*Dimension) bool {
+	if len(metric.Dimensions) != len(dimensions) {
+		return false
+	}
+	for _, d := range dimensions {
+		selected := false
+		for _, d2 := range metric.Dimensions {
+			if d.Name == *d2.Name {
+				if d.Value == "" || d.Value == "*" || d.Value == *d2.Value {
+					selected = true
+				}
+			}
+		}
+		if !selected {
+			return false
+		}
+	}
+	return true
+}
--- a/plugins/outputs/cloudwatch/README.md
+++ b/plugins/outputs/cloudwatch/README.md
@@ -6,9 +6,12 @@ This plugin will send metrics to Amazon CloudWatch.

 This plugin uses a credential chain for Authentication with the CloudWatch
 API endpoint. In the following order the plugin will attempt to authenticate.
-1. [IAMS Role](http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html)
-2. [Environment Variables](https://github.com/aws/aws-sdk-go/wiki/configuring-sdk)
-3. [Shared Credentials](https://github.com/aws/aws-sdk-go/wiki/configuring-sdk)
+1. Assumed credentials via STS if `role_arn` attribute is specified (source credentials are evaluated from subsequent rules)
+2. Explicit credentials from `access_key`, `secret_key`, and `token` attributes
+3. Shared profile from `profile` attribute
+4. [Environment Variables](https://github.com/aws/aws-sdk-go/wiki/configuring-sdk#environment-variables)
+5. [Shared Credentials](https://github.com/aws/aws-sdk-go/wiki/configuring-sdk#shared-credentials-file)
+6. [EC2 Instance Profile](http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html)

 ## Config

--- a/plugins/outputs/cloudwatch/cloudwatch.go
+++ b/plugins/outputs/cloudwatch/cloudwatch.go
@@ -8,19 +8,23 @@ import (
 	"time"

 	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/aws/credentials"
-	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/cloudwatch"

 	"github.com/influxdata/telegraf"
+	internalaws "github.com/influxdata/telegraf/internal/config/aws"
 	"github.com/influxdata/telegraf/plugins/outputs"
 )

 type CloudWatch struct {
-	Region    string `toml:"region"`     // AWS Region
-	AccessKey string `toml:"access_key"` // Explicit AWS Access Key ID
-	SecretKey string `toml:"secret_key"` // Explicit AWS Secret Access Key
-	Namespace string `toml:"namespace"`  // CloudWatch Metrics Namespace
+	Region    string `toml:"region"`
+	AccessKey string `toml:"access_key"`
+	SecretKey string `toml:"secret_key"`
+	RoleARN   string `toml:"role_arn"`
+	Profile   string `toml:"profile"`
+	Filename  string `toml:"shared_credential_file"`
+	Token     string `toml:"token"`
+
+	Namespace string `toml:"namespace"` // CloudWatch Metrics Namespace
 	svc       *cloudwatch.CloudWatch
 }

@@ -30,12 +34,18 @@ var sampleConfig = `

  ## Amazon Credentials
  ## Credentials are loaded in the following order
-  ## 1) explicit credentials from 'access_key' and 'secret_key'
-  ## 2) environment variables
-  ## 3) shared credentials file
-  ## 4) EC2 Instance Profile
+  ## 1) Assumed credentials via STS if role_arn is specified
+  ## 2) explicit credentials from 'access_key' and 'secret_key'
+  ## 3) shared profile from 'profile'
+  ## 4) environment variables
+  ## 5) shared credentials file
+  ## 6) EC2 Instance Profile
  #access_key = ""
  #secret_key = ""
+  #token = ""
+  #role_arn = ""
+  #profile = ""
+  #shared_credential_file = ""

  ## Namespace for the CloudWatch MetricDatums
  namespace = 'InfluxData/Telegraf'
@@ -50,14 +60,18 @@ func (c *CloudWatch) Description() string {
 }

 func (c *CloudWatch) Connect() error {
-	Config := &aws.Config{
-		Region: aws.String(c.Region),
-	}
-	if c.AccessKey != "" || c.SecretKey != "" {
-		Config.Credentials = credentials.NewStaticCredentials(c.AccessKey, c.SecretKey, "")
+	credentialConfig := &internalaws.CredentialConfig{
+		Region:    c.Region,
+		AccessKey: c.AccessKey,
+		SecretKey: c.SecretKey,
+		RoleARN:   c.RoleARN,
+		Profile:   c.Profile,
+		Filename:  c.Filename,
+		Token:     c.Token,
 	}
+	configProvider := credentialConfig.Credentials()

-	svc := cloudwatch.New(session.New(Config))
+	svc := cloudwatch.New(configProvider)

 	params := &cloudwatch.ListMetricsInput{
 		Namespace: aws.String(c.Namespace),
--- a/plugins/outputs/kinesis/README.md
+++ b/plugins/outputs/kinesis/README.md
@@ -13,9 +13,12 @@ maybe useful for users to review Amazons official documentation which is availab

 This plugin uses a credential chain for Authentication with the Kinesis API endpoint. In the following order the plugin
 will attempt to authenticate.
-1. [IAMS Role](http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html)
-2. [Environment Variables](https://github.com/aws/aws-sdk-go/wiki/configuring-sdk)
-3. [Shared Credentials](https://github.com/aws/aws-sdk-go/wiki/configuring-sdk)
+1. Assumed credentials via STS if `role_arn` attribute is specified (source credentials are evaluated from subsequent rules)
+2. Explicit credentials from `access_key`, `secret_key`, and `token` attributes
+3. Shared profile from `profile` attribute
+4. [Environment Variables](https://github.com/aws/aws-sdk-go/wiki/configuring-sdk#environment-variables)
+5. [Shared Credentials](https://github.com/aws/aws-sdk-go/wiki/configuring-sdk#shared-credentials-file)
+6. [EC2 Instance Profile](http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html)


 ## Config
@@ -58,4 +61,4 @@ String is defined using the default Point.String() value and translated to []byt

 #### custom

-Custom is a string defined by a number of values in the FormatMetric() function.
+Custom is a string defined by a number of values in the FormatMetric() function.
--- a/plugins/outputs/kinesis/kinesis.go
+++ b/plugins/outputs/kinesis/kinesis.go
@@ -8,18 +8,22 @@ import (
 	"time"

 	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/aws/credentials"
-	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/kinesis"

 	"github.com/influxdata/telegraf"
+	internalaws "github.com/influxdata/telegraf/internal/config/aws"
 	"github.com/influxdata/telegraf/plugins/outputs"
 )

 type KinesisOutput struct {
-	Region       string `toml:"region"`
-	AccessKey    string `toml:"access_key"`
-	SecretKey    string `toml:"secret_key"`
+	Region    string `toml:"region"`
+	AccessKey string `toml:"access_key"`
+	SecretKey string `toml:"secret_key"`
+	RoleARN   string `toml:"role_arn"`
+	Profile   string `toml:"profile"`
+	Filename  string `toml:"shared_credential_file"`
+	Token     string `toml:"token"`
+
 	StreamName   string `toml:"streamname"`
 	PartitionKey string `toml:"partitionkey"`
 	Format       string `toml:"format"`
@@ -33,12 +37,18 @@ var sampleConfig = `

  ## Amazon Credentials
  ## Credentials are loaded in the following order
-  ## 1) explicit credentials from 'access_key' and 'secret_key'
-  ## 2) environment variables
-  ## 3) shared credentials file
-  ## 4) EC2 Instance Profile
+  ## 1) Assumed credentials via STS if role_arn is specified
+  ## 2) explicit credentials from 'access_key' and 'secret_key'
+  ## 3) shared profile from 'profile'
+  ## 4) environment variables
+  ## 5) shared credentials file
+  ## 6) EC2 Instance Profile
  #access_key = ""
  #secret_key = ""
+  #token = ""
+  #role_arn = ""
+  #profile = ""
+  #shared_credential_file = ""

  ## Kinesis StreamName must exist prior to starting telegraf.
  streamname = "StreamName"
@@ -75,13 +85,18 @@ func (k *KinesisOutput) Connect() error {
 	if k.Debug {
 		log.Printf("kinesis: Establishing a connection to Kinesis in %+v", k.Region)
 	}
-	Config := &aws.Config{
-		Region: aws.String(k.Region),
+
+	credentialConfig := &internalaws.CredentialConfig{
+		Region:    k.Region,
+		AccessKey: k.AccessKey,
+		SecretKey: k.SecretKey,
+		RoleARN:   k.RoleARN,
+		Profile:   k.Profile,
+		Filename:  k.Filename,
+		Token:     k.Token,
 	}
-	if k.AccessKey != "" || k.SecretKey != "" {
-		Config.Credentials = credentials.NewStaticCredentials(k.AccessKey, k.SecretKey, "")
-	}
-	svc := kinesis.New(session.New(Config))
+	configProvider := credentialConfig.Credentials()
+	svc := kinesis.New(configProvider)

 	KinesisParams := &kinesis.ListStreamsInput{
 		Limit: aws.Int64(100),