Internally name all patterns for log parsing flexibility

closes #1436

This also fixes the bad behavior of waiting until runtime to return log
parsing pattern compile errors when a pattern was simply unfound.

closes #1418

Also protect against user error when the telegraf user does not have
permission to open the provided file. We will now error and exit in this
case, rather than silently waiting to get permission to open it.

plugins/inputs/logparser/grok/grok.go

@@ -53,7 +53,12 @@ var (
 )
 
 type Parser struct {
-	Patterns           []string
+	Patterns []string
+	// namedPatterns is a list of internally-assigned names to the patterns
+	// specified by the user in Patterns.
+	// They will look like:
+	//   GROK_INTERNAL_PATTERN_0, GROK_INTERNAL_PATTERN_1, etc.
+	namedPatterns      []string
 	CustomPatterns     string
 	CustomPatternFiles []string
 	Measurement        string
@@ -98,13 +103,24 @@ func (p *Parser) Compile() error {
 		return err
 	}
 
-	p.CustomPatterns = DEFAULT_PATTERNS + p.CustomPatterns
+	// Give Patterns fake names so that they can be treated as named
+	// "custom patterns"
+	p.namedPatterns = make([]string, len(p.Patterns))
+	for i, pattern := range p.Patterns {
+		name := fmt.Sprintf("GROK_INTERNAL_PATTERN_%d", i)
+		p.CustomPatterns += "\n" + name + " " + pattern + "\n"
+		p.namedPatterns[i] = "%{" + name + "}"
+	}
 
+	// Combine user-supplied CustomPatterns with DEFAULT_PATTERNS and parse
+	// them together as the same type of pattern.
+	p.CustomPatterns = DEFAULT_PATTERNS + p.CustomPatterns
 	if len(p.CustomPatterns) != 0 {
 		scanner := bufio.NewScanner(strings.NewReader(p.CustomPatterns))
 		p.addCustomPatterns(scanner)
 	}
 
+	// Parse any custom pattern files supplied.
 	for _, filename := range p.CustomPatternFiles {
 		file, err := os.Open(filename)
 		if err != nil {
@@ -127,7 +143,7 @@ func (p *Parser) ParseLine(line string) (telegraf.Metric, error) {
 	var values map[string]string
 	// the matching pattern string
 	var patternName string
-	for _, pattern := range p.Patterns {
+	for _, pattern := range p.namedPatterns {
 		if values, err = p.g.Parse(pattern, line); err != nil {
 			return nil, err
 		}

plugins/inputs/logparser/grok/grok_test.go

@@ -207,7 +207,7 @@ func TestBuiltinCombinedLogFormat(t *testing.T) {
 
 func TestCompileStringAndParse(t *testing.T) {
 	p := &Parser{
-		Patterns: []string{"%{TEST_LOG_A}", "%{TEST_LOG_B}"},
+		Patterns: []string{"%{TEST_LOG_A}"},
 		CustomPatterns: `
 			DURATION %{NUMBER}[nuµm]?s
 			RESPONSE_CODE %{NUMBER:response_code:tag}
@@ -230,6 +230,41 @@ func TestCompileStringAndParse(t *testing.T) {
 	assert.Equal(t, map[string]string{"response_code": "200"}, metricA.Tags())
 }
 
+func TestCompileErrorsOnInvalidPattern(t *testing.T) {
+	p := &Parser{
+		Patterns: []string{"%{TEST_LOG_A}", "%{TEST_LOG_B}"},
+		CustomPatterns: `
+			DURATION %{NUMBER}[nuµm]?s
+			RESPONSE_CODE %{NUMBER:response_code:tag}
+			RESPONSE_TIME %{DURATION:response_time:duration}
+			TEST_LOG_A %{NUMBER:myfloat:float} %{RESPONSE_CODE} %{IPORHOST:clientip} %{RESPONSE_TIME}
+		`,
+	}
+	assert.Error(t, p.Compile())
+
+	metricA, _ := p.ParseLine(`1.25 200 192.168.1.1 5.432µs`)
+	require.Nil(t, metricA)
+}
+
+func TestParsePatternsWithoutCustom(t *testing.T) {
+	p := &Parser{
+		Patterns: []string{"%{POSINT:ts:ts-epochnano} response_time=%{POSINT:response_time:int} mymetric=%{NUMBER:metric:float}"},
+	}
+	assert.NoError(t, p.Compile())
+
+	metricA, err := p.ParseLine(`1466004605359052000 response_time=20821 mymetric=10890.645`)
+	require.NotNil(t, metricA)
+	assert.NoError(t, err)
+	assert.Equal(t,
+		map[string]interface{}{
+			"response_time": int64(20821),
+			"metric":        float64(10890.645),
+		},
+		metricA.Fields())
+	assert.Equal(t, map[string]string{}, metricA.Tags())
+	assert.Equal(t, time.Unix(0, 1466004605359052000), metricA.Time())
+}
+
 func TestParseEpochNano(t *testing.T) {
 	p := &Parser{
 		Patterns: []string{"%{MYAPP}"},
@@ -413,7 +448,7 @@ func TestParseErrors(t *testing.T) {
 			TEST_LOG_A %{HTTPDATE:ts:ts-httpd} %{WORD:myword:int} %{}
 		`,
 	}
-	assert.NoError(t, p.Compile())
+	assert.Error(t, p.Compile())
 	_, err := p.ParseLine(`[04/Jun/2016:12:41:45 +0100] notnumber 200 192.168.1.1 5.432µs 101`)
 	assert.Error(t, err)