Improve docs to clarify common issues (#5054)

plugins/inputs/logparser/README.md

@@ -76,6 +76,7 @@ line and the `semantic_name` is used to name the field or tag.  The extension
 other special handling.
 
 By default all named captures are converted into string fields.
+If a pattern does not have a semantic name it will not be captured.
 Timestamp modifiers can be used to convert captures to the timestamp of the
 parsed metric.  If no timestamp is parsed the metric will be created using the
 current time.

plugins/inputs/logparser/grok/patterns/influx-patterns

@@ -1,73 +0,0 @@
-# Captures are a slightly modified version of logstash "grok" patterns, with
-#  the format %{<capture syntax>[:<semantic name>][:<modifier>]}
-# By default all named captures are converted into string fields.
-# Modifiers can be used to convert captures to other types or tags.
-# Timestamp modifiers can be used to convert captures to the timestamp of the
-#  parsed metric.
-
-# View logstash grok pattern docs here:
-#   https://www.elastic.co/guide/en/logstash/current/plugins-filters-grok.html
-# All default logstash patterns are supported, these can be viewed here:
-#   https://github.com/logstash-plugins/logstash-patterns-core/blob/master/patterns/grok-patterns
-
-# Available modifiers:
-#   string   (default if nothing is specified)
-#   int
-#   float
-#   duration (ie, 5.23ms gets converted to int nanoseconds)
-#   tag      (converts the field into a tag)
-#   drop     (drops the field completely)
-# Timestamp modifiers:
-#   ts-ansic         ("Mon Jan _2 15:04:05 2006")
-#   ts-unix          ("Mon Jan _2 15:04:05 MST 2006")
-#   ts-ruby          ("Mon Jan 02 15:04:05 -0700 2006")
-#   ts-rfc822        ("02 Jan 06 15:04 MST")
-#   ts-rfc822z       ("02 Jan 06 15:04 -0700")
-#   ts-rfc850        ("Monday, 02-Jan-06 15:04:05 MST")
-#   ts-rfc1123       ("Mon, 02 Jan 2006 15:04:05 MST")
-#   ts-rfc1123z      ("Mon, 02 Jan 2006 15:04:05 -0700")
-#   ts-rfc3339       ("2006-01-02T15:04:05Z07:00")
-#   ts-rfc3339nano   ("2006-01-02T15:04:05.999999999Z07:00")
-#   ts-httpd         ("02/Jan/2006:15:04:05 -0700")
-#   ts-epoch         (seconds since unix epoch)
-#   ts-epochnano     (nanoseconds since unix epoch)
-#   ts-"CUSTOM"
-# CUSTOM time layouts must be within quotes and be the representation of the
-# "reference time", which is Mon Jan 2 15:04:05 -0700 MST 2006
-# See https://golang.org/pkg/time/#Parse for more details.
-
-# Example log file pattern, example log looks like this:
-#   [04/Jun/2016:12:41:45 +0100] 1.25 200 192.168.1.1 5.432µs
-# Breakdown of the DURATION pattern below:
-#   NUMBER  is a builtin logstash grok pattern matching float & int numbers.
-#   [nuµm]? is a regex specifying 0 or 1 of the characters within brackets.
-#   s       is also regex, this pattern must end in "s".
-# so DURATION will match something like '5.324ms' or '6.1µs' or '10s'
-DURATION %{NUMBER}[nuµm]?s
-RESPONSE_CODE %{NUMBER:response_code:tag}
-RESPONSE_TIME %{DURATION:response_time_ns:duration}
-EXAMPLE_LOG \[%{HTTPDATE:ts:ts-httpd}\] %{NUMBER:myfloat:float} %{RESPONSE_CODE} %{IPORHOST:clientip} %{RESPONSE_TIME}
-
-# Wider-ranging username matching vs. logstash built-in %{USER}
-NGUSERNAME [a-zA-Z0-9\.\@\-\+_%]+
-NGUSER %{NGUSERNAME}
-# Wider-ranging client IP matching
-CLIENT (?:%{IPORHOST}|%{HOSTPORT}|::1)
-
-##
-## COMMON LOG PATTERNS
-##
-
-# apache & nginx logs, this is also known as the "common log format"
-#   see https://en.wikipedia.org/wiki/Common_Log_Format
-COMMON_LOG_FORMAT %{CLIENT:client_ip} %{NOTSPACE:ident} %{NOTSPACE:auth} \[%{HTTPDATE:ts:ts-httpd}\] "(?:%{WORD:verb:tag} %{NOTSPACE:request}(?: HTTP/%{NUMBER:http_version:float})?|%{DATA})" %{NUMBER:resp_code:tag} (?:%{NUMBER:resp_bytes:int}|-)
-
-# Combined log format is the same as the common log format but with the addition
-# of two quoted strings at the end for "referrer" and "agent"
-#   See Examples at http://httpd.apache.org/docs/current/mod/mod_log_config.html
-COMBINED_LOG_FORMAT %{COMMON_LOG_FORMAT} %{QS:referrer} %{QS:agent}
-
-# HTTPD log formats
-HTTPD20_ERRORLOG \[%{HTTPDERROR_DATE:timestamp}\] \[%{LOGLEVEL:loglevel:tag}\] (?:\[client %{IPORHOST:clientip}\] ){0,1}%{GREEDYDATA:errormsg}
-HTTPD24_ERRORLOG \[%{HTTPDERROR_DATE:timestamp}\] \[%{WORD:module}:%{LOGLEVEL:loglevel:tag}\] \[pid %{POSINT:pid:int}:tid %{NUMBER:tid:int}\]( \(%{POSINT:proxy_errorcode:int}\)%{DATA:proxy_errormessage}:)?( \[client %{IPORHOST:client}:%{POSINT:clientport}\])? %{DATA:errorcode}: %{GREEDYDATA:message}
-HTTPD_ERRORLOG %{HTTPD20_ERRORLOG}|%{HTTPD24_ERRORLOG}

plugins/inputs/logparser/logparser_test.go

@@ -15,7 +15,7 @@ import (
 func TestStartNoParsers(t *testing.T) {
 	logparser := &LogParserPlugin{
 		FromBeginning: true,
-		Files:         []string{"grok/testdata/*.log"},
+		Files:         []string{"testdata/*.log"},
 	}
 
 	acc := testutil.Accumulator{}
@@ -27,10 +27,10 @@ func TestGrokParseLogFilesNonExistPattern(t *testing.T) {
 
 	logparser := &LogParserPlugin{
 		FromBeginning: true,
-		Files:         []string{thisdir + "grok/testdata/*.log"},
+		Files:         []string{thisdir + "testdata/*.log"},
 		GrokConfig: GrokConfig{
 			Patterns:           []string{"%{FOOBAR}"},
-			CustomPatternFiles: []string{thisdir + "grok/testdata/test-patterns"},
+			CustomPatternFiles: []string{thisdir + "testdata/test-patterns"},
 		},
 	}
 
@@ -46,10 +46,10 @@ func TestGrokParseLogFiles(t *testing.T) {
 		GrokConfig: GrokConfig{
 			MeasurementName:    "logparser_grok",
 			Patterns:           []string{"%{TEST_LOG_A}", "%{TEST_LOG_B}"},
-			CustomPatternFiles: []string{thisdir + "grok/testdata/test-patterns"},
+			CustomPatternFiles: []string{thisdir + "testdata/test-patterns"},
 		},
 		FromBeginning: true,
-		Files:         []string{thisdir + "grok/testdata/*.log"},
+		Files:         []string{thisdir + "testdata/*.log"},
 	}
 
 	acc := testutil.Accumulator{}
@@ -67,7 +67,7 @@ func TestGrokParseLogFiles(t *testing.T) {
 		},
 		map[string]string{
 			"response_code": "200",
-			"path":          thisdir + "grok/testdata/test_a.log",
+			"path":          thisdir + "testdata/test_a.log",
 		})
 
 	acc.AssertContainsTaggedFields(t, "logparser_grok",
@@ -77,7 +77,7 @@ func TestGrokParseLogFiles(t *testing.T) {
 			"nomodifier": "nomodifier",
 		},
 		map[string]string{
-			"path": thisdir + "grok/testdata/test_b.log",
+			"path": thisdir + "testdata/test_b.log",
 		})
 }
 
@@ -94,7 +94,7 @@ func TestGrokParseLogFilesAppearLater(t *testing.T) {
 		GrokConfig: GrokConfig{
 			MeasurementName:    "logparser_grok",
 			Patterns:           []string{"%{TEST_LOG_A}", "%{TEST_LOG_B}"},
-			CustomPatternFiles: []string{thisdir + "grok/testdata/test-patterns"},
+			CustomPatternFiles: []string{thisdir + "testdata/test-patterns"},
 		},
 	}
 
@@ -103,7 +103,7 @@ func TestGrokParseLogFilesAppearLater(t *testing.T) {
 
 	assert.Equal(t, acc.NFields(), 0)
 
-	_ = os.Symlink(thisdir+"grok/testdata/test_a.log", emptydir+"/test_a.log")
+	_ = os.Symlink(thisdir+"testdata/test_a.log", emptydir+"/test_a.log")
 	assert.NoError(t, acc.GatherError(logparser.Gather))
 	acc.Wait(1)
 
@@ -129,11 +129,11 @@ func TestGrokParseLogFilesOneBad(t *testing.T) {
 
 	logparser := &LogParserPlugin{
 		FromBeginning: true,
-		Files:         []string{thisdir + "grok/testdata/test_a.log"},
+		Files:         []string{thisdir + "testdata/test_a.log"},
 		GrokConfig: GrokConfig{
 			MeasurementName:    "logparser_grok",
 			Patterns:           []string{"%{TEST_LOG_A}", "%{TEST_LOG_BAD}"},
-			CustomPatternFiles: []string{thisdir + "grok/testdata/test-patterns"},
+			CustomPatternFiles: []string{thisdir + "testdata/test-patterns"},
 		},
 	}
 
@@ -153,7 +153,7 @@ func TestGrokParseLogFilesOneBad(t *testing.T) {
 		},
 		map[string]string{
 			"response_code": "200",
-			"path":          thisdir + "grok/testdata/test_a.log",
+			"path":          thisdir + "testdata/test_a.log",
 		})
 }