Add input plugin for OpenBSD/FreeBSD pf (#3405)
This commit is contained in:
Nathan Ferch
Daniel Nelson
parent
4337c98b41
commit
d758008c1e
|
|
@ -64,6 +64,7 @@ import (
|
||||||
_ "github.com/influxdata/telegraf/plugins/inputs/openldap"
|
_ "github.com/influxdata/telegraf/plugins/inputs/openldap"
|
||||||
_ "github.com/influxdata/telegraf/plugins/inputs/opensmtpd"
|
_ "github.com/influxdata/telegraf/plugins/inputs/opensmtpd"
|
||||||
_ "github.com/influxdata/telegraf/plugins/inputs/passenger"
|
_ "github.com/influxdata/telegraf/plugins/inputs/passenger"
|
||||||
|
_ "github.com/influxdata/telegraf/plugins/inputs/pf"
|
||||||
_ "github.com/influxdata/telegraf/plugins/inputs/phpfpm"
|
_ "github.com/influxdata/telegraf/plugins/inputs/phpfpm"
|
||||||
_ "github.com/influxdata/telegraf/plugins/inputs/ping"
|
_ "github.com/influxdata/telegraf/plugins/inputs/ping"
|
||||||
_ "github.com/influxdata/telegraf/plugins/inputs/postfix"
|
_ "github.com/influxdata/telegraf/plugins/inputs/postfix"
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,68 @@
|
||||||
|
# PF Plugin
|
||||||
|
|
||||||
|
The pf plugin gathers information from the FreeBSD/OpenBSD pf firewall. Currently it can retrive information about the state table: the number of current entries in the table, and counters for the number of searches, inserts, and removals to the table.
|
||||||
|
|
||||||
|
The pf plugin retrives this information by invoking the `pfstat` command. The `pfstat` command requires read access to the device file `/dev/pf`. You have several options to permit telegraf to run `pfctl`:
|
||||||
|
|
||||||
|
* Run telegraf as root. This is strongly discouraged.
|
||||||
|
* Change the ownership and permissions for /dev/pf such that the user telegraf runs at can read the /dev/pf device file. This is probably not that good of an idea either.
|
||||||
|
* Configure sudo to grant telegraf to run `pfctl` as root. This is the most restrictive option, but require sudo setup.
|
||||||
|
|
||||||
|
### Using sudo
|
||||||
|
|
||||||
|
You may edit your sudo configuration with the following:
|
||||||
|
|
||||||
|
```sudo
|
||||||
|
telegraf ALL=(root) NOPASSWD: /sbin/pfctl -s info
|
||||||
|
```
|
||||||
|
|
||||||
|
### Configuration:
|
||||||
|
|
||||||
|
```toml
|
||||||
|
# use sudo to run pfctl
|
||||||
|
use_sudo = false
|
||||||
|
```
|
||||||
|
|
||||||
|
### Measurements & Fields:
|
||||||
|
|
||||||
|
|
||||||
|
- pf
|
||||||
|
- entries (integer, count)
|
||||||
|
- searches (integer, count)
|
||||||
|
- inserts (integer, count)
|
||||||
|
- removals (integer, count)
|
||||||
|
|
||||||
|
### Example Output:
|
||||||
|
|
||||||
|
```
|
||||||
|
> pfctl -s info
|
||||||
|
Status: Enabled for 0 days 00:26:05 Debug: Urgent
|
||||||
|
|
||||||
|
State Table Total Rate
|
||||||
|
current entries 2
|
||||||
|
searches 11325 7.2/s
|
||||||
|
inserts 5 0.0/s
|
||||||
|
removals 3 0.0/s
|
||||||
|
Counters
|
||||||
|
match 11226 7.2/s
|
||||||
|
bad-offset 0 0.0/s
|
||||||
|
fragment 0 0.0/s
|
||||||
|
short 0 0.0/s
|
||||||
|
normalize 0 0.0/s
|
||||||
|
memory 0 0.0/s
|
||||||
|
bad-timestamp 0 0.0/s
|
||||||
|
congestion 0 0.0/s
|
||||||
|
ip-option 0 0.0/s
|
||||||
|
proto-cksum 0 0.0/s
|
||||||
|
state-mismatch 0 0.0/s
|
||||||
|
state-insert 0 0.0/s
|
||||||
|
state-limit 0 0.0/s
|
||||||
|
src-limit 0 0.0/s
|
||||||
|
synproxy 0 0.0/s
|
||||||
|
```
|
||||||
|
|
||||||
|
```
|
||||||
|
> ./telegraf --config telegraf.conf --input-filter pf --test
|
||||||
|
* Plugin: inputs.pf, Collection 1
|
||||||
|
> pf,host=columbia entries=3i,searches=2668i,inserts=12i,removals=9i 1510941775000000000
|
||||||
|
```
|
||||||
|
|
@ -0,0 +1,192 @@
|
||||||
|
package pf
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bufio"
|
||||||
|
"fmt"
|
||||||
|
"os/exec"
|
||||||
|
"regexp"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"github.com/influxdata/telegraf"
|
||||||
|
"github.com/influxdata/telegraf/plugins/inputs"
|
||||||
|
)
|
||||||
|
|
||||||
|
const measurement = "pf"
|
||||||
|
const pfctlCommand = "pfctl"
|
||||||
|
|
||||||
|
type PF struct {
|
||||||
|
PfctlCommand string
|
||||||
|
PfctlArgs []string
|
||||||
|
UseSudo bool
|
||||||
|
StateTable []*Entry
|
||||||
|
infoFunc func() (string, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (pf *PF) Description() string {
|
||||||
|
return "Gather counters from PF"
|
||||||
|
}
|
||||||
|
|
||||||
|
func (pf *PF) SampleConfig() string {
|
||||||
|
return `
|
||||||
|
## PF require root access on most systems.
|
||||||
|
## Setting 'use_sudo' to true will make use of sudo to run pfctl.
|
||||||
|
## Users must configure sudo to allow telegraf user to run pfctl with no password.
|
||||||
|
## pfctl can be restricted to only list command "pfctl -s info".
|
||||||
|
use_sudo = false
|
||||||
|
`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Gather is the entrypoint for the plugin.
|
||||||
|
func (pf *PF) Gather(acc telegraf.Accumulator) error {
|
||||||
|
if pf.PfctlCommand == "" {
|
||||||
|
var err error
|
||||||
|
if pf.PfctlCommand, pf.PfctlArgs, err = pf.buildPfctlCmd(); err != nil {
|
||||||
|
acc.AddError(fmt.Errorf("Can't construct pfctl commandline: %s", err))
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
o, err := pf.infoFunc()
|
||||||
|
if err != nil {
|
||||||
|
acc.AddError(err)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
if perr := pf.parsePfctlOutput(o, acc); perr != nil {
|
||||||
|
acc.AddError(perr)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
var errParseHeader = fmt.Errorf("Cannot find header in %s output", pfctlCommand)
|
||||||
|
|
||||||
|
func errMissingData(tag string) error {
|
||||||
|
return fmt.Errorf("struct data for tag \"%s\" not found in %s output", tag, pfctlCommand)
|
||||||
|
}
|
||||||
|
|
||||||
|
type pfctlOutputStanza struct {
|
||||||
|
HeaderRE *regexp.Regexp
|
||||||
|
ParseFunc func([]string, telegraf.Accumulator) error
|
||||||
|
Found bool
|
||||||
|
}
|
||||||
|
|
||||||
|
var pfctlOutputStanzas = []*pfctlOutputStanza{
|
||||||
|
&pfctlOutputStanza{
|
||||||
|
HeaderRE: regexp.MustCompile("^State Table"),
|
||||||
|
ParseFunc: parseStateTable,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
var anyTableHeaderRE = regexp.MustCompile("^[A-Z]")
|
||||||
|
|
||||||
|
func (pf *PF) parsePfctlOutput(pfoutput string, acc telegraf.Accumulator) error {
|
||||||
|
scanner := bufio.NewScanner(strings.NewReader(pfoutput))
|
||||||
|
for scanner.Scan() {
|
||||||
|
line := scanner.Text()
|
||||||
|
for _, s := range pfctlOutputStanzas {
|
||||||
|
if s.HeaderRE.MatchString(line) {
|
||||||
|
var stanzaLines []string
|
||||||
|
scanner.Scan()
|
||||||
|
line = scanner.Text()
|
||||||
|
for !anyTableHeaderRE.MatchString(line) {
|
||||||
|
stanzaLines = append(stanzaLines, line)
|
||||||
|
scanner.Scan()
|
||||||
|
line = scanner.Text()
|
||||||
|
}
|
||||||
|
if perr := s.ParseFunc(stanzaLines, acc); perr != nil {
|
||||||
|
return perr
|
||||||
|
}
|
||||||
|
s.Found = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, s := range pfctlOutputStanzas {
|
||||||
|
if !s.Found {
|
||||||
|
return errParseHeader
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
type Entry struct {
|
||||||
|
Field string
|
||||||
|
PfctlTitle string
|
||||||
|
Value int64
|
||||||
|
}
|
||||||
|
|
||||||
|
var StateTable = []*Entry{
|
||||||
|
&Entry{"entries", "current entries", -1},
|
||||||
|
&Entry{"searches", "searches", -1},
|
||||||
|
&Entry{"inserts", "inserts", -1},
|
||||||
|
&Entry{"removals", "removals", -1},
|
||||||
|
}
|
||||||
|
|
||||||
|
var stateTableRE = regexp.MustCompile(`^ (.*?)\s+(\d+)`)
|
||||||
|
|
||||||
|
func parseStateTable(lines []string, acc telegraf.Accumulator) error {
|
||||||
|
for _, v := range lines {
|
||||||
|
entries := stateTableRE.FindStringSubmatch(v)
|
||||||
|
if entries != nil {
|
||||||
|
for _, f := range StateTable {
|
||||||
|
if f.PfctlTitle == entries[1] {
|
||||||
|
var err error
|
||||||
|
if f.Value, err = strconv.ParseInt(entries[2], 10, 64); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fields := make(map[string]interface{})
|
||||||
|
for _, v := range StateTable {
|
||||||
|
if v.Value == -1 {
|
||||||
|
return errMissingData(v.PfctlTitle)
|
||||||
|
}
|
||||||
|
fields[v.Field] = v.Value
|
||||||
|
}
|
||||||
|
|
||||||
|
acc.AddFields(measurement, fields, make(map[string]string))
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (pf *PF) callPfctl() (string, error) {
|
||||||
|
cmd := execCommand(pf.PfctlCommand, pf.PfctlArgs...)
|
||||||
|
out, oerr := cmd.Output()
|
||||||
|
if oerr != nil {
|
||||||
|
ee, ok := oerr.(*exec.ExitError)
|
||||||
|
if !ok {
|
||||||
|
return string(out), fmt.Errorf("error running %s: %s: (unable to get stderr)", pfctlCommand, oerr)
|
||||||
|
}
|
||||||
|
return string(out), fmt.Errorf("error running %s: %s: %s", pfctlCommand, oerr, ee.Stderr)
|
||||||
|
}
|
||||||
|
return string(out), oerr
|
||||||
|
}
|
||||||
|
|
||||||
|
var execLookPath = exec.LookPath
|
||||||
|
var execCommand = exec.Command
|
||||||
|
|
||||||
|
func (pf *PF) buildPfctlCmd() (string, []string, error) {
|
||||||
|
cmd, err := execLookPath(pfctlCommand)
|
||||||
|
if err != nil {
|
||||||
|
return "", nil, fmt.Errorf("can't locate %s: %v", pfctlCommand, err)
|
||||||
|
}
|
||||||
|
args := []string{"-s", "info"}
|
||||||
|
if pf.UseSudo {
|
||||||
|
args = append([]string{cmd}, args...)
|
||||||
|
cmd, err = execLookPath("sudo")
|
||||||
|
if err != nil {
|
||||||
|
return "", nil, fmt.Errorf("can't locate sudo: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return cmd, args, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func init() {
|
||||||
|
inputs.Add("pf", func() telegraf.Input {
|
||||||
|
pf := new(PF)
|
||||||
|
pf.infoFunc = pf.callPfctl
|
||||||
|
return pf
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,243 @@
|
||||||
|
package pf
|
||||||
|
|
||||||
|
import (
|
||||||
|
"log"
|
||||||
|
"reflect"
|
||||||
|
"strconv"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/influxdata/telegraf/testutil"
|
||||||
|
)
|
||||||
|
|
||||||
|
type measurementResult struct {
|
||||||
|
tags map[string]string
|
||||||
|
fields map[string]interface{}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPfctlInvocation(t *testing.T) {
|
||||||
|
type pfctlInvocationTestCase struct {
|
||||||
|
config PF
|
||||||
|
cmd string
|
||||||
|
args []string
|
||||||
|
}
|
||||||
|
|
||||||
|
var testCases = []pfctlInvocationTestCase{
|
||||||
|
// 0: no sudo
|
||||||
|
pfctlInvocationTestCase{
|
||||||
|
config: PF{UseSudo: false},
|
||||||
|
cmd: "fakepfctl",
|
||||||
|
args: []string{"-s", "info"},
|
||||||
|
},
|
||||||
|
// 1: with sudo
|
||||||
|
pfctlInvocationTestCase{
|
||||||
|
config: PF{UseSudo: true},
|
||||||
|
cmd: "fakesudo",
|
||||||
|
args: []string{"fakepfctl", "-s", "info"},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for i, tt := range testCases {
|
||||||
|
execLookPath = func(cmd string) (string, error) { return "fake" + cmd, nil }
|
||||||
|
t.Run(strconv.Itoa(i), func(t *testing.T) {
|
||||||
|
log.Printf("running #%d\n", i)
|
||||||
|
cmd, args, err := tt.config.buildPfctlCmd()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("error when running buildPfctlCmd: %s", err)
|
||||||
|
}
|
||||||
|
if tt.cmd != cmd || !reflect.DeepEqual(tt.args, args) {
|
||||||
|
t.Errorf("%d: expected %s - %#v got %s - %#v", i, tt.cmd, tt.args, cmd, args)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPfMeasurements(t *testing.T) {
|
||||||
|
type pfTestCase struct {
|
||||||
|
TestInput string
|
||||||
|
err error
|
||||||
|
measurements []measurementResult
|
||||||
|
}
|
||||||
|
|
||||||
|
testCases := []pfTestCase{
|
||||||
|
// 0: nil input should raise an error
|
||||||
|
pfTestCase{TestInput: "", err: errParseHeader},
|
||||||
|
// 1: changes to pfctl output should raise an error
|
||||||
|
pfTestCase{TestInput: `Status: Enabled for 161 days 21:24:45 Debug: Urgent
|
||||||
|
|
||||||
|
Interface Stats for re1 IPv4 IPv6
|
||||||
|
Bytes In 2585823744614 1059233657221
|
||||||
|
Bytes Out 1227266932673 3274698578875
|
||||||
|
Packets In
|
||||||
|
Passed 2289953086 1945437219
|
||||||
|
Blocked 392835739 48609
|
||||||
|
Packets Out
|
||||||
|
Passed 1649146326 2605569054
|
||||||
|
Blocked 107 0
|
||||||
|
|
||||||
|
State Table Total Rate
|
||||||
|
Current Entrys 649
|
||||||
|
searches 18421725761 1317.0/s
|
||||||
|
inserts 156762508 11.2/s
|
||||||
|
removals 156761859 11.2/s
|
||||||
|
Counters
|
||||||
|
match 473002784 33.8/s
|
||||||
|
bad-offset 0 0.0/s
|
||||||
|
fragment 2729 0.0/s
|
||||||
|
short 107 0.0/s
|
||||||
|
normalize 1685 0.0/s
|
||||||
|
memory 101 0.0/s
|
||||||
|
bad-timestamp 0 0.0/s
|
||||||
|
congestion 0 0.0/s
|
||||||
|
ip-option 152301 0.0/s
|
||||||
|
proto-cksum 108 0.0/s
|
||||||
|
state-mismatch 24393 0.0/s
|
||||||
|
state-insert 92 0.0/s
|
||||||
|
state-limit 0 0.0/s
|
||||||
|
src-limit 0 0.0/s
|
||||||
|
synproxy 0 0.0/s
|
||||||
|
`,
|
||||||
|
err: errMissingData("current entries"),
|
||||||
|
},
|
||||||
|
// 2: bad numbers should raise an error
|
||||||
|
pfTestCase{TestInput: `Status: Enabled for 0 days 00:26:05 Debug: Urgent
|
||||||
|
|
||||||
|
State Table Total Rate
|
||||||
|
current entries -23
|
||||||
|
searches 11325 7.2/s
|
||||||
|
inserts 5 0.0/s
|
||||||
|
removals 3 0.0/s
|
||||||
|
Counters
|
||||||
|
match 11226 7.2/s
|
||||||
|
bad-offset 0 0.0/s
|
||||||
|
fragment 0 0.0/s
|
||||||
|
short 0 0.0/s
|
||||||
|
normalize 0 0.0/s
|
||||||
|
memory 0 0.0/s
|
||||||
|
bad-timestamp 0 0.0/s
|
||||||
|
congestion 0 0.0/s
|
||||||
|
ip-option 0 0.0/s
|
||||||
|
proto-cksum 0 0.0/s
|
||||||
|
state-mismatch 0 0.0/s
|
||||||
|
state-insert 0 0.0/s
|
||||||
|
state-limit 0 0.0/s
|
||||||
|
src-limit 0 0.0/s
|
||||||
|
synproxy 0 0.0/s
|
||||||
|
`,
|
||||||
|
err: errMissingData("current entries"),
|
||||||
|
},
|
||||||
|
pfTestCase{TestInput: `Status: Enabled for 0 days 00:26:05 Debug: Urgent
|
||||||
|
|
||||||
|
State Table Total Rate
|
||||||
|
current entries 2
|
||||||
|
searches 11325 7.2/s
|
||||||
|
inserts 5 0.0/s
|
||||||
|
removals 3 0.0/s
|
||||||
|
Counters
|
||||||
|
match 11226 7.2/s
|
||||||
|
bad-offset 0 0.0/s
|
||||||
|
fragment 0 0.0/s
|
||||||
|
short 0 0.0/s
|
||||||
|
normalize 0 0.0/s
|
||||||
|
memory 0 0.0/s
|
||||||
|
bad-timestamp 0 0.0/s
|
||||||
|
congestion 0 0.0/s
|
||||||
|
ip-option 0 0.0/s
|
||||||
|
proto-cksum 0 0.0/s
|
||||||
|
state-mismatch 0 0.0/s
|
||||||
|
state-insert 0 0.0/s
|
||||||
|
state-limit 0 0.0/s
|
||||||
|
src-limit 0 0.0/s
|
||||||
|
synproxy 0 0.0/s
|
||||||
|
`,
|
||||||
|
measurements: []measurementResult{
|
||||||
|
measurementResult{
|
||||||
|
fields: map[string]interface{}{
|
||||||
|
"entries": int64(2),
|
||||||
|
"searches": int64(11325),
|
||||||
|
"inserts": int64(5),
|
||||||
|
"removals": int64(3)},
|
||||||
|
tags: map[string]string{},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
pfTestCase{TestInput: `Status: Enabled for 161 days 21:24:45 Debug: Urgent
|
||||||
|
|
||||||
|
Interface Stats for re1 IPv4 IPv6
|
||||||
|
Bytes In 2585823744614 1059233657221
|
||||||
|
Bytes Out 1227266932673 3274698578875
|
||||||
|
Packets In
|
||||||
|
Passed 2289953086 1945437219
|
||||||
|
Blocked 392835739 48609
|
||||||
|
Packets Out
|
||||||
|
Passed 1649146326 2605569054
|
||||||
|
Blocked 107 0
|
||||||
|
|
||||||
|
State Table Total Rate
|
||||||
|
current entries 649
|
||||||
|
searches 18421725761 1317.0/s
|
||||||
|
inserts 156762508 11.2/s
|
||||||
|
removals 156761859 11.2/s
|
||||||
|
Counters
|
||||||
|
match 473002784 33.8/s
|
||||||
|
bad-offset 0 0.0/s
|
||||||
|
fragment 2729 0.0/s
|
||||||
|
short 107 0.0/s
|
||||||
|
normalize 1685 0.0/s
|
||||||
|
memory 101 0.0/s
|
||||||
|
bad-timestamp 0 0.0/s
|
||||||
|
congestion 0 0.0/s
|
||||||
|
ip-option 152301 0.0/s
|
||||||
|
proto-cksum 108 0.0/s
|
||||||
|
state-mismatch 24393 0.0/s
|
||||||
|
state-insert 92 0.0/s
|
||||||
|
state-limit 0 0.0/s
|
||||||
|
src-limit 0 0.0/s
|
||||||
|
synproxy 0 0.0/s
|
||||||
|
`,
|
||||||
|
measurements: []measurementResult{
|
||||||
|
measurementResult{
|
||||||
|
fields: map[string]interface{}{
|
||||||
|
"entries": int64(649),
|
||||||
|
"searches": int64(18421725761),
|
||||||
|
"inserts": int64(156762508),
|
||||||
|
"removals": int64(156761859)},
|
||||||
|
tags: map[string]string{},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for i, tt := range testCases {
|
||||||
|
t.Run(strconv.Itoa(i), func(t *testing.T) {
|
||||||
|
log.Printf("running #%d\n", i)
|
||||||
|
pf := &PF{
|
||||||
|
infoFunc: func() (string, error) {
|
||||||
|
return tt.TestInput, nil
|
||||||
|
},
|
||||||
|
}
|
||||||
|
acc := new(testutil.Accumulator)
|
||||||
|
err := acc.GatherError(pf.Gather)
|
||||||
|
if !reflect.DeepEqual(tt.err, err) {
|
||||||
|
t.Errorf("%d: expected error '%#v' got '%#v'", i, tt.err, err)
|
||||||
|
}
|
||||||
|
n := 0
|
||||||
|
for j, v := range tt.measurements {
|
||||||
|
if len(acc.Metrics) < n+1 {
|
||||||
|
t.Errorf("%d: expected at least %d values got %d", i, n+1, len(acc.Metrics))
|
||||||
|
break
|
||||||
|
}
|
||||||
|
m := acc.Metrics[n]
|
||||||
|
if !reflect.DeepEqual(m.Measurement, measurement) {
|
||||||
|
t.Errorf("%d %d: expected measurement '%#v' got '%#v'\n", i, j, measurement, m.Measurement)
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(m.Tags, v.tags) {
|
||||||
|
t.Errorf("%d %d: expected tags\n%#v got\n%#v\n", i, j, v.tags, m.Tags)
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(m.Fields, v.fields) {
|
||||||
|
t.Errorf("%d %d: expected fields\n%#v got\n%#v\n", i, j, v.fields, m.Fields)
|
||||||
|
}
|
||||||
|
n++
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
Loading…
Reference in New Issue