Send TERM to exec processes before sending KILL signal (#6302)

This commit is contained in:
Daniel Nelson 2019-08-22 10:50:02 -07:00 committed by GitHub
parent 0a4d74c827
commit b578586d4a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 129 additions and 49 deletions

30
internal/exec.go Normal file
View File

@ -0,0 +1,30 @@
package internal
import (
"bytes"
"os/exec"
"time"
)
// CombinedOutputTimeout runs the given command with the given timeout and
// returns the combined output of stdout and stderr.
// If the command times out, it attempts to kill the process.
func CombinedOutputTimeout(c *exec.Cmd, timeout time.Duration) ([]byte, error) {
var b bytes.Buffer
c.Stdout = &b
c.Stderr = &b
if err := c.Start(); err != nil {
return nil, err
}
err := WaitTimeout(c, timeout)
return b.Bytes(), err
}
// RunTimeout runs the given command with the given timeout.
// If the command times out, it attempts to kill the process.
func RunTimeout(c *exec.Cmd, timeout time.Duration) error {
if err := c.Start(); err != nil {
return err
}
return WaitTimeout(c, timeout)
}

58
internal/exec_unix.go Normal file
View File

@ -0,0 +1,58 @@
// +build !windows
package internal
import (
"log"
"os/exec"
"syscall"
"time"
)
// KillGrace is the amount of time we allow a process to shutdown before
// sending a SIGKILL.
const KillGrace = 5 * time.Second
// WaitTimeout waits for the given command to finish with a timeout.
// It assumes the command has already been started.
// If the command times out, it attempts to kill the process.
func WaitTimeout(c *exec.Cmd, timeout time.Duration) error {
var kill *time.Timer
term := time.AfterFunc(timeout, func() {
err := c.Process.Signal(syscall.SIGTERM)
if err != nil {
log.Printf("E! [agent] Error terminating process: %s", err)
return
}
kill = time.AfterFunc(KillGrace, func() {
err := c.Process.Kill()
if err != nil {
log.Printf("E! [agent] Error killing process: %s", err)
return
}
})
})
err := c.Wait()
// Shutdown all timers
if kill != nil {
kill.Stop()
}
termSent := !term.Stop()
// If the process exited without error treat it as success. This allows a
// process to do a clean shutdown on signal.
if err == nil {
return nil
}
// If SIGTERM was sent then treat any process error as a timeout.
if termSent {
return TimeoutErr
}
// Otherwise there was an error unrelated to termination.
return err
}

41
internal/exec_windows.go Normal file
View File

@ -0,0 +1,41 @@
// +build windows
package internal
import (
"log"
"os/exec"
"time"
)
// WaitTimeout waits for the given command to finish with a timeout.
// It assumes the command has already been started.
// If the command times out, it attempts to kill the process.
func WaitTimeout(c *exec.Cmd, timeout time.Duration) error {
timer := time.AfterFunc(timeout, func() {
err := c.Process.Kill()
if err != nil {
log.Printf("E! [agent] Error killing process: %s", err)
return
}
})
err := c.Wait()
// Shutdown all timers
termSent := !timer.Stop()
// If the process exited without error treat it as success. This allows a
// process to do a clean shutdown on signal.
if err == nil {
return nil
}
// If SIGTERM was sent then treat any process error as a timeout.
if termSent {
return TimeoutErr
}
// Otherwise there was an error unrelated to termination.
return err
}

View File

@ -9,7 +9,6 @@ import (
"errors" "errors"
"fmt" "fmt"
"io" "io"
"log"
"math" "math"
"math/big" "math/big"
"os" "os"
@ -200,54 +199,6 @@ func SnakeCase(in string) string {
return string(out) return string(out)
} }
// CombinedOutputTimeout runs the given command with the given timeout and
// returns the combined output of stdout and stderr.
// If the command times out, it attempts to kill the process.
func CombinedOutputTimeout(c *exec.Cmd, timeout time.Duration) ([]byte, error) {
var b bytes.Buffer
c.Stdout = &b
c.Stderr = &b
if err := c.Start(); err != nil {
return nil, err
}
err := WaitTimeout(c, timeout)
return b.Bytes(), err
}
// RunTimeout runs the given command with the given timeout.
// If the command times out, it attempts to kill the process.
func RunTimeout(c *exec.Cmd, timeout time.Duration) error {
if err := c.Start(); err != nil {
return err
}
return WaitTimeout(c, timeout)
}
// WaitTimeout waits for the given command to finish with a timeout.
// It assumes the command has already been started.
// If the command times out, it attempts to kill the process.
func WaitTimeout(c *exec.Cmd, timeout time.Duration) error {
timer := time.AfterFunc(timeout, func() {
err := c.Process.Kill()
if err != nil {
log.Printf("E! [agent] Error killing process: %s", err)
return
}
})
err := c.Wait()
if err == nil {
timer.Stop()
return nil
}
if !timer.Stop() {
return TimeoutErr
}
return err
}
// RandomSleep will sleep for a random amount of time up to max. // RandomSleep will sleep for a random amount of time up to max.
// If the shutdown channel is closed, it will return before it has finished // If the shutdown channel is closed, it will return before it has finished
// sleeping. // sleeping.