add waitpid resource consumptions

This commit is contained in:
criyle 2019-09-28 17:43:05 -07:00
parent 31edacf69e
commit 4001c7c767
12 changed files with 272 additions and 219 deletions

View File

@ -116,6 +116,8 @@ func main() {
if err == nil && rt.Status != types.StatusNormal {
err = rt.Status
}
debug("setupTime: ", rt.SetUpTime)
debug("runningTime: ", rt.RunningTime)
if err != nil {
debug(err)
c, ok := err.(types.Status)

141
daemon/container_execve.go Normal file
View File

@ -0,0 +1,141 @@
package daemon
import (
"fmt"
"syscall"
"github.com/criyle/go-sandbox/pkg/forkexec"
"github.com/criyle/go-sandbox/pkg/unixsocket"
"github.com/criyle/go-sandbox/types"
)
func (c *containerServer) handleExecve(cmd *Cmd, msg *unixsocket.Msg) error {
var (
files []uintptr
execFile uintptr
)
if msg != nil {
files = intSliceToUintptr(msg.Fds)
// don't leak fds to child
closeOnExecFds(msg.Fds)
// release files after execve
defer closeFds(msg.Fds)
}
// if fexecve, then the first fd must be executable
if cmd.FdExec {
if len(files) == 0 {
return fmt.Errorf("execve: expected fexecve fd")
}
execFile = files[0]
files = files[1:]
}
syncFunc := func(pid int) error {
msg2 := &unixsocket.Msg{
Cred: &syscall.Ucred{
Pid: int32(pid),
Uid: uint32(syscall.Getuid()),
Gid: uint32(syscall.Getgid()),
},
}
if err2 := c.sendReply(&Reply{}, msg2); err2 != nil {
return fmt.Errorf("syncFunc: sendReply(%v)", err2)
}
cmd2, _, err2 := c.recvCmd()
if err2 != nil {
return fmt.Errorf("syncFunc: recvCmd(%v)", err2)
}
if cmd2.Cmd == cmdKill {
return fmt.Errorf("syncFunc: recved kill")
}
return nil
}
r := forkexec.Runner{
Args: cmd.Argv,
Env: cmd.Envv,
ExecFile: execFile,
RLimits: cmd.RLmits,
Files: files,
WorkDir: "/w",
NoNewPrivs: true,
DropCaps: true,
SyncFunc: syncFunc,
}
// starts the runner, error is handled same as wait4 to make communication equal
pid, err := r.Start()
// done is to signal kill goroutine exits
killDone := make(chan struct{})
// waitDone is to signal kill goroutine to collect zombies
waitDone := make(chan struct{})
// recv kill
go func() {
// signal done
defer close(killDone)
// msg must be kill
c.recvCmd()
// kill all
syscall.Kill(-1, syscall.SIGKILL)
// make sure collect zombie does not consume the exit status
<-waitDone
// collect zombies
for {
if pid, err := syscall.Wait4(-1, nil, syscall.WNOHANG, nil); err != nil || pid <= 0 {
break
}
}
}()
// wait pid if no error encoutered for execve
var wstatus syscall.WaitStatus
var rusage syscall.Rusage
if err == nil {
_, err = syscall.Wait4(pid, &wstatus, 0, &rusage)
}
// sync with kill goroutine
close(waitDone)
if err != nil {
c.sendErrorReply("execve: wait4 %v", err)
} else {
var status types.Status
userTime := uint64(rusage.Utime.Sec*1e3 + rusage.Utime.Usec/1e3) // ms
userMem := uint64(rusage.Maxrss) // kb
switch {
case wstatus.Exited():
exitStatus := wstatus.ExitStatus()
c.sendReply(&Reply{
Status: status,
ExitStatus: exitStatus,
UserTime: userTime,
UserMem: userMem,
}, nil)
case wstatus.Signaled():
switch wstatus.Signal() {
// kill signal treats as TLE
case syscall.SIGXCPU, syscall.SIGKILL:
status = types.StatusTLE
case syscall.SIGXFSZ:
status = types.StatusOLE
case syscall.SIGSYS:
status = types.StatusBan
default:
status = types.StatusRE
}
c.sendReply(&Reply{
Status: status,
UserTime: userTime,
UserMem: userMem,
}, nil)
default:
c.sendErrorReply("execve: unknown status %v", wstatus)
}
}
// wait for kill msg and reply done for finish
<-killDone
return c.sendReply(&Reply{}, nil)
}

View File

@ -1,18 +1,17 @@
package daemon
import (
"bytes"
"encoding/gob"
"fmt"
"io"
"os"
"syscall"
"github.com/criyle/go-sandbox/pkg/forkexec"
"github.com/criyle/go-sandbox/pkg/unixsocket"
"github.com/criyle/go-sandbox/types"
)
type containerServer struct {
socket *unixsocket.Socket
}
// Init is called for container init process
// it will check if pid == 1, otherwise it is noop
// Init will do infinite loop on socket commands,
@ -25,6 +24,10 @@ func Init() (err error) {
}
// exit process (with whole container) upon exit this function
// possible reason:
// 1. socket broken (parent exit)
// 2. panic
// 3. undefined cmd (possible race condition)
defer func() {
if err2 := recover(); err2 != nil {
fmt.Fprintf(os.Stderr, "container_panic: %v", err)
@ -33,253 +36,129 @@ func Init() (err error) {
if err != nil {
fmt.Fprintf(os.Stderr, "container_exit: %v", err)
os.Exit(1)
} else {
fmt.Fprintf(os.Stderr, "container_exit")
os.Exit(0)
}
fmt.Fprintf(os.Stderr, "container_exit")
os.Exit(0)
}()
// new_master shared the socket at fd 3 (marked close_exec)
soc, err := unixsocket.NewSocket(3)
const defaultFd = 3
soc, err := unixsocket.NewSocket(defaultFd)
if err != nil {
return fmt.Errorf("container_init: faile to new socket(%v)", err)
}
// serve forever
cs := &containerServer{soc}
return cs.serve()
}
func (c *containerServer) serve() error {
for {
cmd, msg, err := recvCmd(soc)
cmd, msg, err := c.recvCmd()
if err != nil {
return fmt.Errorf("loop: %v", err)
return fmt.Errorf("serve: %v", err)
}
if err := handleCmd(soc, cmd, msg); err != nil {
return fmt.Errorf("loop: failed to execute cmd(%v)", err)
if err := c.handleCmd(cmd, msg); err != nil {
return fmt.Errorf("serve: failed to execute cmd %v", err)
}
}
}
func handleCmd(s *unixsocket.Socket, cmd *Cmd, msg *unixsocket.Msg) error {
func (c *containerServer) handleCmd(cmd *Cmd, msg *unixsocket.Msg) error {
switch cmd.Cmd {
case cmdPing:
return handlePing(s)
return c.handlePing()
case cmdCopyIn:
return handleCopyIn(s, cmd, msg)
return c.handleCopyIn(cmd, msg)
case cmdOpen:
return handleOpen(s, cmd)
return c.handleOpen(cmd)
case cmdDelete:
return handleDelete(s, cmd)
return c.handleDelete(cmd)
case cmdReset:
return handleReset(s)
return c.handleReset()
case cmdExecve:
return handleExecve(s, cmd, msg)
return c.handleExecve(cmd, msg)
}
return fmt.Errorf("Unknown command: %v", cmd.Cmd)
return fmt.Errorf("unknown command: %v", cmd.Cmd)
}
func handlePing(s *unixsocket.Socket) error {
return sendReply(s, &Reply{}, nil)
func (c *containerServer) handlePing() error {
return c.sendReply(&Reply{}, nil)
}
func handleCopyIn(s *unixsocket.Socket, cmd *Cmd, msg *unixsocket.Msg) error {
func (c *containerServer) handleCopyIn(cmd *Cmd, msg *unixsocket.Msg) error {
if len(msg.Fds) != 1 {
closeFds(msg.Fds)
return sendErrorReply(s, "copyin: unexpected number of fds(%d)", len(msg.Fds))
return c.sendErrorReply("copyin: unexpected number of fds(%d)", len(msg.Fds))
}
inf := os.NewFile(uintptr(msg.Fds[0]), cmd.Path)
if inf == nil {
return sendErrorReply(s, "copyin: newfile failed %v", msg.Fds[0])
return c.sendErrorReply("copyin: newfile failed %v", msg.Fds[0])
}
defer inf.Close()
// have 0777 permission to be able copy in executables
outf, err := os.OpenFile(cmd.Path, os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0777)
if err != nil {
return sendErrorReply(s, "copyin: open write file %v", err)
return c.sendErrorReply("copyin: open write file %v", err)
}
defer outf.Close()
_, err = io.Copy(outf, inf)
if err != nil {
return sendErrorReply(s, "copyin: io.copy %v", err)
if _, err = io.Copy(outf, inf); err != nil {
return c.sendErrorReply("copyin: io.copy %v", err)
}
return sendReply(s, &Reply{}, nil)
return c.sendReply(&Reply{}, nil)
}
func handleOpen(s *unixsocket.Socket, cmd *Cmd) error {
func (c *containerServer) handleOpen(cmd *Cmd) error {
outf, err := os.Open(cmd.Path)
if err != nil {
return sendErrorReply(s, "open: %v", err)
return c.sendErrorReply("open: %v", err)
}
defer outf.Close()
return sendReply(s, &Reply{}, &unixsocket.Msg{
return c.sendReply(&Reply{}, &unixsocket.Msg{
Fds: []int{int(outf.Fd())},
})
}
func handleDelete(s *unixsocket.Socket, cmd *Cmd) error {
func (c *containerServer) handleDelete(cmd *Cmd) error {
if err := os.Remove(cmd.Path); err != nil {
return sendErrorReply(s, "delete: %v", err)
return c.sendErrorReply("delete: %v", err)
}
return sendReply(s, &Reply{}, nil)
return c.sendReply(&Reply{}, nil)
}
func handleReset(s *unixsocket.Socket) error {
func (c *containerServer) handleReset() error {
if err := removeContents("/tmp"); err != nil {
return sendErrorReply(s, "reset: /tmp %v", err)
return c.sendErrorReply("reset: /tmp %v", err)
}
if err := removeContents("/w"); err != nil {
return sendErrorReply(s, "reset: /w %v", err)
return c.sendErrorReply("reset: /w %v", err)
}
return sendReply(s, &Reply{}, nil)
return c.sendReply(&Reply{}, nil)
}
func handleExecve(s *unixsocket.Socket, cmd *Cmd, msg *unixsocket.Msg) error {
var (
files []uintptr
execFile uintptr
)
if msg != nil {
files = intSliceToUintptr(msg.Fds)
// don't leak fds to child
closeOnExecFds(msg.Fds)
// release files after execve
defer closeFds(msg.Fds)
}
// if fexecve, then the first fd must be executable
if cmd.FdExec {
if len(files) == 0 {
return fmt.Errorf("execve: expected fexecve fd")
}
execFile = files[0]
files = files[1:]
}
syncFunc := func(pid int) error {
msg2 := unixsocket.Msg{
Cred: &syscall.Ucred{
Pid: int32(pid),
Uid: uint32(syscall.Getuid()),
Gid: uint32(syscall.Getgid()),
},
}
if err2 := sendReply(s, &Reply{}, &msg2); err2 != nil {
return fmt.Errorf("syncFunc: sendReply(%v)", err2)
}
cmd2, _, err2 := recvCmd(s)
if err2 != nil {
return fmt.Errorf("syncFunc: recvCmd(%v)", err2)
}
if cmd2.Cmd == cmdKill {
return fmt.Errorf("syncFunc: recved kill")
}
return nil
}
r := forkexec.Runner{
Args: cmd.Argv,
Env: cmd.Envv,
ExecFile: execFile,
RLimits: cmd.RLmits,
Files: files,
WorkDir: "/w",
NoNewPrivs: true,
DropCaps: true,
SyncFunc: syncFunc,
}
// starts the runner, error is handled same as wait4 to make communication equal
pid, err := r.Start()
// done is to signal kill goroutine exits
killDone := make(chan struct{})
// waitDone is to signal kill goroutine to collect zombies
waitDone := make(chan struct{})
// recv kill
go func() {
// signal done
defer close(killDone)
// msg must be kill
recvCmd(s)
// kill all
syscall.Kill(-1, syscall.SIGKILL)
// make sure collect zombie does not consume the exit status
<-waitDone
// collect zombies
for {
if pid, err := syscall.Wait4(-1, nil, syscall.WNOHANG, nil); err != nil || pid <= 0 {
break
}
}
}()
// wait pid if no error encoutered for execve
var wstatus syscall.WaitStatus
if err == nil {
_, err = syscall.Wait4(pid, &wstatus, 0, nil)
}
// sync with kill goroutine
close(waitDone)
func (c *containerServer) recvCmd() (*Cmd, *unixsocket.Msg, error) {
cmd := new(Cmd)
msg, err := (*socket)(c.socket).RecvMsg(cmd)
if err != nil {
sendErrorReply(s, "execve: wait4 %v", err)
} else {
switch {
case wstatus.Exited():
sendReply(s, &Reply{ExitStatus: wstatus.ExitStatus()}, nil)
case wstatus.Signaled():
var status types.Status
switch wstatus.Signal() {
// kill signal treats as TLE
case syscall.SIGXCPU, syscall.SIGKILL:
status = types.StatusTLE
case syscall.SIGXFSZ:
status = types.StatusOLE
case syscall.SIGSYS:
status = types.StatusBan
default:
status = types.StatusRE
}
sendReply(s, &Reply{Status: status}, nil)
default:
sendErrorReply(s, "execve: unknown status %v", wstatus)
}
return nil, nil, err
}
// wait for kill msg and reply done for finish
<-killDone
return sendReply(s, &Reply{}, nil)
return cmd, msg, nil
}
func recvCmd(s *unixsocket.Socket) (*Cmd, *unixsocket.Msg, error) {
var cmd Cmd
buffer := GetBuffer()
defer PutBuffer(buffer)
n, msg, err := s.RecvMsg(buffer)
if err != nil {
return nil, nil, fmt.Errorf("failed RecvMsg(%v)", err)
}
if err := gob.NewDecoder(bytes.NewReader(buffer[:n])).Decode(&cmd); err != nil {
return nil, nil, fmt.Errorf("failed to decode(%v)", err)
}
return &cmd, msg, nil
}
func sendReply(s *unixsocket.Socket, reply *Reply, msg *unixsocket.Msg) error {
var buffer bytes.Buffer
if err := gob.NewEncoder(&buffer).Encode(reply); err != nil {
return err
}
if err := s.SendMsg(buffer.Bytes(), msg); err != nil {
return err
}
return nil
func (c *containerServer) sendReply(reply *Reply, msg *unixsocket.Msg) error {
return (*socket)(c.socket).SendMsg(reply, msg)
}
// sendErrorReply sends error reply
func sendErrorReply(s *unixsocket.Socket, ft string, v ...interface{}) error {
reply := Reply{Error: fmt.Sprintf(ft, v...)}
return sendReply(s, &reply, nil)
func (c *containerServer) sendErrorReply(ft string, v ...interface{}) error {
return c.sendReply(&Reply{Error: fmt.Sprintf(ft, v...)}, nil)
}

View File

@ -53,4 +53,6 @@ type Reply struct {
Error string // empty if no error
ExitStatus int // waitpid exit status
Status types.Status // return status
UserTime uint64 // waitpid user CPU (ms)
UserMem uint64 // waitpid user memory (kb)
}

View File

@ -35,7 +35,7 @@ type Master struct {
buff *pipe.Buffer
}
// New creates new master with underlying container
// Build creates new master with underlying container
func (b *Builder) Build() (*Master, error) {
var (
err error

View File

@ -1,8 +1,6 @@
package daemon
import (
"bytes"
"encoding/gob"
"fmt"
"os"
@ -102,31 +100,14 @@ func (m *Master) recvAckReply(name string) error {
}
func (m *Master) recvReply() (*Reply, *unixsocket.Msg, error) {
buff := GetBuffer()
defer PutBuffer(buff)
n, msg, err := m.socket.RecvMsg(buff)
if err != nil {
return nil, nil, fmt.Errorf("failed to RecvMsg(%v)", err)
}
reply := new(Reply)
if err := gob.NewDecoder(bytes.NewReader(buff[:n])).Decode(reply); err != nil {
return nil, nil, fmt.Errorf("failed to decode(%v)", err)
msg, err := (*socket)(m.socket).RecvMsg(reply)
if err != nil {
return nil, nil, err
}
return reply, msg, nil
}
func (m *Master) sendCmd(cmd *Cmd, msg *unixsocket.Msg) error {
buf := GetBuffer()
defer PutBuffer(buf)
// use buf pool to reduce allocation
buff := bytes.NewBuffer(buf[:0])
if err := gob.NewEncoder(buff).Encode(cmd); err != nil {
return fmt.Errorf("failed to encode(%v)", err)
}
if err := m.socket.SendMsg(buff.Bytes(), msg); err != nil {
return fmt.Errorf("failed to SendMsg(%v)", err)
}
return nil
return (*socket)(m.socket).SendMsg(cmd, msg)
}

View File

@ -2,6 +2,7 @@ package daemon
import (
"fmt"
"time"
"github.com/criyle/go-sandbox/pkg/rlimit"
"github.com/criyle/go-sandbox/pkg/unixsocket"
@ -25,6 +26,8 @@ type ExecveParam struct {
// accepts done for cancelation
func (m *Master) Execve(done <-chan struct{}, param *ExecveParam) (<-chan types.Result, error) {
var files []int
sTime := time.Now()
if param.ExecFile > 0 {
files = append(files, int(param.ExecFile))
}
@ -60,6 +63,7 @@ func (m *Master) Execve(done <-chan struct{}, param *ExecveParam) (<-chan types.
if err := m.sendCmd(&Cmd{Cmd: cmdOk}, nil); err != nil {
return nil, fmt.Errorf("execve: ok failed(%v)", err)
}
mTime := time.Now()
// make sure goroutine not leaked (blocked) even if result is not consumed
result := make(chan types.Result, 1)
waitDone := make(chan struct{})
@ -75,9 +79,13 @@ func (m *Master) Execve(done <-chan struct{}, param *ExecveParam) (<-chan types.
status = types.StatusFatal
}
result <- types.Result{
ExitStatus: reply2.ExitStatus,
Status: status,
Error: reply2.Error,
Status: status,
ExitStatus: reply2.ExitStatus,
UserTime: reply2.UserTime,
UserMem: reply2.UserMem,
Error: reply2.Error,
SetUpTime: mTime.Sub(sTime),
RunningTime: time.Since(mTime),
}
}()
// Kill (if wait is done, a kill message need to be send to collect zombies)

44
daemon/socket.go Normal file
View File

@ -0,0 +1,44 @@
package daemon
import (
"bytes"
"encoding/gob"
"fmt"
"github.com/criyle/go-sandbox/pkg/unixsocket"
)
type socket unixsocket.Socket
func (s *socket) RecvMsg(e interface{}) (*unixsocket.Msg, error) {
soc := (*unixsocket.Socket)(s)
buff := GetBuffer()
defer PutBuffer(buff)
n, msg, err := soc.RecvMsg(buff)
if err != nil {
return nil, fmt.Errorf("RecvMsg: %v", err)
}
if err := gob.NewDecoder(bytes.NewReader(buff[:n])).Decode(e); err != nil {
return nil, fmt.Errorf("RecvMsg: failed to decode %v", err)
}
return msg, nil
}
func (s *socket) SendMsg(e interface{}, msg *unixsocket.Msg) error {
soc := (*unixsocket.Socket)(s)
buf := GetBuffer()
defer PutBuffer(buf)
// use buf pool to reduce allocation
buff := bytes.NewBuffer(buf[:0])
if err := gob.NewEncoder(buff).Encode(e); err != nil {
return fmt.Errorf("SendMsg: failed to encode %v", err)
}
if err := soc.SendMsg(buff.Bytes(), msg); err != nil {
return fmt.Errorf("SendMsg: failed to SendMsg %v", err)
}
return nil
}

2
go.mod
View File

@ -4,5 +4,5 @@ go 1.12
require (
github.com/seccomp/libseccomp-golang v0.9.1
golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe
golang.org/x/sys v0.0.0-20190927073244-c990c680b611
)

4
go.sum
View File

@ -1,4 +1,4 @@
github.com/seccomp/libseccomp-golang v0.9.1 h1:NJjM5DNFOs0s3kYE1WUOr6G8V97sdt46rlXTMfXGWBo=
github.com/seccomp/libseccomp-golang v0.9.1/go.mod h1:GbW5+tmTXfcxTToHLXlScSlAvWlF4P2Ca7zGrPiEpWo=
golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe h1:6fAMxZRR6sl1Uq8U61gxU+kPTs2tR8uOySCbBP7BN/M=
golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190927073244-c990c680b611 h1:q9u40nxWT5zRClI/uU9dHCiYGottAg6Nzz4YUQyHxdA=
golang.org/x/sys v0.0.0-20190927073244-c990c680b611/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=

View File

@ -472,10 +472,10 @@ childerror:
func handleChildFailed(pid uintptr) {
var wstatus syscall.WaitStatus
// make sure not blocked
unix.Kill(int(pid), syscall.SIGKILL)
syscall.Kill(int(pid), syscall.SIGKILL)
// child failed; wait for it to exit, to make sure the zombies don't accumulate
_, err3 := syscall.Wait4(int(pid), &wstatus, 0, nil)
for err3 == syscall.EINTR {
_, err3 = syscall.Wait4(int(pid), &wstatus, 0, nil)
_, err := syscall.Wait4(int(pid), &wstatus, 0, nil)
for err == syscall.EINTR {
_, err = syscall.Wait4(int(pid), &wstatus, 0, nil)
}
}

View File

@ -9,11 +9,7 @@ type Result struct {
Error string // potential detailed error message
UserTime uint64 // used user CPU time (in ms)
UserMem uint64 // used user memory (in kb)
Stat // collects time usage for the runner
}
// Stat is the time usages in ns
type Stat struct {
// collects time usage for the runner
SetUpTime time.Duration
RunningTime time.Duration
}