From 4001c7c7671fad066d974dcbe960bccc7e1639ea Mon Sep 17 00:00:00 2001 From: criyle Date: Sat, 28 Sep 2019 17:43:05 -0700 Subject: [PATCH] add waitpid resource consumptions --- cmd/runprog/main.go | 2 + daemon/container_execve.go | 141 +++++++++++++++++++++ daemon/container_init.go | 239 +++++++++--------------------------- daemon/deamon.go | 2 + daemon/master.go | 2 +- daemon/master_cmd.go | 27 +--- daemon/master_cmd_execve.go | 14 ++- daemon/socket.go | 44 +++++++ go.mod | 2 +- go.sum | 4 +- pkg/forkexec/fork.go | 8 +- types/types.go | 6 +- 12 files changed, 272 insertions(+), 219 deletions(-) create mode 100644 daemon/container_execve.go create mode 100644 daemon/socket.go diff --git a/cmd/runprog/main.go b/cmd/runprog/main.go index 3c8c795..64b094a 100644 --- a/cmd/runprog/main.go +++ b/cmd/runprog/main.go @@ -116,6 +116,8 @@ func main() { if err == nil && rt.Status != types.StatusNormal { err = rt.Status } + debug("setupTime: ", rt.SetUpTime) + debug("runningTime: ", rt.RunningTime) if err != nil { debug(err) c, ok := err.(types.Status) diff --git a/daemon/container_execve.go b/daemon/container_execve.go new file mode 100644 index 0000000..4ecc11e --- /dev/null +++ b/daemon/container_execve.go @@ -0,0 +1,141 @@ +package daemon + +import ( + "fmt" + "syscall" + + "github.com/criyle/go-sandbox/pkg/forkexec" + "github.com/criyle/go-sandbox/pkg/unixsocket" + "github.com/criyle/go-sandbox/types" +) + +func (c *containerServer) handleExecve(cmd *Cmd, msg *unixsocket.Msg) error { + var ( + files []uintptr + execFile uintptr + ) + if msg != nil { + files = intSliceToUintptr(msg.Fds) + // don't leak fds to child + closeOnExecFds(msg.Fds) + // release files after execve + defer closeFds(msg.Fds) + } + + // if fexecve, then the first fd must be executable + if cmd.FdExec { + if len(files) == 0 { + return fmt.Errorf("execve: expected fexecve fd") + } + execFile = files[0] + files = files[1:] + } + + syncFunc := func(pid int) error { + msg2 := &unixsocket.Msg{ + Cred: &syscall.Ucred{ + Pid: int32(pid), + Uid: uint32(syscall.Getuid()), + Gid: uint32(syscall.Getgid()), + }, + } + if err2 := c.sendReply(&Reply{}, msg2); err2 != nil { + return fmt.Errorf("syncFunc: sendReply(%v)", err2) + } + cmd2, _, err2 := c.recvCmd() + if err2 != nil { + return fmt.Errorf("syncFunc: recvCmd(%v)", err2) + } + if cmd2.Cmd == cmdKill { + return fmt.Errorf("syncFunc: recved kill") + } + return nil + } + + r := forkexec.Runner{ + Args: cmd.Argv, + Env: cmd.Envv, + ExecFile: execFile, + RLimits: cmd.RLmits, + Files: files, + WorkDir: "/w", + NoNewPrivs: true, + DropCaps: true, + SyncFunc: syncFunc, + } + // starts the runner, error is handled same as wait4 to make communication equal + pid, err := r.Start() + + // done is to signal kill goroutine exits + killDone := make(chan struct{}) + // waitDone is to signal kill goroutine to collect zombies + waitDone := make(chan struct{}) + + // recv kill + go func() { + // signal done + defer close(killDone) + // msg must be kill + c.recvCmd() + // kill all + syscall.Kill(-1, syscall.SIGKILL) + // make sure collect zombie does not consume the exit status + <-waitDone + // collect zombies + for { + if pid, err := syscall.Wait4(-1, nil, syscall.WNOHANG, nil); err != nil || pid <= 0 { + break + } + } + }() + + // wait pid if no error encoutered for execve + var wstatus syscall.WaitStatus + var rusage syscall.Rusage + if err == nil { + _, err = syscall.Wait4(pid, &wstatus, 0, &rusage) + } + // sync with kill goroutine + close(waitDone) + + if err != nil { + c.sendErrorReply("execve: wait4 %v", err) + } else { + var status types.Status + userTime := uint64(rusage.Utime.Sec*1e3 + rusage.Utime.Usec/1e3) // ms + userMem := uint64(rusage.Maxrss) // kb + switch { + case wstatus.Exited(): + exitStatus := wstatus.ExitStatus() + c.sendReply(&Reply{ + Status: status, + ExitStatus: exitStatus, + UserTime: userTime, + UserMem: userMem, + }, nil) + + case wstatus.Signaled(): + switch wstatus.Signal() { + // kill signal treats as TLE + case syscall.SIGXCPU, syscall.SIGKILL: + status = types.StatusTLE + case syscall.SIGXFSZ: + status = types.StatusOLE + case syscall.SIGSYS: + status = types.StatusBan + default: + status = types.StatusRE + } + c.sendReply(&Reply{ + Status: status, + UserTime: userTime, + UserMem: userMem, + }, nil) + default: + c.sendErrorReply("execve: unknown status %v", wstatus) + } + } + // wait for kill msg and reply done for finish + <-killDone + return c.sendReply(&Reply{}, nil) +} diff --git a/daemon/container_init.go b/daemon/container_init.go index 6a3fa3f..f4a88a5 100644 --- a/daemon/container_init.go +++ b/daemon/container_init.go @@ -1,18 +1,17 @@ package daemon import ( - "bytes" - "encoding/gob" "fmt" "io" "os" - "syscall" - "github.com/criyle/go-sandbox/pkg/forkexec" "github.com/criyle/go-sandbox/pkg/unixsocket" - "github.com/criyle/go-sandbox/types" ) +type containerServer struct { + socket *unixsocket.Socket +} + // Init is called for container init process // it will check if pid == 1, otherwise it is noop // Init will do infinite loop on socket commands, @@ -25,6 +24,10 @@ func Init() (err error) { } // exit process (with whole container) upon exit this function + // possible reason: + // 1. socket broken (parent exit) + // 2. panic + // 3. undefined cmd (possible race condition) defer func() { if err2 := recover(); err2 != nil { fmt.Fprintf(os.Stderr, "container_panic: %v", err) @@ -33,253 +36,129 @@ func Init() (err error) { if err != nil { fmt.Fprintf(os.Stderr, "container_exit: %v", err) os.Exit(1) - } else { - fmt.Fprintf(os.Stderr, "container_exit") - os.Exit(0) } + fmt.Fprintf(os.Stderr, "container_exit") + os.Exit(0) }() // new_master shared the socket at fd 3 (marked close_exec) - soc, err := unixsocket.NewSocket(3) + const defaultFd = 3 + soc, err := unixsocket.NewSocket(defaultFd) if err != nil { return fmt.Errorf("container_init: faile to new socket(%v)", err) } + + // serve forever + cs := &containerServer{soc} + return cs.serve() +} + +func (c *containerServer) serve() error { for { - cmd, msg, err := recvCmd(soc) + cmd, msg, err := c.recvCmd() if err != nil { - return fmt.Errorf("loop: %v", err) + return fmt.Errorf("serve: %v", err) } - if err := handleCmd(soc, cmd, msg); err != nil { - return fmt.Errorf("loop: failed to execute cmd(%v)", err) + if err := c.handleCmd(cmd, msg); err != nil { + return fmt.Errorf("serve: failed to execute cmd %v", err) } } } -func handleCmd(s *unixsocket.Socket, cmd *Cmd, msg *unixsocket.Msg) error { +func (c *containerServer) handleCmd(cmd *Cmd, msg *unixsocket.Msg) error { switch cmd.Cmd { case cmdPing: - return handlePing(s) + return c.handlePing() case cmdCopyIn: - return handleCopyIn(s, cmd, msg) + return c.handleCopyIn(cmd, msg) case cmdOpen: - return handleOpen(s, cmd) + return c.handleOpen(cmd) case cmdDelete: - return handleDelete(s, cmd) + return c.handleDelete(cmd) case cmdReset: - return handleReset(s) + return c.handleReset() case cmdExecve: - return handleExecve(s, cmd, msg) + return c.handleExecve(cmd, msg) } - return fmt.Errorf("Unknown command: %v", cmd.Cmd) + return fmt.Errorf("unknown command: %v", cmd.Cmd) } -func handlePing(s *unixsocket.Socket) error { - return sendReply(s, &Reply{}, nil) +func (c *containerServer) handlePing() error { + return c.sendReply(&Reply{}, nil) } -func handleCopyIn(s *unixsocket.Socket, cmd *Cmd, msg *unixsocket.Msg) error { +func (c *containerServer) handleCopyIn(cmd *Cmd, msg *unixsocket.Msg) error { if len(msg.Fds) != 1 { closeFds(msg.Fds) - return sendErrorReply(s, "copyin: unexpected number of fds(%d)", len(msg.Fds)) + return c.sendErrorReply("copyin: unexpected number of fds(%d)", len(msg.Fds)) } inf := os.NewFile(uintptr(msg.Fds[0]), cmd.Path) if inf == nil { - return sendErrorReply(s, "copyin: newfile failed %v", msg.Fds[0]) + return c.sendErrorReply("copyin: newfile failed %v", msg.Fds[0]) } defer inf.Close() // have 0777 permission to be able copy in executables outf, err := os.OpenFile(cmd.Path, os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0777) if err != nil { - return sendErrorReply(s, "copyin: open write file %v", err) + return c.sendErrorReply("copyin: open write file %v", err) } defer outf.Close() - _, err = io.Copy(outf, inf) - if err != nil { - return sendErrorReply(s, "copyin: io.copy %v", err) + if _, err = io.Copy(outf, inf); err != nil { + return c.sendErrorReply("copyin: io.copy %v", err) } - return sendReply(s, &Reply{}, nil) + return c.sendReply(&Reply{}, nil) } -func handleOpen(s *unixsocket.Socket, cmd *Cmd) error { +func (c *containerServer) handleOpen(cmd *Cmd) error { outf, err := os.Open(cmd.Path) if err != nil { - return sendErrorReply(s, "open: %v", err) + return c.sendErrorReply("open: %v", err) } defer outf.Close() - return sendReply(s, &Reply{}, &unixsocket.Msg{ + return c.sendReply(&Reply{}, &unixsocket.Msg{ Fds: []int{int(outf.Fd())}, }) } -func handleDelete(s *unixsocket.Socket, cmd *Cmd) error { +func (c *containerServer) handleDelete(cmd *Cmd) error { if err := os.Remove(cmd.Path); err != nil { - return sendErrorReply(s, "delete: %v", err) + return c.sendErrorReply("delete: %v", err) } - return sendReply(s, &Reply{}, nil) + return c.sendReply(&Reply{}, nil) } -func handleReset(s *unixsocket.Socket) error { +func (c *containerServer) handleReset() error { if err := removeContents("/tmp"); err != nil { - return sendErrorReply(s, "reset: /tmp %v", err) + return c.sendErrorReply("reset: /tmp %v", err) } if err := removeContents("/w"); err != nil { - return sendErrorReply(s, "reset: /w %v", err) + return c.sendErrorReply("reset: /w %v", err) } - return sendReply(s, &Reply{}, nil) + return c.sendReply(&Reply{}, nil) } -func handleExecve(s *unixsocket.Socket, cmd *Cmd, msg *unixsocket.Msg) error { - var ( - files []uintptr - execFile uintptr - ) - if msg != nil { - files = intSliceToUintptr(msg.Fds) - // don't leak fds to child - closeOnExecFds(msg.Fds) - // release files after execve - defer closeFds(msg.Fds) - } - - // if fexecve, then the first fd must be executable - if cmd.FdExec { - if len(files) == 0 { - return fmt.Errorf("execve: expected fexecve fd") - } - execFile = files[0] - files = files[1:] - } - - syncFunc := func(pid int) error { - msg2 := unixsocket.Msg{ - Cred: &syscall.Ucred{ - Pid: int32(pid), - Uid: uint32(syscall.Getuid()), - Gid: uint32(syscall.Getgid()), - }, - } - if err2 := sendReply(s, &Reply{}, &msg2); err2 != nil { - return fmt.Errorf("syncFunc: sendReply(%v)", err2) - } - cmd2, _, err2 := recvCmd(s) - if err2 != nil { - return fmt.Errorf("syncFunc: recvCmd(%v)", err2) - } - if cmd2.Cmd == cmdKill { - return fmt.Errorf("syncFunc: recved kill") - } - return nil - } - r := forkexec.Runner{ - Args: cmd.Argv, - Env: cmd.Envv, - ExecFile: execFile, - RLimits: cmd.RLmits, - Files: files, - WorkDir: "/w", - NoNewPrivs: true, - DropCaps: true, - SyncFunc: syncFunc, - } - // starts the runner, error is handled same as wait4 to make communication equal - pid, err := r.Start() - - // done is to signal kill goroutine exits - killDone := make(chan struct{}) - // waitDone is to signal kill goroutine to collect zombies - waitDone := make(chan struct{}) - - // recv kill - go func() { - // signal done - defer close(killDone) - // msg must be kill - recvCmd(s) - // kill all - syscall.Kill(-1, syscall.SIGKILL) - // make sure collect zombie does not consume the exit status - <-waitDone - // collect zombies - for { - if pid, err := syscall.Wait4(-1, nil, syscall.WNOHANG, nil); err != nil || pid <= 0 { - break - } - } - }() - - // wait pid if no error encoutered for execve - var wstatus syscall.WaitStatus - if err == nil { - _, err = syscall.Wait4(pid, &wstatus, 0, nil) - } - // sync with kill goroutine - close(waitDone) - +func (c *containerServer) recvCmd() (*Cmd, *unixsocket.Msg, error) { + cmd := new(Cmd) + msg, err := (*socket)(c.socket).RecvMsg(cmd) if err != nil { - sendErrorReply(s, "execve: wait4 %v", err) - } else { - switch { - case wstatus.Exited(): - sendReply(s, &Reply{ExitStatus: wstatus.ExitStatus()}, nil) - - case wstatus.Signaled(): - var status types.Status - switch wstatus.Signal() { - // kill signal treats as TLE - case syscall.SIGXCPU, syscall.SIGKILL: - status = types.StatusTLE - case syscall.SIGXFSZ: - status = types.StatusOLE - case syscall.SIGSYS: - status = types.StatusBan - default: - status = types.StatusRE - } - sendReply(s, &Reply{Status: status}, nil) - default: - sendErrorReply(s, "execve: unknown status %v", wstatus) - } + return nil, nil, err } - // wait for kill msg and reply done for finish - <-killDone - return sendReply(s, &Reply{}, nil) + return cmd, msg, nil } -func recvCmd(s *unixsocket.Socket) (*Cmd, *unixsocket.Msg, error) { - var cmd Cmd - buffer := GetBuffer() - defer PutBuffer(buffer) - n, msg, err := s.RecvMsg(buffer) - if err != nil { - return nil, nil, fmt.Errorf("failed RecvMsg(%v)", err) - } - if err := gob.NewDecoder(bytes.NewReader(buffer[:n])).Decode(&cmd); err != nil { - return nil, nil, fmt.Errorf("failed to decode(%v)", err) - } - return &cmd, msg, nil -} - -func sendReply(s *unixsocket.Socket, reply *Reply, msg *unixsocket.Msg) error { - var buffer bytes.Buffer - if err := gob.NewEncoder(&buffer).Encode(reply); err != nil { - return err - } - if err := s.SendMsg(buffer.Bytes(), msg); err != nil { - return err - } - return nil +func (c *containerServer) sendReply(reply *Reply, msg *unixsocket.Msg) error { + return (*socket)(c.socket).SendMsg(reply, msg) } // sendErrorReply sends error reply -func sendErrorReply(s *unixsocket.Socket, ft string, v ...interface{}) error { - reply := Reply{Error: fmt.Sprintf(ft, v...)} - return sendReply(s, &reply, nil) +func (c *containerServer) sendErrorReply(ft string, v ...interface{}) error { + return c.sendReply(&Reply{Error: fmt.Sprintf(ft, v...)}, nil) } diff --git a/daemon/deamon.go b/daemon/deamon.go index 0b64c16..4087f66 100644 --- a/daemon/deamon.go +++ b/daemon/deamon.go @@ -53,4 +53,6 @@ type Reply struct { Error string // empty if no error ExitStatus int // waitpid exit status Status types.Status // return status + UserTime uint64 // waitpid user CPU (ms) + UserMem uint64 // waitpid user memory (kb) } diff --git a/daemon/master.go b/daemon/master.go index f3506a3..4594438 100644 --- a/daemon/master.go +++ b/daemon/master.go @@ -35,7 +35,7 @@ type Master struct { buff *pipe.Buffer } -// New creates new master with underlying container +// Build creates new master with underlying container func (b *Builder) Build() (*Master, error) { var ( err error diff --git a/daemon/master_cmd.go b/daemon/master_cmd.go index 6d56dc2..c717580 100644 --- a/daemon/master_cmd.go +++ b/daemon/master_cmd.go @@ -1,8 +1,6 @@ package daemon import ( - "bytes" - "encoding/gob" "fmt" "os" @@ -102,31 +100,14 @@ func (m *Master) recvAckReply(name string) error { } func (m *Master) recvReply() (*Reply, *unixsocket.Msg, error) { - buff := GetBuffer() - defer PutBuffer(buff) - - n, msg, err := m.socket.RecvMsg(buff) - if err != nil { - return nil, nil, fmt.Errorf("failed to RecvMsg(%v)", err) - } - reply := new(Reply) - if err := gob.NewDecoder(bytes.NewReader(buff[:n])).Decode(reply); err != nil { - return nil, nil, fmt.Errorf("failed to decode(%v)", err) + msg, err := (*socket)(m.socket).RecvMsg(reply) + if err != nil { + return nil, nil, err } return reply, msg, nil } func (m *Master) sendCmd(cmd *Cmd, msg *unixsocket.Msg) error { - buf := GetBuffer() - defer PutBuffer(buf) - // use buf pool to reduce allocation - buff := bytes.NewBuffer(buf[:0]) - if err := gob.NewEncoder(buff).Encode(cmd); err != nil { - return fmt.Errorf("failed to encode(%v)", err) - } - if err := m.socket.SendMsg(buff.Bytes(), msg); err != nil { - return fmt.Errorf("failed to SendMsg(%v)", err) - } - return nil + return (*socket)(m.socket).SendMsg(cmd, msg) } diff --git a/daemon/master_cmd_execve.go b/daemon/master_cmd_execve.go index 82c3da5..1f98060 100644 --- a/daemon/master_cmd_execve.go +++ b/daemon/master_cmd_execve.go @@ -2,6 +2,7 @@ package daemon import ( "fmt" + "time" "github.com/criyle/go-sandbox/pkg/rlimit" "github.com/criyle/go-sandbox/pkg/unixsocket" @@ -25,6 +26,8 @@ type ExecveParam struct { // accepts done for cancelation func (m *Master) Execve(done <-chan struct{}, param *ExecveParam) (<-chan types.Result, error) { var files []int + sTime := time.Now() + if param.ExecFile > 0 { files = append(files, int(param.ExecFile)) } @@ -60,6 +63,7 @@ func (m *Master) Execve(done <-chan struct{}, param *ExecveParam) (<-chan types. if err := m.sendCmd(&Cmd{Cmd: cmdOk}, nil); err != nil { return nil, fmt.Errorf("execve: ok failed(%v)", err) } + mTime := time.Now() // make sure goroutine not leaked (blocked) even if result is not consumed result := make(chan types.Result, 1) waitDone := make(chan struct{}) @@ -75,9 +79,13 @@ func (m *Master) Execve(done <-chan struct{}, param *ExecveParam) (<-chan types. status = types.StatusFatal } result <- types.Result{ - ExitStatus: reply2.ExitStatus, - Status: status, - Error: reply2.Error, + Status: status, + ExitStatus: reply2.ExitStatus, + UserTime: reply2.UserTime, + UserMem: reply2.UserMem, + Error: reply2.Error, + SetUpTime: mTime.Sub(sTime), + RunningTime: time.Since(mTime), } }() // Kill (if wait is done, a kill message need to be send to collect zombies) diff --git a/daemon/socket.go b/daemon/socket.go new file mode 100644 index 0000000..7b86740 --- /dev/null +++ b/daemon/socket.go @@ -0,0 +1,44 @@ +package daemon + +import ( + "bytes" + "encoding/gob" + "fmt" + + "github.com/criyle/go-sandbox/pkg/unixsocket" +) + +type socket unixsocket.Socket + +func (s *socket) RecvMsg(e interface{}) (*unixsocket.Msg, error) { + soc := (*unixsocket.Socket)(s) + buff := GetBuffer() + defer PutBuffer(buff) + + n, msg, err := soc.RecvMsg(buff) + if err != nil { + return nil, fmt.Errorf("RecvMsg: %v", err) + } + + if err := gob.NewDecoder(bytes.NewReader(buff[:n])).Decode(e); err != nil { + return nil, fmt.Errorf("RecvMsg: failed to decode %v", err) + } + return msg, nil +} + +func (s *socket) SendMsg(e interface{}, msg *unixsocket.Msg) error { + soc := (*unixsocket.Socket)(s) + buf := GetBuffer() + defer PutBuffer(buf) + + // use buf pool to reduce allocation + buff := bytes.NewBuffer(buf[:0]) + if err := gob.NewEncoder(buff).Encode(e); err != nil { + return fmt.Errorf("SendMsg: failed to encode %v", err) + } + + if err := soc.SendMsg(buff.Bytes(), msg); err != nil { + return fmt.Errorf("SendMsg: failed to SendMsg %v", err) + } + return nil +} diff --git a/go.mod b/go.mod index 3716cb4..8d51618 100644 --- a/go.mod +++ b/go.mod @@ -4,5 +4,5 @@ go 1.12 require ( github.com/seccomp/libseccomp-golang v0.9.1 - golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe + golang.org/x/sys v0.0.0-20190927073244-c990c680b611 ) diff --git a/go.sum b/go.sum index 641720b..5ac3b84 100644 --- a/go.sum +++ b/go.sum @@ -1,4 +1,4 @@ github.com/seccomp/libseccomp-golang v0.9.1 h1:NJjM5DNFOs0s3kYE1WUOr6G8V97sdt46rlXTMfXGWBo= github.com/seccomp/libseccomp-golang v0.9.1/go.mod h1:GbW5+tmTXfcxTToHLXlScSlAvWlF4P2Ca7zGrPiEpWo= -golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe h1:6fAMxZRR6sl1Uq8U61gxU+kPTs2tR8uOySCbBP7BN/M= -golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190927073244-c990c680b611 h1:q9u40nxWT5zRClI/uU9dHCiYGottAg6Nzz4YUQyHxdA= +golang.org/x/sys v0.0.0-20190927073244-c990c680b611/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= diff --git a/pkg/forkexec/fork.go b/pkg/forkexec/fork.go index d4d2296..f5325aa 100644 --- a/pkg/forkexec/fork.go +++ b/pkg/forkexec/fork.go @@ -472,10 +472,10 @@ childerror: func handleChildFailed(pid uintptr) { var wstatus syscall.WaitStatus // make sure not blocked - unix.Kill(int(pid), syscall.SIGKILL) + syscall.Kill(int(pid), syscall.SIGKILL) // child failed; wait for it to exit, to make sure the zombies don't accumulate - _, err3 := syscall.Wait4(int(pid), &wstatus, 0, nil) - for err3 == syscall.EINTR { - _, err3 = syscall.Wait4(int(pid), &wstatus, 0, nil) + _, err := syscall.Wait4(int(pid), &wstatus, 0, nil) + for err == syscall.EINTR { + _, err = syscall.Wait4(int(pid), &wstatus, 0, nil) } } diff --git a/types/types.go b/types/types.go index 1595152..e491b3d 100644 --- a/types/types.go +++ b/types/types.go @@ -9,11 +9,7 @@ type Result struct { Error string // potential detailed error message UserTime uint64 // used user CPU time (in ms) UserMem uint64 // used user memory (in kb) - Stat // collects time usage for the runner -} - -// Stat is the time usages in ns -type Stat struct { + // collects time usage for the runner SetUpTime time.Duration RunningTime time.Duration }