diff --git a/cmd/runprog/main.go b/cmd/runprog/main.go index 93e7482..3c8c795 100644 --- a/cmd/runprog/main.go +++ b/cmd/runprog/main.go @@ -44,9 +44,12 @@ func printUsage() { os.Exit(2) } -func main() { - daemon.ContainerInit() +func init() { + // container init + daemon.Init() +} +func main() { flag.Usage = printUsage flag.Uint64Var(&timeLimit, "tl", 1, "Set time limit (in second)") flag.Uint64Var(&realTimeLimit, "rtl", 0, "Set real time limit (in second)") @@ -222,7 +225,11 @@ func start() (*types.Result, error) { } defer os.RemoveAll(root) - m, err := daemon.New(root) + b := daemon.Builder{ + Root: root, + } + + m, err := b.Build() if err != nil { return nil, fmt.Errorf("failed to new master: %v", err) } diff --git a/daemon/consts.go b/daemon/consts.go index a365420..529c54f 100644 --- a/daemon/consts.go +++ b/daemon/consts.go @@ -11,4 +11,6 @@ const ( cmdKill = "kill" initArg = "init" + + currentExec = "/proc/self/exe" ) diff --git a/daemon/container_init.go b/daemon/container_init.go index b04d534..6a3fa3f 100644 --- a/daemon/container_init.go +++ b/daemon/container_init.go @@ -13,11 +13,11 @@ import ( "github.com/criyle/go-sandbox/types" ) -// ContainerInit is called for container init process +// Init is called for container init process // it will check if pid == 1, otherwise it is noop -// ContainerInit will do infinite loop on socket commands, -// and exits when at socket close -func ContainerInit() (err error) { +// Init will do infinite loop on socket commands, +// and exits when at socket close, use it in init function +func Init() (err error) { // noop if self is not container init process // Notice: docker init is also 1, additional check for args[1] == init if os.Getpid() != 1 || len(os.Args) != 2 || os.Args[1] != initArg { @@ -26,6 +26,10 @@ func ContainerInit() (err error) { // exit process (with whole container) upon exit this function defer func() { + if err2 := recover(); err2 != nil { + fmt.Fprintf(os.Stderr, "container_panic: %v", err) + os.Exit(1) + } if err != nil { fmt.Fprintf(os.Stderr, "container_exit: %v", err) os.Exit(1) @@ -71,7 +75,7 @@ func handleCmd(s *unixsocket.Socket, cmd *Cmd, msg *unixsocket.Msg) error { case cmdExecve: return handleExecve(s, cmd, msg) } - return nil + return fmt.Errorf("Unknown command: %v", cmd.Cmd) } func handlePing(s *unixsocket.Socket) error { @@ -79,8 +83,9 @@ func handlePing(s *unixsocket.Socket) error { } func handleCopyIn(s *unixsocket.Socket, cmd *Cmd, msg *unixsocket.Msg) error { - if len(msg.Fds) == 0 { - return sendErrorReply(s, "copyin: did not receive fd") + if len(msg.Fds) != 1 { + closeFds(msg.Fds) + return sendErrorReply(s, "copyin: unexpected number of fds(%d)", len(msg.Fds)) } inf := os.NewFile(uintptr(msg.Fds[0]), cmd.Path) if inf == nil { @@ -109,27 +114,23 @@ func handleOpen(s *unixsocket.Socket, cmd *Cmd) error { } defer outf.Close() - msg := unixsocket.Msg{ + return sendReply(s, &Reply{}, &unixsocket.Msg{ Fds: []int{int(outf.Fd())}, - } - return sendReply(s, &Reply{}, &msg) + }) } func handleDelete(s *unixsocket.Socket, cmd *Cmd) error { - err := os.Remove(cmd.Path) - if err != nil { + if err := os.Remove(cmd.Path); err != nil { return sendErrorReply(s, "delete: %v", err) } return sendReply(s, &Reply{}, nil) } func handleReset(s *unixsocket.Socket) error { - err := removeContents("/tmp") - if err != nil { + if err := removeContents("/tmp"); err != nil { return sendErrorReply(s, "reset: /tmp %v", err) } - err = removeContents("/w") - if err != nil { + if err := removeContents("/w"); err != nil { return sendErrorReply(s, "reset: /w %v", err) } return sendReply(s, &Reply{}, nil) @@ -142,13 +143,17 @@ func handleExecve(s *unixsocket.Socket, cmd *Cmd, msg *unixsocket.Msg) error { ) if msg != nil { files = intSliceToUintptr(msg.Fds) + // don't leak fds to child + closeOnExecFds(msg.Fds) + // release files after execve + defer closeFds(msg.Fds) } - // don't leak fds to child - closeOnExecFds(msg.Fds) - // release files after execve - defer closeFds(msg.Fds) - if cmd.FdExec && len(files) > 0 { + // if fexecve, then the first fd must be executable + if cmd.FdExec { + if len(files) == 0 { + return fmt.Errorf("execve: expected fexecve fd") + } execFile = files[0] files = files[1:] } @@ -168,8 +173,7 @@ func handleExecve(s *unixsocket.Socket, cmd *Cmd, msg *unixsocket.Msg) error { if err2 != nil { return fmt.Errorf("syncFunc: recvCmd(%v)", err2) } - switch cmd2.Cmd { - case cmdKill: + if cmd2.Cmd == cmdKill { return fmt.Errorf("syncFunc: recved kill") } return nil @@ -185,7 +189,7 @@ func handleExecve(s *unixsocket.Socket, cmd *Cmd, msg *unixsocket.Msg) error { DropCaps: true, SyncFunc: syncFunc, } - // starts the runner, error is handled same as wait4 to make api simple + // starts the runner, error is handled same as wait4 to make communication equal pid, err := r.Start() // done is to signal kill goroutine exits @@ -205,8 +209,7 @@ func handleExecve(s *unixsocket.Socket, cmd *Cmd, msg *unixsocket.Msg) error { <-waitDone // collect zombies for { - pid, err := syscall.Wait4(-1, nil, syscall.WNOHANG, nil) - if err != nil || pid <= 0 { + if pid, err := syscall.Wait4(-1, nil, syscall.WNOHANG, nil); err != nil || pid <= 0 { break } } @@ -219,13 +222,13 @@ func handleExecve(s *unixsocket.Socket, cmd *Cmd, msg *unixsocket.Msg) error { } // sync with kill goroutine close(waitDone) + if err != nil { sendErrorReply(s, "execve: wait4 %v", err) } else { switch { case wstatus.Exited(): - reply := Reply{ExitStatus: wstatus.ExitStatus()} - sendReply(s, &reply, nil) + sendReply(s, &Reply{ExitStatus: wstatus.ExitStatus()}, nil) case wstatus.Signaled(): var status types.Status @@ -240,8 +243,7 @@ func handleExecve(s *unixsocket.Socket, cmd *Cmd, msg *unixsocket.Msg) error { default: status = types.StatusRE } - reply := Reply{Status: status} - sendReply(s, &reply, nil) + sendReply(s, &Reply{Status: status}, nil) default: sendErrorReply(s, "execve: unknown status %v", wstatus) } diff --git a/daemon/master.go b/daemon/master.go index 55f9f2a..f3506a3 100644 --- a/daemon/master.go +++ b/daemon/master.go @@ -7,71 +7,100 @@ import ( "github.com/criyle/go-sandbox/pkg/forkexec" "github.com/criyle/go-sandbox/pkg/memfd" "github.com/criyle/go-sandbox/pkg/mount" + "github.com/criyle/go-sandbox/pkg/pipe" "github.com/criyle/go-sandbox/pkg/unixsocket" "golang.org/x/sys/unix" ) +// Builder builds instance of container masters +type Builder struct { + // Root is container root mount path, empty uses current work path + Root string + + // Mounts defines container mount points, empty uses default mounts + Mounts []mount.SyscallParams + + // Stderr defines whether to collect container stderr output for debug + Stderr bool + + // ExecFile defines executable that called Init, otherwise defer current + // executable (/proc/self/exe) + ExecFile *os.File +} + // Master manages single pre-forked container type Master struct { pid int // underlying container init pid socket *unixsocket.Socket // master - container communication + buff *pipe.Buffer } // New creates new master with underlying container -func New(root string) (*Master, error) { - // dummy stdin / stdout / stderr - fnull, err := os.OpenFile(os.DevNull, os.O_RDWR, 0666) +func (b *Builder) Build() (*Master, error) { + var ( + err error + buff *pipe.Buffer + ) + // container mount points + mounts := b.Mounts + if len(mounts) == 0 { + if mounts, err = mount.NewBuilder().WithMounts(DefaultMounts).Build(true); err != nil { + return nil, fmt.Errorf("daemon: failed to build rootfs mount %v", err) + } + } + root := b.Root + if root == "" { + if root, err = os.Getwd(); err != nil { + return nil, fmt.Errorf("daemon: failed to get work directory %v", err) + } + } + // prepare stdin / stdout / stderr + fnull, err := os.OpenFile(os.DevNull, os.O_RDWR, os.ModePerm) if err != nil { return nil, fmt.Errorf("daemon: failed to open devNull(%v)", err) } defer fnull.Close() - // prepare self memfd - self, err := os.Open("/proc/self/exe") - if err != nil { - return nil, fmt.Errorf("daemon: failed to open /proc/self/exe(%v)", err) + files := make([]uintptr, 0, 4) + files = append(files, []uintptr{fnull.Fd(), fnull.Fd()}...) + if b.Stderr { + buff, err = pipe.NewBuffer(bufferSize) + if err != nil { + return nil, fmt.Errorf("daemon: failed to open stderr pipe: %v", err) + } + defer buff.W.Close() + files = append(files, buff.W.Fd()) + } else { + files = append(files, fnull.Fd()) } - defer self.Close() - execFile, err := memfd.DupToMemfd("daemon", self) + // prepare self memfd + execFile, err := b.exec() if err != nil { - return nil, fmt.Errorf("daemon: failed to create memfd(%v)", err) + return nil, fmt.Errorf("deamon: %v", err) } defer execFile.Close() // prepare socket - ins, outs, err := unixsocket.NewSocketPair() + ins, outs, err := newPassCredSocketPair() if err != nil { - return nil, fmt.Errorf("daemon: failed to create socket(%v)", err) + return nil, fmt.Errorf("daemon: failed to create socket: %v", err) } - defer outs.Conn.Close() + defer outs.Close() - outf, err := outs.Conn.File() + outf, err := outs.File() if err != nil { - ins.Conn.Close() + ins.Close() return nil, fmt.Errorf("daemon: failed to dup file outs(%v)", err) } defer outf.Close() - if err = ins.SetPassCred(1); err != nil { - ins.Conn.Close() - return nil, fmt.Errorf("daemon: failed to set pass_cred ins(%v)", err) - } - if err = outs.SetPassCred(1); err != nil { - ins.Conn.Close() - return nil, fmt.Errorf("daemon: failed to set pass_cred outs(%v)", err) - } - mounts, err := mount.NewBuilder().WithMounts(DefaultMounts).Build(true) - if err != nil { - ins.Conn.Close() - return nil, fmt.Errorf("daemon: failed to build rootfs mount %v", err) - } - + files = append(files, uintptr(outf.Fd())) r := &forkexec.Runner{ Args: []string{os.Args[0], initArg}, Env: []string{DefaultPath}, ExecFile: execFile.Fd(), - Files: []uintptr{fnull.Fd(), fnull.Fd(), fnull.Fd(), uintptr(outf.Fd())}, + Files: files, WorkDir: "/w", UnshareFlags: forkexec.UnshareFlags, Mounts: mounts, @@ -81,16 +110,64 @@ func New(root string) (*Master, error) { } pid, err := r.Start() if err != nil { - ins.Conn.Close() + ins.Close() return nil, fmt.Errorf("daemon: failed to execve(%v)", err) } - return &Master{pid, ins}, nil + return &Master{pid, ins, buff}, nil } // Destroy kill the daemon process (with container) +// if stderr enabled, collect the output as error func (m *Master) Destroy() error { var wstatus unix.WaitStatus unix.Kill(m.pid, unix.SIGKILL) - _, err := unix.Wait4(m.pid, &wstatus, 0, nil) - return err + if _, err := unix.Wait4(m.pid, &wstatus, 0, nil); err != nil { + return err + } + if m.buff != nil { + <-m.buff.Done + return fmt.Errorf("destroy: %s", m.buff.Buffer.Bytes()) + } + return nil +} + +// exec prepares executable +func (b *Builder) exec() (*os.File, error) { + if b.ExecFile != nil { + return b.ExecFile, nil + } + return OpenCurrentExec() +} + +// OpenCurrentExec opens current executable and dup it to memfd +func OpenCurrentExec() (*os.File, error) { + self, err := os.Open(currentExec) + if err != nil { + return nil, fmt.Errorf("failed to open %v: %v", currentExec, err) + } + defer self.Close() + + execFile, err := memfd.DupToMemfd("daemon", self) + if err != nil { + return nil, fmt.Errorf("failed to create memfd: %v", err) + } + return execFile, nil +} + +func newPassCredSocketPair() (*unixsocket.Socket, *unixsocket.Socket, error) { + ins, outs, err := unixsocket.NewSocketPair() + if err != nil { + return nil, nil, err + } + if err = ins.SetPassCred(1); err != nil { + ins.Close() + outs.Close() + return nil, nil, err + } + if err = outs.SetPassCred(1); err != nil { + ins.Close() + outs.Close() + return nil, nil, err + } + return ins, outs, nil } diff --git a/daemon/master_cmd.go b/daemon/master_cmd.go index 260c3f1..6d56dc2 100644 --- a/daemon/master_cmd.go +++ b/daemon/master_cmd.go @@ -55,11 +55,13 @@ func (m *Master) Open(p string) (*os.File, error) { if reply.Error != "" { return nil, fmt.Errorf("open: %v", reply.Error) } - if len(msg.Fds) == 0 { - return nil, fmt.Errorf("open: did not receive fd") + if len(msg.Fds) != 1 { + closeFds(msg.Fds) + return nil, fmt.Errorf("open: unexpected number of fd %v", len(msg.Fds)) } f := os.NewFile(uintptr(msg.Fds[0]), p) if f == nil { + closeFds(msg.Fds) return nil, fmt.Errorf("open: failed %v", msg.Fds[0]) } return f, nil @@ -91,31 +93,36 @@ func (m *Master) Reset() error { func (m *Master) recvAckReply(name string) error { reply, _, err := m.recvReply() if err != nil { - return fmt.Errorf(name+": %v", err) + return fmt.Errorf("%v: recvAck %v", name, err) } if reply.Error != "" { - return fmt.Errorf(name+": %v", reply.Error) + return fmt.Errorf("%v: container error %v", name, reply.Error) } return nil } func (m *Master) recvReply() (*Reply, *unixsocket.Msg, error) { - var reply Reply buff := GetBuffer() defer PutBuffer(buff) + n, msg, err := m.socket.RecvMsg(buff) if err != nil { return nil, nil, fmt.Errorf("failed to RecvMsg(%v)", err) } - if err := gob.NewDecoder(bytes.NewReader(buff[:n])).Decode(&reply); err != nil { + + reply := new(Reply) + if err := gob.NewDecoder(bytes.NewReader(buff[:n])).Decode(reply); err != nil { return nil, nil, fmt.Errorf("failed to decode(%v)", err) } - return &reply, msg, nil + return reply, msg, nil } func (m *Master) sendCmd(cmd *Cmd, msg *unixsocket.Msg) error { - var buff bytes.Buffer - if err := gob.NewEncoder(&buff).Encode(cmd); err != nil { + buf := GetBuffer() + defer PutBuffer(buf) + // use buf pool to reduce allocation + buff := bytes.NewBuffer(buf[:0]) + if err := gob.NewEncoder(buff).Encode(cmd); err != nil { return fmt.Errorf("failed to encode(%v)", err) } if err := m.socket.SendMsg(buff.Bytes(), msg); err != nil { diff --git a/daemon/master_cmd_execve.go b/daemon/master_cmd_execve.go index bad29c9..82c3da5 100644 --- a/daemon/master_cmd_execve.go +++ b/daemon/master_cmd_execve.go @@ -44,27 +44,27 @@ func (m *Master) Execve(done <-chan struct{}, param *ExecveParam) (<-chan types. } reply, msg, err := m.recvReply() if err != nil { - return nil, fmt.Errorf("execve: RecvReply %v", err) + return nil, fmt.Errorf("execve: recvReply %v", err) } if reply.Error != "" || msg == nil || msg.Cred == nil { - m.sendCmd(&Cmd{Cmd: cmdKill}, nil) + m.execveSyncKill() return nil, fmt.Errorf("execve: no pid recved or error(%v)", reply.Error) } if param.SyncFunc != nil { if err := param.SyncFunc(int(msg.Cred.Pid)); err != nil { - m.sendCmd(&Cmd{Cmd: cmdKill}, nil) + m.execveSyncKill() return nil, fmt.Errorf("execve: syncfunc failed(%v)", err) } } + // send to syncFunc ack ok if err := m.sendCmd(&Cmd{Cmd: cmdOk}, nil); err != nil { return nil, fmt.Errorf("execve: ok failed(%v)", err) } // make sure goroutine not leaked (blocked) even if result is not consumed - wait := make(chan types.Result, 1) + result := make(chan types.Result, 1) waitDone := make(chan struct{}) // Wait go func() { - defer close(wait) reply2, _, _ := m.recvReply() close(waitDone) // done signal (should recv after kill) @@ -74,7 +74,7 @@ func (m *Master) Execve(done <-chan struct{}, param *ExecveParam) (<-chan types. if reply2.Error != "" { status = types.StatusFatal } - wait <- types.Result{ + result <- types.Result{ ExitStatus: reply2.ExitStatus, Status: status, Error: reply2.Error, @@ -88,5 +88,14 @@ func (m *Master) Execve(done <-chan struct{}, param *ExecveParam) (<-chan types. } m.sendCmd(&Cmd{Cmd: cmdKill}, nil) }() - return wait, nil + return result, nil +} + +// handleSyncKill will send kill (for sync func), kill (for wait to kill all) and +// then recv error and finish signal when terminated by syncFunc error +func (m *Master) execveSyncKill() { + m.sendCmd(&Cmd{Cmd: cmdKill}, nil) + m.sendCmd(&Cmd{Cmd: cmdKill}, nil) + m.recvReply() + m.recvReply() } diff --git a/go.mod b/go.mod index 5754fc4..3716cb4 100644 --- a/go.mod +++ b/go.mod @@ -4,5 +4,5 @@ go 1.12 require ( github.com/seccomp/libseccomp-golang v0.9.1 - golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3 + golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe ) diff --git a/go.sum b/go.sum index c2acd20..641720b 100644 --- a/go.sum +++ b/go.sum @@ -1,4 +1,4 @@ github.com/seccomp/libseccomp-golang v0.9.1 h1:NJjM5DNFOs0s3kYE1WUOr6G8V97sdt46rlXTMfXGWBo= github.com/seccomp/libseccomp-golang v0.9.1/go.mod h1:GbW5+tmTXfcxTToHLXlScSlAvWlF4P2Ca7zGrPiEpWo= -golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3 h1:7TYNF4UdlohbFwpNH04CoPMp1cHUZgO1Ebq5r2hIjfo= -golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe h1:6fAMxZRR6sl1Uq8U61gxU+kPTs2tR8uOySCbBP7BN/M= +golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= diff --git a/pkg/unixsocket/socket.go b/pkg/unixsocket/socket.go index 956598b..eb43058 100644 --- a/pkg/unixsocket/socket.go +++ b/pkg/unixsocket/socket.go @@ -20,7 +20,7 @@ var oobPool = sync.Pool{ // Socket represents a unix socket type Socket struct { - Conn *net.UnixConn + *net.UnixConn } // Msg is the oob msg with the message @@ -67,7 +67,7 @@ func NewSocketPair() (*Socket, *Socket, error) { } outs, err := NewSocket(fd[1]) if err != nil { - ins.Conn.Close() + ins.Close() syscall.Close(fd[1]) return nil, nil, fmt.Errorf("NewSocketPair: failed to call NewSocket outs(%v)", err) } @@ -76,7 +76,7 @@ func NewSocketPair() (*Socket, *Socket, error) { // SetPassCred set sockopt for pass cred for unix socket func (s *Socket) SetPassCred(option int) error { - sysconn, err := s.Conn.SyscallConn() + sysconn, err := s.SyscallConn() if err != nil { return err } @@ -96,7 +96,7 @@ func (s *Socket) SendMsg(b []byte, m *Msg) error { oob = append(oob, syscall.UnixCredentials(m.Cred)...) } } - _, _, err := s.Conn.WriteMsgUnix(b, oob, nil) + _, _, err := s.WriteMsgUnix(b, oob, nil) if err != nil { return err } @@ -107,7 +107,7 @@ func (s *Socket) SendMsg(b []byte, m *Msg) error { func (s *Socket) RecvMsg(b []byte) (int, *Msg, error) { oob := oobPool.Get().([]byte) defer oobPool.Put(oob) - n, oobn, _, _, err := s.Conn.ReadMsgUnix(b, oob) + n, oobn, _, _, err := s.ReadMsgUnix(b, oob) if err != nil { return 0, nil, err }