From 6b97e1f663323c1b379224ce901986a78514e56f Mon Sep 17 00:00:00 2001 From: criyle Date: Wed, 25 Dec 2019 19:51:28 +0800 Subject: [PATCH] use open syscall to copyin files --- README.md | 27 +++++++++++ daemon/consts.go | 2 + daemon/container_execve.go | 25 ++++++---- daemon/container_init.go | 83 ++++++++++++++++----------------- daemon/deamon.go | 92 ++++++++++++++++++++++++++----------- daemon/master_cmd.go | 58 +++++++++-------------- daemon/master_cmd_execve.go | 36 ++++++++++----- 7 files changed, 199 insertions(+), 124 deletions(-) diff --git a/README.md b/README.md index 31f8f16..73c6824 100644 --- a/README.md +++ b/README.md @@ -44,6 +44,33 @@ Default file access syscall check: 1. Pre-fork container daemons to run programs inside 2. Unix socket to pass fd inside / outside +Container / Master Communication Protocol (single thread): + +- ping (alive check): + - reply: pong +- conf (set configuration): + - reply pong +- open (open files in given mode inside container): + - send: []OpenCmd + - reply: "success", file fds / "error" +- delete (unlink file / rmdir dir inside container): + - send: path + - reply: "finished" / "error" +- reset (clean up container for later use (clear workdir / tmp)): + - send: + - reply: "success" +- execve: (execute file inside container): + - send: argv, env, rLimits, fds + - reply: + - success: "success", pid + - failed: "failed" + - send (success): "init_finished" (as cmd) + - reply: "finished" / send: "kill" (as cmd) + - send: "kill" (as cmd) / reply: "finished" + - reply: + +Any socket related error will cause the daemon exit (with all process inside container) + ## Packages (/pkg) - seccomp: provides seccomp type definition diff --git a/daemon/consts.go b/daemon/consts.go index f779bb3..c7166a3 100644 --- a/daemon/consts.go +++ b/daemon/consts.go @@ -20,4 +20,6 @@ const ( containerName = "daemon" containerWD = "/w" + + containerMaxProc = 1 ) diff --git a/daemon/container_execve.go b/daemon/container_execve.go index f240d66..d7fd0e9 100644 --- a/daemon/container_execve.go +++ b/daemon/container_execve.go @@ -9,12 +9,15 @@ import ( "github.com/criyle/go-sandbox/types" ) -func (c *containerServer) handleExecve(cmd *Cmd, msg *unixsocket.Msg) error { +func (c *containerServer) handleExecve(cmd *ExecCmd, msg *unixsocket.Msg) error { var ( files []uintptr execFile uintptr cred *syscall.Credential ) + if cmd == nil { + return c.sendErrorReply("execve: no parameter provided") + } if msg != nil { files = intSliceToUintptr(msg.Fds) // don't leak fds to child @@ -118,10 +121,12 @@ func (c *containerServer) handleExecve(cmd *Cmd, msg *unixsocket.Msg) error { case wstatus.Exited(): exitStatus := wstatus.ExitStatus() c.sendReply(&Reply{ - Status: status, - ExitStatus: exitStatus, - UserTime: userTime, - UserMem: userMem, + ExecReply: &ExecReply{ + Status: status, + ExitStatus: exitStatus, + UserTime: userTime, + UserMem: userMem, + }, }, nil) case wstatus.Signaled(): @@ -137,14 +142,18 @@ func (c *containerServer) handleExecve(cmd *Cmd, msg *unixsocket.Msg) error { status = types.StatusRE } c.sendReply(&Reply{ - Status: status, - UserTime: userTime, - UserMem: userMem, + ExecReply: &ExecReply{ + Status: status, + UserTime: userTime, + UserMem: userMem, + }, }, nil) + default: c.sendErrorReply("execve: unknown status %v", wstatus) } } + // wait for kill msg and reply done for finish <-killDone return c.sendReply(&Reply{}, nil) diff --git a/daemon/container_init.go b/daemon/container_init.go index ae73818..082acc2 100644 --- a/daemon/container_init.go +++ b/daemon/container_init.go @@ -2,8 +2,9 @@ package daemon import ( "fmt" - "io" "os" + "runtime" + "syscall" "github.com/criyle/go-sandbox/pkg/unixsocket" ) @@ -13,6 +14,7 @@ type containerServer struct { containerConfig } +// ContainerConfig set the container config type containerConfig struct { Cred bool } @@ -46,6 +48,9 @@ func Init() (err error) { os.Exit(0) }() + // limit container resource usage + runtime.GOMAXPROCS(containerMaxProc) + // new_master shared the socket at fd 3 (marked close_exec) const defaultFd = 3 soc, err := unixsocket.NewSocket(defaultFd) @@ -76,22 +81,19 @@ func (c *containerServer) handleCmd(cmd *Cmd, msg *unixsocket.Msg) error { return c.handlePing() case cmdConf: - return c.handleConf(cmd) - - case cmdCopyIn: - return c.handleCopyIn(cmd, msg) + return c.handleConf(cmd.ConfCmd) case cmdOpen: - return c.handleOpen(cmd) + return c.handleOpen(cmd.OpenCmd) case cmdDelete: - return c.handleDelete(cmd) + return c.handleDelete(cmd.DeleteCmd) case cmdReset: return c.handleReset() case cmdExecve: - return c.handleExecve(cmd, msg) + return c.handleExecve(cmd.ExecCmd, msg) } return fmt.Errorf("unknown command: %v", cmd.Cmd) } @@ -100,51 +102,37 @@ func (c *containerServer) handlePing() error { return c.sendReply(&Reply{}, nil) } -func (c *containerServer) handleConf(cmd *Cmd) error { - if cmd.Conf != nil { - c.containerConfig = *cmd.Conf +func (c *containerServer) handleConf(conf *ConfCmd) error { + if conf != nil { + c.containerConfig = conf.Conf } return c.sendReply(&Reply{}, nil) } -func (c *containerServer) handleCopyIn(cmd *Cmd, msg *unixsocket.Msg) error { - if len(msg.Fds) != 1 { - closeFds(msg.Fds) - return c.sendErrorReply("copyin: unexpected number of fds(%d)", len(msg.Fds)) +func (c *containerServer) handleOpen(open []OpenCmd) error { + if len(open) == 0 { + return c.sendErrorReply("open: no open parameter received") } - inf := os.NewFile(uintptr(msg.Fds[0]), cmd.Path) - if inf == nil { - return c.sendErrorReply("copyin: newfile failed %v", msg.Fds[0]) - } - defer inf.Close() - // have 0777 permission to be able copy in executables - outf, err := os.OpenFile(cmd.Path, os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0777) - if err != nil { - return c.sendErrorReply("copyin: open write file %v", err) + // open files + fds := make([]int, 0, len(open)) + for _, o := range open { + outFile, err := os.OpenFile(o.Path, o.Flag, o.Perm) + if err != nil { + return c.sendErrorReply("open: %v", err) + } + defer outFile.Close() + fds = append(fds, int(outFile.Fd())) } - defer outf.Close() - if _, err = io.Copy(outf, inf); err != nil { - return c.sendErrorReply("copyin: io.copy %v", err) - } - return c.sendReply(&Reply{}, nil) + return c.sendReply(&Reply{}, &unixsocket.Msg{Fds: fds}) } -func (c *containerServer) handleOpen(cmd *Cmd) error { - outf, err := os.Open(cmd.Path) - if err != nil { - return c.sendErrorReply("open: %v", err) +func (c *containerServer) handleDelete(delete *DeleteCmd) error { + if delete == nil { + return c.sendErrorReply("delete: no parameter provided") } - defer outf.Close() - - return c.sendReply(&Reply{}, &unixsocket.Msg{ - Fds: []int{int(outf.Fd())}, - }) -} - -func (c *containerServer) handleDelete(cmd *Cmd) error { - if err := os.Remove(cmd.Path); err != nil { + if err := os.Remove(delete.Path); err != nil { return c.sendErrorReply("delete: %v", err) } return c.sendReply(&Reply{}, nil) @@ -175,5 +163,14 @@ func (c *containerServer) sendReply(reply *Reply, msg *unixsocket.Msg) error { // sendErrorReply sends error reply func (c *containerServer) sendErrorReply(ft string, v ...interface{}) error { - return c.sendReply(&Reply{Error: fmt.Sprintf(ft, v...)}, nil) + errorReply := &ErrorReply{ + Msg: fmt.Sprintf(ft, v...), + } + // store errno + if len(v) == 1 { + if errno, ok := v[0].(syscall.Errno); ok { + errorReply.Errno = &errno + } + } + return c.sendReply(&Reply{Error: errorReply}, nil) } diff --git a/daemon/deamon.go b/daemon/deamon.go index aaa391d..06edfaf 100644 --- a/daemon/deamon.go +++ b/daemon/deamon.go @@ -5,57 +5,97 @@ package daemon /* -Protocol between client and daemon (not thread safe): +Container / Master Communication Protocol (single thread): + - ping (alive check): - reply: pong + - reply: pong - conf (set configuration): - reply pong -- copyin (copy file into container): - send: path, - reply: "finished" (after copy finished) / "error" -- open (open file in read-only mode inside container): - send: path - reply: "success", / "error" + - reply pong +- open (open files in given mode inside container): + - send: []OpenCmd + - reply: "success", file fds / "error" - delete (unlink file / rmdir dir inside container): - send: path - reply: "finished" / "error" + - send: path + - reply: "finished" / "error" - reset (clean up container for later use (clear workdir / tmp)): - send: - reply: "success" + - send: + - reply: "success" - execve: (execute file inside container): - send: argv, env, rLimits, - reply: - - success: "success", + - send: argv, env, rLimits, fds + - reply: + - success: "success", pid - failed: "failed" - send (success): "init_finished" (as cmd) + - send (success): "init_finished" (as cmd) - reply: "finished" / send: "kill" (as cmd) - send: "kill" (as cmd) / reply: "finished" - reply: + - reply: Any socket related error will cause the daemon exit (with all process inside container) */ import ( + "os" + "syscall" + "github.com/criyle/go-sandbox/pkg/rlimit" "github.com/criyle/go-sandbox/types" ) // Cmd is the control message send into daemon type Cmd struct { - Cmd string // type of the cmd - Path string // path (copyin / open) - Argv []string // execve argv - Env []string // execve env - RLimits []rlimit.RLimit // execve posix rlimit - FdExec bool // if use fexecve (fd[0] as exec) - Conf *containerConfig // to set configuration + Cmd string // type of the cmd + + OpenCmd []OpenCmd // open argument + DeleteCmd *DeleteCmd // delete argument + ExecCmd *ExecCmd // execve argument + ConfCmd *ConfCmd // to set configuration +} + +// OpenCmd correspond to a single open syscall +type OpenCmd struct { + Path string + Flag int + Perm os.FileMode +} + +// DeleteCmd stores delete command +type DeleteCmd struct { + Path string +} + +// ExecCmd stores execve parameter +type ExecCmd struct { + Argv []string // execve argv + Env []string // execve env + RLimits []rlimit.RLimit // execve posix rlimit + FdExec bool // if use fexecve (fd[0] as exec) +} + +// ConfCmd stores conf parameter +type ConfCmd struct { + Conf containerConfig } // Reply is the reply message send back to controller type Reply struct { - Error string // empty if no error + Error *ErrorReply // nil if no error + ExecReply *ExecReply +} + +// ErrorReply stores error returned back from container +type ErrorReply struct { + Msg string + Errno *syscall.Errno +} + +// ExecReply stores execve result +type ExecReply struct { ExitStatus int // waitpid exit status Status types.Status // return status UserTime uint64 // waitpid user CPU (ms) UserMem uint64 // waitpid user memory (kb) } + +func (e *ErrorReply) Error() string { + return e.Msg +} diff --git a/daemon/master_cmd.go b/daemon/master_cmd.go index 7b9bac8..ae9e3b2 100644 --- a/daemon/master_cmd.go +++ b/daemon/master_cmd.go @@ -35,8 +35,8 @@ func (m *Master) conf(conf *containerConfig) error { defer m.mu.Unlock() cmd := Cmd{ - Cmd: cmdConf, - Conf: conf, + Cmd: cmdConf, + ConfCmd: &ConfCmd{Conf: *conf}, } if err := m.sendCmd(&cmd, nil); err != nil { return fmt.Errorf("conf: %v", err) @@ -44,34 +44,15 @@ func (m *Master) conf(conf *containerConfig) error { return m.recvAckReply("conf") } -// CopyIn copies file to container -func (m *Master) CopyIn(f *os.File, p string) error { +// Open open files in container +func (m *Master) Open(p []OpenCmd) ([]*os.File, error) { m.mu.Lock() defer m.mu.Unlock() // send copyin cmd := Cmd{ - Cmd: cmdCopyIn, - Path: p, - } - msg := unixsocket.Msg{ - Fds: []int{int(f.Fd())}, - } - if err := m.sendCmd(&cmd, &msg); err != nil { - return fmt.Errorf("copyin: %v", err) - } - return m.recvAckReply("copyin") -} - -// Open open file in container -func (m *Master) Open(p string) (*os.File, error) { - m.mu.Lock() - defer m.mu.Unlock() - - // send copyin - cmd := Cmd{ - Cmd: cmdOpen, - Path: p, + Cmd: cmdOpen, + OpenCmd: p, } if err := m.sendCmd(&cmd, nil); err != nil { return nil, fmt.Errorf("open: %v", err) @@ -80,19 +61,24 @@ func (m *Master) Open(p string) (*os.File, error) { if err != nil { return nil, fmt.Errorf("open: %v", err) } - if reply.Error != "" { + if reply.Error != nil { return nil, fmt.Errorf("open: %v", reply.Error) } - if len(msg.Fds) != 1 { + if len(msg.Fds) != len(p) { closeFds(msg.Fds) - return nil, fmt.Errorf("open: unexpected number of fd %v", len(msg.Fds)) + return nil, fmt.Errorf("open: unexpected number of fd %v / %v", len(msg.Fds), len(p)) } - f := os.NewFile(uintptr(msg.Fds[0]), p) - if f == nil { - closeFds(msg.Fds) - return nil, fmt.Errorf("open: failed %v", msg.Fds[0]) + + ret := make([]*os.File, 0, len(p)) + for i, fd := range msg.Fds { + f := os.NewFile(uintptr(fd), p[i].Path) + if f == nil { + closeFds(msg.Fds) + return nil, fmt.Errorf("open: failed %v", msg.Fds[0]) + } + ret = append(ret, f) } - return f, nil + return ret, nil } // Delete remove file from container @@ -101,8 +87,8 @@ func (m *Master) Delete(p string) error { defer m.mu.Unlock() cmd := Cmd{ - Cmd: cmdDelete, - Path: p, + Cmd: cmdDelete, + DeleteCmd: &DeleteCmd{Path: p}, } if err := m.sendCmd(&cmd, nil); err != nil { return fmt.Errorf("delete: %v", err) @@ -129,7 +115,7 @@ func (m *Master) recvAckReply(name string) error { if err != nil { return fmt.Errorf("%v: recvAck %v", name, err) } - if reply.Error != "" { + if reply.Error != nil { return fmt.Errorf("%v: container error %v", name, reply.Error) } return nil diff --git a/daemon/master_cmd_execve.go b/daemon/master_cmd_execve.go index bbba357..c138da4 100644 --- a/daemon/master_cmd_execve.go +++ b/daemon/master_cmd_execve.go @@ -41,13 +41,16 @@ func (m *Master) Execve(done <-chan struct{}, param *ExecveParam) (<-chan types. msg := &unixsocket.Msg{ Fds: files, } - cmd := Cmd{ - Cmd: cmdExecve, + execCmd := &ExecCmd{ Argv: param.Args, Env: param.Env, RLimits: param.RLimits, FdExec: param.ExecFile > 0, } + cmd := Cmd{ + Cmd: cmdExecve, + ExecCmd: execCmd, + } if err := m.sendCmd(&cmd, msg); err != nil { m.mu.Unlock() return nil, fmt.Errorf("execve: sendCmd %v", err) @@ -59,7 +62,7 @@ func (m *Master) Execve(done <-chan struct{}, param *ExecveParam) (<-chan types. return nil, fmt.Errorf("execve: recvReply %v", err) } // if sync function did not involved - if reply.Error != "" || msg == nil || msg.Cred == nil { + if reply.Error != nil || msg == nil || msg.Cred == nil { // tell kill function to exit and sync m.execveSyncKill() m.mu.Unlock() @@ -95,6 +98,7 @@ func (m *Master) Execve(done <-chan struct{}, param *ExecveParam) (<-chan types. m.recvReply() // unlock after last read / write m.mu.Unlock() + // handle potential error if err != nil { result <- types.Result{ @@ -103,17 +107,27 @@ func (m *Master) Execve(done <-chan struct{}, param *ExecveParam) (<-chan types. } return } - // emit result after all communication finish - status := reply2.Status - if reply2.Error != "" { - status = types.StatusFatal + if reply2.ExecReply == nil { + result <- types.Result{ + Status: types.StatusFatal, + Error: "execve: no reply received", + } + return } + // emit result after all communication finish + status := reply2.ExecReply.Status + errMsg := "" + if reply2.Error != nil { + status = types.StatusFatal + errMsg = reply2.Error.Error() + } + result <- types.Result{ Status: status, - ExitStatus: reply2.ExitStatus, - UserTime: reply2.UserTime, - UserMem: reply2.UserMem, - Error: reply2.Error, + ExitStatus: reply2.ExecReply.ExitStatus, + UserTime: reply2.ExecReply.UserTime, + UserMem: reply2.ExecReply.UserMem, + Error: errMsg, SetUpTime: mTime.Sub(sTime), RunningTime: time.Since(mTime), }