diff --git a/README.md b/README.md index a175330..23183b5 100644 --- a/README.md +++ b/README.md @@ -92,7 +92,7 @@ type Runner interface { ### Pre-forked Container Protocol -1. Pre-fork container daemons to run programs inside +1. Pre-fork container to run programs inside 2. Unix socket to pass fd inside / outside Container / Master Communication Protocol (single thread): @@ -120,7 +120,7 @@ Container / Master Communication Protocol (single thread): - send: "kill" (as cmd) / reply: "finished" - reply: -Any socket related error will cause the daemon exit (with all process inside container) +Any socket related error will cause the container exit (with all process inside container) ## Packages (/pkg) @@ -137,7 +137,7 @@ Any socket related error will cause the daemon exit (with all process inside con ## Packages - config: defines arch & language specified trace condition for ptrace runner from UOJ -- daemon: creates pre-forked container to run programs inside +- container: creates pre-forked container to run programs inside - runner: interface to run program - ptrace: wrapper to call forkexec and ptracer - filehandler: an example implementation of UOJ file set diff --git a/cmd/runprog/main.go b/cmd/runprog/main.go index 7403029..538afaf 100644 --- a/cmd/runprog/main.go +++ b/cmd/runprog/main.go @@ -10,7 +10,7 @@ import ( "time" "github.com/criyle/go-sandbox/config" - "github.com/criyle/go-sandbox/daemon" + "github.com/criyle/go-sandbox/container" "github.com/criyle/go-sandbox/pkg/cgroup" "github.com/criyle/go-sandbox/pkg/memfd" "github.com/criyle/go-sandbox/pkg/mount" @@ -44,9 +44,9 @@ func printUsage() { os.Exit(2) } +// container init func init() { - // container init - daemon.Init() + container.Init() } func main() { @@ -71,7 +71,7 @@ func main() { flag.Var(&addRawWritable, "add-writable-raw", "Add a writable file (don't transform to its real path)") flag.BoolVar(&useCGroup, "cgroup", false, "Use cgroup to colloct resource usage") flag.BoolVar(&memfile, "memfd", false, "Use memfd as exec file") - flag.StringVar(&runt, "runner", "ptrace", "Runner for the program (ptrace, ns, daemon)") + flag.StringVar(&runt, "runner", "ptrace", "Runner for the program (ptrace, ns, container)") flag.Parse() args = flag.Args() @@ -133,13 +133,13 @@ func main() { } } -type daemonRunner struct { - *daemon.Master - *daemon.ExecveParam +type containerRunner struct { + container.Environment + container.ExecveParam } -func (r *daemonRunner) Run(c context.Context) <-chan types.Result { - return r.Master.Execve(c, r.ExecveParam) +func (r *containerRunner) Run(c context.Context) <-chan types.Result { + return r.Environment.Execve(c, r.ExecveParam) } func start() (*types.Result, error) { @@ -231,29 +231,29 @@ func start() (*types.Result, error) { MemoryLimit: types.Size(memoryLimit << 20), } - if runt == "daemon" { + if runt == "container" { root, err := ioutil.TempDir("", "dm") if err != nil { - return nil, fmt.Errorf("cannot make temp root for daemon namespace: %v", err) + return nil, fmt.Errorf("cannot make temp root for container namespace: %v", err) } defer os.RemoveAll(root) - b := daemon.Builder{ + b := container.Builder{ Root: root, } m, err := b.Build() if err != nil { - return nil, fmt.Errorf("failed to new master: %v", err) + return nil, fmt.Errorf("failed to new container: %v", err) } defer m.Destroy() err = m.Ping() if err != nil { - return nil, fmt.Errorf("failed to ping daemon: %v", err) + return nil, fmt.Errorf("failed to ping container: %v", err) } - runner = &daemonRunner{ - Master: m, - ExecveParam: &daemon.ExecveParam{ + runner = &containerRunner{ + Environment: m, + ExecveParam: container.ExecveParam{ Args: args, Env: []string{pathEnv}, Fds: fds, diff --git a/daemon/consts.go b/container/consts.go similarity index 87% rename from daemon/consts.go rename to container/consts.go index c7166a3..7107e4a 100644 --- a/daemon/consts.go +++ b/container/consts.go @@ -1,4 +1,4 @@ -package daemon +package container const ( cmdPing = "ping" @@ -18,7 +18,7 @@ const ( containerUID = 1000 containerGID = 1000 - containerName = "daemon" + containerName = "go-sandbox" containerWD = "/w" containerMaxProc = 1 diff --git a/container/container_cmd.go b/container/container_cmd.go new file mode 100644 index 0000000..f285bd9 --- /dev/null +++ b/container/container_cmd.go @@ -0,0 +1,86 @@ +package container + +import ( + "fmt" + "os" + "syscall" + + "github.com/criyle/go-sandbox/pkg/unixsocket" +) + +func (c *containerServer) handlePing() error { + return c.sendReply(&reply{}, nil) +} + +func (c *containerServer) handleConf(conf *confCmd) error { + if conf != nil { + c.containerConfig = conf.Conf + } + return c.sendReply(&reply{}, nil) +} + +func (c *containerServer) handleOpen(open []OpenCmd) error { + if len(open) == 0 { + return c.sendErrorReply("open: no open parameter received") + } + + // open files + fds := make([]int, 0, len(open)) + for _, o := range open { + outFile, err := os.OpenFile(o.Path, o.Flag, o.Perm) + if err != nil { + return c.sendErrorReply("open: %v", err) + } + defer outFile.Close() + fds = append(fds, int(outFile.Fd())) + } + + return c.sendReply(&reply{}, &unixsocket.Msg{Fds: fds}) +} + +func (c *containerServer) handleDelete(delete *deleteCmd) error { + if delete == nil { + return c.sendErrorReply("delete: no parameter provided") + } + if err := os.Remove(delete.Path); err != nil { + return c.sendErrorReply("delete: %v", err) + } + return c.sendReply(&reply{}, nil) +} + +func (c *containerServer) handleReset() error { + if err := removeContents("/tmp"); err != nil { + return c.sendErrorReply("reset: /tmp %v", err) + } + if err := removeContents("/w"); err != nil { + return c.sendErrorReply("reset: /w %v", err) + } + return c.sendReply(&reply{}, nil) +} + +func (c *containerServer) recvCmd() (*cmd, *unixsocket.Msg, error) { + cm := new(cmd) + msg, err := (*socket)(c.socket).RecvMsg(cm) + if err != nil { + return nil, nil, err + } + return cm, msg, nil +} + +func (c *containerServer) sendReply(rep *reply, msg *unixsocket.Msg) error { + return (*socket)(c.socket).SendMsg(rep, msg) +} + +// sendErrorReply sends error reply +func (c *containerServer) sendErrorReply(ft string, v ...interface{}) error { + errRep := &errorReply{ + Msg: fmt.Sprintf(ft, v...), + } + // store errno + if len(v) == 1 { + if errno, ok := v[0].(syscall.Errno); ok { + errRep.Errno = &errno + } + } + return c.sendReply(&reply{Error: errRep}, nil) +} diff --git a/daemon/container_execve.go b/container/container_exec.go similarity index 85% rename from daemon/container_execve.go rename to container/container_exec.go index eed1c8f..544a5eb 100644 --- a/daemon/container_execve.go +++ b/container/container_exec.go @@ -1,4 +1,4 @@ -package daemon +package container import ( "fmt" @@ -10,7 +10,7 @@ import ( "github.com/criyle/go-sandbox/types" ) -func (c *containerServer) handleExecve(cmd *ExecCmd, msg *unixsocket.Msg) error { +func (c *containerServer) handleExecve(cmd *execCmd, msg *unixsocket.Msg) error { var ( files []uintptr execFile uintptr @@ -37,21 +37,21 @@ func (c *containerServer) handleExecve(cmd *ExecCmd, msg *unixsocket.Msg) error } syncFunc := func(pid int) error { - msg2 := &unixsocket.Msg{ + msg := &unixsocket.Msg{ Cred: &syscall.Ucred{ Pid: int32(pid), Uid: uint32(syscall.Getuid()), Gid: uint32(syscall.Getgid()), }, } - if err2 := c.sendReply(&Reply{}, msg2); err2 != nil { - return fmt.Errorf("syncFunc: sendReply(%v)", err2) + if err := c.sendReply(&reply{}, msg); err != nil { + return fmt.Errorf("syncFunc: sendReply %w", err) } - cmd2, _, err2 := c.recvCmd() - if err2 != nil { - return fmt.Errorf("syncFunc: recvCmd(%v)", err2) + cmd, _, err := c.recvCmd() + if err != nil { + return fmt.Errorf("syncFunc: recvCmd %w", err) } - if cmd2.Cmd == cmdKill { + if cmd.Cmd == cmdKill { return fmt.Errorf("syncFunc: received kill") } return nil @@ -121,8 +121,11 @@ func (c *containerServer) handleExecve(cmd *ExecCmd, msg *unixsocket.Msg) error switch { case wstatus.Exited(): exitStatus := wstatus.ExitStatus() - c.sendReply(&Reply{ - ExecReply: &ExecReply{ + if exitStatus != 0 { + status = types.StatusNonzeroExitStatus + } + c.sendReply(&reply{ + ExecReply: &execReply{ Status: status, ExitStatus: exitStatus, Time: userTime, @@ -142,8 +145,8 @@ func (c *containerServer) handleExecve(cmd *ExecCmd, msg *unixsocket.Msg) error default: status = types.StatusSignalled } - c.sendReply(&Reply{ - ExecReply: &ExecReply{ + c.sendReply(&reply{ + ExecReply: &execReply{ ExitStatus: int(wstatus.Signal()), Status: status, Time: userTime, @@ -158,5 +161,5 @@ func (c *containerServer) handleExecve(cmd *ExecCmd, msg *unixsocket.Msg) error // wait for kill msg and reply done for finish <-killDone - return c.sendReply(&Reply{}, nil) + return c.sendReply(&reply{}, nil) } diff --git a/container/container_init.go b/container/container_init.go new file mode 100644 index 0000000..fba979b --- /dev/null +++ b/container/container_init.go @@ -0,0 +1,93 @@ +package container + +import ( + "fmt" + "os" + "runtime" + + "github.com/criyle/go-sandbox/pkg/unixsocket" +) + +type containerServer struct { + socket *unixsocket.Socket + containerConfig +} + +// Init is called for container init process +// it will check if pid == 1, otherwise it is noop +// Init will do infinite loop on socket commands, +// and exits when at socket close, use it in init function +func Init() (err error) { + // noop if self is not container init process + // Notice: docker init is also 1, additional check for args[1] == init + if os.Getpid() != 1 || len(os.Args) != 2 || os.Args[1] != initArg { + return nil + } + + // exit process (with whole container) upon exit this function + // possible reason: + // 1. socket broken (parent exit) + // 2. panic + // 3. undefined cmd (possible race condition) + defer func() { + if err := recover(); err != nil { + fmt.Fprintf(os.Stderr, "container_exit: panic: %v\n", err) + os.Exit(1) + } + if err != nil { + fmt.Fprintf(os.Stderr, "container_exit: %v\n", err) + os.Exit(1) + } + fmt.Fprintf(os.Stderr, "container_exit\n") + os.Exit(0) + }() + + // limit container resource usage + runtime.GOMAXPROCS(containerMaxProc) + + // new_master shared the socket at fd 3 (marked close_exec) + const defaultFd = 3 + soc, err := unixsocket.NewSocket(defaultFd) + if err != nil { + return fmt.Errorf("container_init: failed to new socket %w", err) + } + + // serve forever + cs := &containerServer{socket: soc} + return cs.serve() +} + +func (c *containerServer) serve() error { + for { + cmd, msg, err := c.recvCmd() + if err != nil { + return fmt.Errorf("serve: recvCmd %w", err) + } + if err := c.handleCmd(cmd, msg); err != nil { + return fmt.Errorf("serve: failed to execute cmd %w", err) + } + } +} + +func (c *containerServer) handleCmd(cmd *cmd, msg *unixsocket.Msg) error { + switch cmd.Cmd { + case cmdPing: + return c.handlePing() + + case cmdConf: + return c.handleConf(cmd.ConfCmd) + + case cmdOpen: + return c.handleOpen(cmd.OpenCmd) + + case cmdDelete: + return c.handleDelete(cmd.DeleteCmd) + + case cmdReset: + return c.handleReset() + + case cmdExecve: + return c.handleExecve(cmd.ExecCmd, msg) + } + return fmt.Errorf("unknown command: %s", cmd.Cmd) +} diff --git a/container/doc.go b/container/doc.go new file mode 100644 index 0000000..cc8a720 --- /dev/null +++ b/container/doc.go @@ -0,0 +1,44 @@ +// Package container provides pre-forked container environment to +// run programs in isolated Linux namespaces. +// +// Overview +// +// It creates container within unshared container and communicate +// with host process using unix socket with +// oob for fd / pid and commands encoded by gob. +// +// Protocol +// +// Host to container communication protocol is single threaded and always initiated by +// the host: +// +// - ping (alive check): +// - reply: pong +// +// - conf (set configuration): +// - reply pong +// +// - open (open files in given mode inside container): +// - send: []OpenCmd +// - reply: "success", file fds / "error" +// +// - delete (unlink file / rmdir dir inside container): +// - send: path +// - reply: "finished" / "error" +// +// - reset (clean up container for later use (clear workdir / tmp)): +// - send: +// - reply: "success" +// +// - execve: (execute file inside container): +// - send: argv, env, rLimits, fds +// - reply: +// - success: "success", pid +// - failed: "failed" +// - send (success): "init_finished" (as cmd) +// - reply: "finished" / send: "kill" (as cmd) +// - send: "kill" (as cmd) / reply: "finished" +// - reply: +// +// Any socket related error will cause the container exit with all process inside container +package container diff --git a/daemon/master.go b/container/environment.go similarity index 70% rename from daemon/master.go rename to container/environment.go index d22335a..e6de43d 100644 --- a/daemon/master.go +++ b/container/environment.go @@ -1,6 +1,7 @@ -package daemon +package container import ( + "context" "fmt" "os" "sync" @@ -10,13 +11,14 @@ import ( "github.com/criyle/go-sandbox/pkg/memfd" "github.com/criyle/go-sandbox/pkg/mount" "github.com/criyle/go-sandbox/pkg/unixsocket" + "github.com/criyle/go-sandbox/types" "golang.org/x/sys/unix" ) -// DefaultPath defines path evironment variable for container init process -const DefaultPath = "PATH=/usr/local/bin:/usr/bin:/bin" +// PathEnv defines path environment variable for the container init process +const PathEnv = "PATH=/usr/local/bin:/usr/bin:/bin" -// Builder builds instance of container masters +// Builder builds instance of container environment type Builder struct { // Root is container root mount path, empty uses current work path Root string @@ -41,15 +43,25 @@ type CredGenerator interface { Get() syscall.Credential } -// Master manages single pre-forked container -type Master struct { +// Environment holds single progrem containered environment +type Environment interface { + Ping() error + Open([]OpenCmd) ([]*os.File, error) + Delete(p string) error + Reset() error + Execve(context.Context, ExecveParam) <-chan types.Result + Destroy() error +} + +// container manages single pre-forked container environment +type container struct { pid int // underlying container init pid socket *unixsocket.Socket // master - container communication mu sync.Mutex // lock to avoid race condition } -// Build creates new master with underlying container -func (b *Builder) Build() (*Master, error) { +// Build creates new environment with underlying container +func (b *Builder) Build() (Environment, error) { var ( err error cred syscall.Credential @@ -64,19 +76,22 @@ func (b *Builder) Build() (*Master, error) { WithTmpfs("w", ""). // work dir WithTmpfs("tmp", ""). // tmp Build(true); err != nil { - return nil, fmt.Errorf("daemon: failed to build rootfs mount %v", err) + return nil, fmt.Errorf("container: failed to build rootfs mount %w", err) } } + + // container root directory on the host root := b.Root if root == "" { if root, err = os.Getwd(); err != nil { - return nil, fmt.Errorf("daemon: failed to get work directory %v", err) + return nil, fmt.Errorf("container: failed to get work directory %w", err) } } + // prepare stdin / stdout / stderr devNull, err := os.OpenFile(os.DevNull, os.O_RDWR, os.ModePerm) if err != nil { - return nil, fmt.Errorf("daemon: failed to open devNull(%v)", err) + return nil, fmt.Errorf("container: failed to open devNull %w", err) } defer devNull.Close() @@ -88,30 +103,30 @@ func (b *Builder) Build() (*Master, error) { files = append(files, devNull.Fd()) } - // prepare self memfd + // prepare container exec file execFile, err := b.exec() if err != nil { - return nil, fmt.Errorf("deamon: %v", err) + return nil, fmt.Errorf("container: prepare exec %w", err) } defer execFile.Close() - // prepare socket + // prepare host <-> container unix socket ins, outs, err := newPassCredSocketPair() if err != nil { - return nil, fmt.Errorf("daemon: failed to create socket: %v", err) + return nil, fmt.Errorf("container: failed to create socket: %w", err) } defer outs.Close() outf, err := outs.File() if err != nil { ins.Close() - return nil, fmt.Errorf("daemon: failed to dup file outs(%v)", err) + return nil, fmt.Errorf("container: failed to dup container socket fd %w", err) } defer outf.Close() files = append(files, uintptr(outf.Fd())) - // prepare credential + // prepare container running credential if b.CredGenerator != nil { cred = b.CredGenerator.Get() uidMap, gidMap = getIDMapping(&cred) @@ -119,7 +134,7 @@ func (b *Builder) Build() (*Master, error) { r := &forkexec.Runner{ Args: []string{os.Args[0], initArg}, - Env: []string{DefaultPath}, + Env: []string{PathEnv}, ExecFile: execFile.Fd(), Files: files, WorkDir: containerWD, @@ -134,41 +149,42 @@ func (b *Builder) Build() (*Master, error) { pid, err := r.Start() if err != nil { ins.Close() - return nil, fmt.Errorf("daemon: failed to execve(%v)", err) + return nil, fmt.Errorf("container: failed to start container %w", err) } - m := &Master{ + + c := &container{ pid: pid, socket: ins, } // set configuration and check if container creation successful - if err = m.conf(&containerConfig{ + if err = c.conf(&containerConfig{ Cred: b.CredGenerator != nil, }); err != nil { - m.Destroy() + c.Destroy() return nil, err } - return m, nil + return c, nil } -// Destroy kill the daemon process (with container) +// Destroy kill the container process (with its children) // if stderr enabled, collect the output as error -func (m *Master) Destroy() error { +func (c *container) Destroy() error { // close socket (abort any ongoing command) - m.socket.Close() + c.socket.Close() // wait commands terminates - m.mu.Lock() - defer m.mu.Unlock() + c.mu.Lock() + defer c.mu.Unlock() // kill process var wstatus unix.WaitStatus - unix.Kill(m.pid, unix.SIGKILL) + unix.Kill(c.pid, unix.SIGKILL) // wait for container process to exit - _, err := unix.Wait4(m.pid, &wstatus, 0, nil) + _, err := unix.Wait4(c.pid, &wstatus, 0, nil) for err == unix.EINTR { - _, err = unix.Wait4(m.pid, &wstatus, 0, nil) + _, err = unix.Wait4(c.pid, &wstatus, 0, nil) } return err } @@ -181,7 +197,7 @@ func (b *Builder) exec() (*os.File, error) { return OpenCurrentExec() } -// OpenCurrentExec opens current executable and dup it to memfd +// OpenCurrentExec opens current executable and dup and seal it to memfd func OpenCurrentExec() (*os.File, error) { self, err := os.Open(currentExec) if err != nil { @@ -189,7 +205,7 @@ func OpenCurrentExec() (*os.File, error) { } defer self.Close() - execFile, err := memfd.DupToMemfd("daemon", self) + execFile, err := memfd.DupToMemfd("init", self) if err != nil { return nil, fmt.Errorf("failed to create memfd: %v", err) } diff --git a/daemon/master_cmd.go b/container/host_cmd.go similarity index 52% rename from daemon/master_cmd.go rename to container/host_cmd.go index ae9e3b2..84da958 100644 --- a/daemon/master_cmd.go +++ b/container/host_cmd.go @@ -1,4 +1,4 @@ -package daemon +package container import ( "fmt" @@ -9,55 +9,55 @@ import ( ) // Ping send ping message to container -func (m *Master) Ping() error { - m.mu.Lock() - defer m.mu.Unlock() +func (c *container) Ping() error { + c.mu.Lock() + defer c.mu.Unlock() // avoid infinite wait (max 3s) const pingWait = 3 * time.Second - m.socket.SetDeadline(time.Now().Add(pingWait)) - defer m.socket.SetDeadline(time.Time{}) + c.socket.SetDeadline(time.Now().Add(pingWait)) + defer c.socket.SetDeadline(time.Time{}) // send ping - cmd := Cmd{ + cmd := cmd{ Cmd: cmdPing, } - if err := m.sendCmd(&cmd, nil); err != nil { + if err := c.sendCmd(&cmd, nil); err != nil { return fmt.Errorf("ping: %v", err) } // receive no error - return m.recvAckReply("ping") + return c.recvAckReply("ping") } // conf send configuration to container (used by builder only) -func (m *Master) conf(conf *containerConfig) error { - m.mu.Lock() - defer m.mu.Unlock() +func (c *container) conf(conf *containerConfig) error { + c.mu.Lock() + defer c.mu.Unlock() - cmd := Cmd{ + cmd := cmd{ Cmd: cmdConf, - ConfCmd: &ConfCmd{Conf: *conf}, + ConfCmd: &confCmd{Conf: *conf}, } - if err := m.sendCmd(&cmd, nil); err != nil { + if err := c.sendCmd(&cmd, nil); err != nil { return fmt.Errorf("conf: %v", err) } - return m.recvAckReply("conf") + return c.recvAckReply("conf") } // Open open files in container -func (m *Master) Open(p []OpenCmd) ([]*os.File, error) { - m.mu.Lock() - defer m.mu.Unlock() +func (c *container) Open(p []OpenCmd) ([]*os.File, error) { + c.mu.Lock() + defer c.mu.Unlock() // send copyin - cmd := Cmd{ + cmd := cmd{ Cmd: cmdOpen, OpenCmd: p, } - if err := m.sendCmd(&cmd, nil); err != nil { + if err := c.sendCmd(&cmd, nil); err != nil { return nil, fmt.Errorf("open: %v", err) } - reply, msg, err := m.recvReply() + reply, msg, err := c.recvReply() if err != nil { return nil, fmt.Errorf("open: %v", err) } @@ -82,36 +82,36 @@ func (m *Master) Open(p []OpenCmd) ([]*os.File, error) { } // Delete remove file from container -func (m *Master) Delete(p string) error { - m.mu.Lock() - defer m.mu.Unlock() +func (c *container) Delete(p string) error { + c.mu.Lock() + defer c.mu.Unlock() - cmd := Cmd{ + cmd := cmd{ Cmd: cmdDelete, - DeleteCmd: &DeleteCmd{Path: p}, + DeleteCmd: &deleteCmd{Path: p}, } - if err := m.sendCmd(&cmd, nil); err != nil { + if err := c.sendCmd(&cmd, nil); err != nil { return fmt.Errorf("delete: %v", err) } - return m.recvAckReply("delete") + return c.recvAckReply("delete") } // Reset remove all from /tmp and /w -func (m *Master) Reset() error { - m.mu.Lock() - defer m.mu.Unlock() +func (c *container) Reset() error { + c.mu.Lock() + defer c.mu.Unlock() - cmd := Cmd{ + cmd := cmd{ Cmd: cmdReset, } - if err := m.sendCmd(&cmd, nil); err != nil { + if err := c.sendCmd(&cmd, nil); err != nil { return fmt.Errorf("reset: %v", err) } - return m.recvAckReply("reset") + return c.recvAckReply("reset") } -func (m *Master) recvAckReply(name string) error { - reply, _, err := m.recvReply() +func (c *container) recvAckReply(name string) error { + reply, _, err := c.recvReply() if err != nil { return fmt.Errorf("%v: recvAck %v", name, err) } @@ -121,15 +121,15 @@ func (m *Master) recvAckReply(name string) error { return nil } -func (m *Master) recvReply() (*Reply, *unixsocket.Msg, error) { - reply := new(Reply) - msg, err := (*socket)(m.socket).RecvMsg(reply) +func (c *container) recvReply() (*reply, *unixsocket.Msg, error) { + reply := new(reply) + msg, err := (*socket)(c.socket).RecvMsg(reply) if err != nil { return nil, nil, err } return reply, msg, nil } -func (m *Master) sendCmd(cmd *Cmd, msg *unixsocket.Msg) error { - return (*socket)(m.socket).SendMsg(cmd, msg) +func (c *container) sendCmd(cmd *cmd, msg *unixsocket.Msg) error { + return (*socket)(c.socket).SendMsg(cmd, msg) } diff --git a/daemon/master_cmd_execve.go b/container/host_exec.go similarity index 77% rename from daemon/master_cmd_execve.go rename to container/host_exec.go index 84a2c64..7bebf97 100644 --- a/daemon/master_cmd_execve.go +++ b/container/host_exec.go @@ -1,4 +1,4 @@ -package daemon +package container import ( "context" @@ -16,20 +16,19 @@ type ExecveParam struct { Env []string Fds []uintptr - // fexecve fd + // fd parameter of fexecve ExecFile uintptr // POSIX Resource limit set by set rlimit RLimits []rlimit.RLimit - // SyncFunc called with pid before execve (e.g. add to cgroup) + // SyncFunc called with pid before execve (for adding the process to cgroups) SyncFunc func(pid int) error } -// Execve runs process inside container -// accepts context for cancelation -func (m *Master) Execve(c context.Context, param *ExecveParam) <-chan types.Result { - m.mu.Lock() +// Execve runs process inside container. It accepts context cancelation as time limit exceeded. +func (c *container) Execve(ctx context.Context, param ExecveParam) <-chan types.Result { + c.mu.Lock() sTime := time.Now() @@ -53,46 +52,46 @@ func (m *Master) Execve(c context.Context, param *ExecveParam) <-chan types.Resu msg := &unixsocket.Msg{ Fds: files, } - execCmd := &ExecCmd{ + execCmd := &execCmd{ Argv: param.Args, Env: param.Env, RLimits: param.RLimits, FdExec: param.ExecFile > 0, } - cmd := Cmd{ + cm := cmd{ Cmd: cmdExecve, ExecCmd: execCmd, } - if err := m.sendCmd(&cmd, msg); err != nil { - m.mu.Unlock() + if err := c.sendCmd(&cm, msg); err != nil { + c.mu.Unlock() return errResult("execve: sendCmd %v", err) } // sync function - reply, msg, err := m.recvReply() + reply, msg, err := c.recvReply() if err != nil { - m.mu.Unlock() + c.mu.Unlock() return errResult("execve: recvReply %v", err) } // if sync function did not involved if reply.Error != nil || msg == nil || msg.Cred == nil { // tell kill function to exit and sync - m.execveSyncKill() - m.mu.Unlock() + c.execveSyncKill() + c.mu.Unlock() return errResult("execve: no pid received or error %v", reply.Error) } if param.SyncFunc != nil { if err := param.SyncFunc(int(msg.Cred.Pid)); err != nil { // tell sync function to exit and recv error - m.execveSyncKill() + c.execveSyncKill() // tell kill function to exit and sync - m.execveSyncKill() - m.mu.Unlock() + c.execveSyncKill() + c.mu.Unlock() return errResult("execve: syncfunc failed %v", err) } } // send to syncFunc ack ok - if err := m.sendCmd(&Cmd{Cmd: cmdOk}, nil); err != nil { - m.mu.Unlock() + if err := c.sendCmd(&cmd{Cmd: cmdOk}, nil); err != nil { + c.mu.Unlock() return errResult("execve: ack failed %v", err) } @@ -102,12 +101,12 @@ func (m *Master) Execve(c context.Context, param *ExecveParam) <-chan types.Resu // Wait go func() { - reply2, _, err := m.recvReply() + reply2, _, err := c.recvReply() close(waitDone) // done signal (should recv after kill) - m.recvReply() + c.recvReply() // unlock after last read / write - m.mu.Unlock() + c.mu.Unlock() // handle potential error if err != nil { @@ -145,17 +144,17 @@ func (m *Master) Execve(c context.Context, param *ExecveParam) <-chan types.Resu // Kill (if wait is done, a kill message need to be send to collect zombies) go func() { select { - case <-c.Done(): + case <-ctx.Done(): case <-waitDone: } - m.sendCmd(&Cmd{Cmd: cmdKill}, nil) + c.sendCmd(&cmd{Cmd: cmdKill}, nil) }() return result } // execveSyncKill will send kill and recv reply -func (m *Master) execveSyncKill() { - m.sendCmd(&Cmd{Cmd: cmdKill}, nil) - m.recvReply() +func (c *container) execveSyncKill() { + c.sendCmd(&cmd{Cmd: cmdKill}, nil) + c.recvReply() } diff --git a/container/protocol.go b/container/protocol.go new file mode 100644 index 0000000..2c3047e --- /dev/null +++ b/container/protocol.go @@ -0,0 +1,74 @@ +package container + +import ( + "os" + "syscall" + "time" + + "github.com/criyle/go-sandbox/pkg/rlimit" + "github.com/criyle/go-sandbox/types" +) + +// cmd is the control message send into container +type cmd struct { + Cmd string // type of the cmd + + OpenCmd []OpenCmd // open argument + DeleteCmd *deleteCmd // delete argument + ExecCmd *execCmd // execve argument + ConfCmd *confCmd // to set configuration +} + +// OpenCmd correspond to a single open syscall +type OpenCmd struct { + Path string + Flag int + Perm os.FileMode +} + +// deleteCmd stores delete command +type deleteCmd struct { + Path string +} + +// execCmd stores execve parameter +type execCmd struct { + Argv []string // execve argv + Env []string // execve env + RLimits []rlimit.RLimit // execve posix rlimit + FdExec bool // if use fexecve (fd[0] as exec) +} + +// confCmd stores conf parameter +type confCmd struct { + Conf containerConfig +} + +// ContainerConfig set the container config +type containerConfig struct { + Cred bool +} + +// reply is the reply message send back to controller +type reply struct { + Error *errorReply // nil if no error + ExecReply *execReply +} + +// errorReply stores error returned back from container +type errorReply struct { + Msg string + Errno *syscall.Errno +} + +// execReply stores execve result +type execReply struct { + ExitStatus int // waitpid exit status + Status types.Status // return status + Time time.Duration // waitpid user CPU (ns) + Memory types.Size // waitpid user memory (byte) +} + +func (e *errorReply) Error() string { + return e.Msg +} diff --git a/daemon/socket.go b/container/socket.go similarity index 60% rename from daemon/socket.go rename to container/socket.go index 7b86740..4e8e006 100644 --- a/daemon/socket.go +++ b/container/socket.go @@ -1,19 +1,30 @@ -package daemon +package container import ( "bytes" "encoding/gob" "fmt" + "sync" "github.com/criyle/go-sandbox/pkg/unixsocket" ) +// 16k buffsize +const bufferSize = 16 << 10 + +var bufferPool = sync.Pool{ + New: func() interface{} { + return make([]byte, bufferSize) + }, +} + type socket unixsocket.Socket func (s *socket) RecvMsg(e interface{}) (*unixsocket.Msg, error) { soc := (*unixsocket.Socket)(s) - buff := GetBuffer() - defer PutBuffer(buff) + + buff := bufferPool.Get().([]byte) + defer bufferPool.Put(buff) n, msg, err := soc.RecvMsg(buff) if err != nil { @@ -21,24 +32,25 @@ func (s *socket) RecvMsg(e interface{}) (*unixsocket.Msg, error) { } if err := gob.NewDecoder(bytes.NewReader(buff[:n])).Decode(e); err != nil { - return nil, fmt.Errorf("RecvMsg: failed to decode %v", err) + return nil, fmt.Errorf("RecvMsg: failed to decode %w", err) } return msg, nil } func (s *socket) SendMsg(e interface{}, msg *unixsocket.Msg) error { soc := (*unixsocket.Socket)(s) - buf := GetBuffer() - defer PutBuffer(buf) + + buf := bufferPool.Get().([]byte) + defer bufferPool.Put(buf) // use buf pool to reduce allocation buff := bytes.NewBuffer(buf[:0]) if err := gob.NewEncoder(buff).Encode(e); err != nil { - return fmt.Errorf("SendMsg: failed to encode %v", err) + return fmt.Errorf("SendMsg: failed to encode %w", err) } if err := soc.SendMsg(buff.Bytes(), msg); err != nil { - return fmt.Errorf("SendMsg: failed to SendMsg %v", err) + return fmt.Errorf("SendMsg: failed to SendMsg %w", err) } return nil } diff --git a/daemon/utils.go b/container/utils.go similarity index 98% rename from daemon/utils.go rename to container/utils.go index 1fc38af..92502cb 100644 --- a/daemon/utils.go +++ b/container/utils.go @@ -1,4 +1,4 @@ -package daemon +package container import ( "os" diff --git a/daemon/container_init.go b/daemon/container_init.go deleted file mode 100644 index 082acc2..0000000 --- a/daemon/container_init.go +++ /dev/null @@ -1,176 +0,0 @@ -package daemon - -import ( - "fmt" - "os" - "runtime" - "syscall" - - "github.com/criyle/go-sandbox/pkg/unixsocket" -) - -type containerServer struct { - socket *unixsocket.Socket - containerConfig -} - -// ContainerConfig set the container config -type containerConfig struct { - Cred bool -} - -// Init is called for container init process -// it will check if pid == 1, otherwise it is noop -// Init will do infinite loop on socket commands, -// and exits when at socket close, use it in init function -func Init() (err error) { - // noop if self is not container init process - // Notice: docker init is also 1, additional check for args[1] == init - if os.Getpid() != 1 || len(os.Args) != 2 || os.Args[1] != initArg { - return nil - } - - // exit process (with whole container) upon exit this function - // possible reason: - // 1. socket broken (parent exit) - // 2. panic - // 3. undefined cmd (possible race condition) - defer func() { - if err2 := recover(); err2 != nil { - fmt.Fprintf(os.Stderr, "container_panic: %v\n", err) - os.Exit(1) - } - if err != nil { - fmt.Fprintf(os.Stderr, "container_exit: %v\n", err) - os.Exit(1) - } - fmt.Fprintf(os.Stderr, "container_exit\n") - os.Exit(0) - }() - - // limit container resource usage - runtime.GOMAXPROCS(containerMaxProc) - - // new_master shared the socket at fd 3 (marked close_exec) - const defaultFd = 3 - soc, err := unixsocket.NewSocket(defaultFd) - if err != nil { - return fmt.Errorf("container_init: failed to new socket(%v)", err) - } - - // serve forever - cs := &containerServer{socket: soc} - return cs.serve() -} - -func (c *containerServer) serve() error { - for { - cmd, msg, err := c.recvCmd() - if err != nil { - return fmt.Errorf("serve: %v", err) - } - if err := c.handleCmd(cmd, msg); err != nil { - return fmt.Errorf("serve: failed to execute cmd %v", err) - } - } -} - -func (c *containerServer) handleCmd(cmd *Cmd, msg *unixsocket.Msg) error { - switch cmd.Cmd { - case cmdPing: - return c.handlePing() - - case cmdConf: - return c.handleConf(cmd.ConfCmd) - - case cmdOpen: - return c.handleOpen(cmd.OpenCmd) - - case cmdDelete: - return c.handleDelete(cmd.DeleteCmd) - - case cmdReset: - return c.handleReset() - - case cmdExecve: - return c.handleExecve(cmd.ExecCmd, msg) - } - return fmt.Errorf("unknown command: %v", cmd.Cmd) -} - -func (c *containerServer) handlePing() error { - return c.sendReply(&Reply{}, nil) -} - -func (c *containerServer) handleConf(conf *ConfCmd) error { - if conf != nil { - c.containerConfig = conf.Conf - } - return c.sendReply(&Reply{}, nil) -} - -func (c *containerServer) handleOpen(open []OpenCmd) error { - if len(open) == 0 { - return c.sendErrorReply("open: no open parameter received") - } - - // open files - fds := make([]int, 0, len(open)) - for _, o := range open { - outFile, err := os.OpenFile(o.Path, o.Flag, o.Perm) - if err != nil { - return c.sendErrorReply("open: %v", err) - } - defer outFile.Close() - fds = append(fds, int(outFile.Fd())) - } - - return c.sendReply(&Reply{}, &unixsocket.Msg{Fds: fds}) -} - -func (c *containerServer) handleDelete(delete *DeleteCmd) error { - if delete == nil { - return c.sendErrorReply("delete: no parameter provided") - } - if err := os.Remove(delete.Path); err != nil { - return c.sendErrorReply("delete: %v", err) - } - return c.sendReply(&Reply{}, nil) -} - -func (c *containerServer) handleReset() error { - if err := removeContents("/tmp"); err != nil { - return c.sendErrorReply("reset: /tmp %v", err) - } - if err := removeContents("/w"); err != nil { - return c.sendErrorReply("reset: /w %v", err) - } - return c.sendReply(&Reply{}, nil) -} - -func (c *containerServer) recvCmd() (*Cmd, *unixsocket.Msg, error) { - cmd := new(Cmd) - msg, err := (*socket)(c.socket).RecvMsg(cmd) - if err != nil { - return nil, nil, err - } - return cmd, msg, nil -} - -func (c *containerServer) sendReply(reply *Reply, msg *unixsocket.Msg) error { - return (*socket)(c.socket).SendMsg(reply, msg) -} - -// sendErrorReply sends error reply -func (c *containerServer) sendErrorReply(ft string, v ...interface{}) error { - errorReply := &ErrorReply{ - Msg: fmt.Sprintf(ft, v...), - } - // store errno - if len(v) == 1 { - if errno, ok := v[0].(syscall.Errno); ok { - errorReply.Errno = &errno - } - } - return c.sendReply(&Reply{Error: errorReply}, nil) -} diff --git a/daemon/deamon.go b/daemon/deamon.go deleted file mode 100644 index da58a5f..0000000 --- a/daemon/deamon.go +++ /dev/null @@ -1,102 +0,0 @@ -// Package daemon provides pre-forked container to reduce the container -// create / destroy costs (about 160ms). It creates daemon within unshared -// container and communicate with original process using unix socket with -// oob for fd / pid and struct are encoded in gob format. -package daemon - -/* -Container / Master Communication Protocol (single thread): - -- ping (alive check): - - reply: pong -- conf (set configuration): - - reply pong -- open (open files in given mode inside container): - - send: []OpenCmd - - reply: "success", file fds / "error" -- delete (unlink file / rmdir dir inside container): - - send: path - - reply: "finished" / "error" -- reset (clean up container for later use (clear workdir / tmp)): - - send: - - reply: "success" -- execve: (execute file inside container): - - send: argv, env, rLimits, fds - - reply: - - success: "success", pid - - failed: "failed" - - send (success): "init_finished" (as cmd) - - reply: "finished" / send: "kill" (as cmd) - - send: "kill" (as cmd) / reply: "finished" - - reply: - -Any socket related error will cause the daemon exit (with all process inside container) -*/ - -import ( - "os" - "syscall" - "time" - - "github.com/criyle/go-sandbox/pkg/rlimit" - "github.com/criyle/go-sandbox/types" -) - -// Cmd is the control message send into daemon -type Cmd struct { - Cmd string // type of the cmd - - OpenCmd []OpenCmd // open argument - DeleteCmd *DeleteCmd // delete argument - ExecCmd *ExecCmd // execve argument - ConfCmd *ConfCmd // to set configuration -} - -// OpenCmd correspond to a single open syscall -type OpenCmd struct { - Path string - Flag int - Perm os.FileMode -} - -// DeleteCmd stores delete command -type DeleteCmd struct { - Path string -} - -// ExecCmd stores execve parameter -type ExecCmd struct { - Argv []string // execve argv - Env []string // execve env - RLimits []rlimit.RLimit // execve posix rlimit - FdExec bool // if use fexecve (fd[0] as exec) -} - -// ConfCmd stores conf parameter -type ConfCmd struct { - Conf containerConfig -} - -// Reply is the reply message send back to controller -type Reply struct { - Error *ErrorReply // nil if no error - ExecReply *ExecReply -} - -// ErrorReply stores error returned back from container -type ErrorReply struct { - Msg string - Errno *syscall.Errno -} - -// ExecReply stores execve result -type ExecReply struct { - ExitStatus int // waitpid exit status - Status types.Status // return status - Time time.Duration // waitpid user CPU (ns) - Memory types.Size // waitpid user memory (byte) -} - -func (e *ErrorReply) Error() string { - return e.Msg -} diff --git a/daemon/pool.go b/daemon/pool.go deleted file mode 100644 index 0f7b19e..0000000 --- a/daemon/pool.go +++ /dev/null @@ -1,24 +0,0 @@ -package daemon - -import ( - "sync" -) - -// 16k buffsize -const bufferSize = 16384 - -var bufferPool = sync.Pool{ - New: func() interface{} { - return make([]byte, bufferSize) - }, -} - -// GetBuffer get buffer from pool -func GetBuffer() []byte { - return bufferPool.Get().([]byte) -} - -// PutBuffer return buffer to the pool -func PutBuffer(x []byte) { - bufferPool.Put(x) -}