diff --git a/README.md b/README.md
index 31f8f16..73c6824 100644
--- a/README.md
+++ b/README.md
@@ -44,6 +44,33 @@ Default file access syscall check:
1. Pre-fork container daemons to run programs inside
2. Unix socket to pass fd inside / outside
+Container / Master Communication Protocol (single thread):
+
+- ping (alive check):
+ - reply: pong
+- conf (set configuration):
+ - reply pong
+- open (open files in given mode inside container):
+ - send: []OpenCmd
+ - reply: "success", file fds / "error"
+- delete (unlink file / rmdir dir inside container):
+ - send: path
+ - reply: "finished" / "error"
+- reset (clean up container for later use (clear workdir / tmp)):
+ - send:
+ - reply: "success"
+- execve: (execute file inside container):
+ - send: argv, env, rLimits, fds
+ - reply:
+ - success: "success", pid
+ - failed: "failed"
+ - send (success): "init_finished" (as cmd)
+ - reply: "finished" / send: "kill" (as cmd)
+ - send: "kill" (as cmd) / reply: "finished"
+ - reply:
+
+Any socket related error will cause the daemon exit (with all process inside container)
+
## Packages (/pkg)
- seccomp: provides seccomp type definition
diff --git a/daemon/consts.go b/daemon/consts.go
index f779bb3..c7166a3 100644
--- a/daemon/consts.go
+++ b/daemon/consts.go
@@ -20,4 +20,6 @@ const (
containerName = "daemon"
containerWD = "/w"
+
+ containerMaxProc = 1
)
diff --git a/daemon/container_execve.go b/daemon/container_execve.go
index f240d66..d7fd0e9 100644
--- a/daemon/container_execve.go
+++ b/daemon/container_execve.go
@@ -9,12 +9,15 @@ import (
"github.com/criyle/go-sandbox/types"
)
-func (c *containerServer) handleExecve(cmd *Cmd, msg *unixsocket.Msg) error {
+func (c *containerServer) handleExecve(cmd *ExecCmd, msg *unixsocket.Msg) error {
var (
files []uintptr
execFile uintptr
cred *syscall.Credential
)
+ if cmd == nil {
+ return c.sendErrorReply("execve: no parameter provided")
+ }
if msg != nil {
files = intSliceToUintptr(msg.Fds)
// don't leak fds to child
@@ -118,10 +121,12 @@ func (c *containerServer) handleExecve(cmd *Cmd, msg *unixsocket.Msg) error {
case wstatus.Exited():
exitStatus := wstatus.ExitStatus()
c.sendReply(&Reply{
- Status: status,
- ExitStatus: exitStatus,
- UserTime: userTime,
- UserMem: userMem,
+ ExecReply: &ExecReply{
+ Status: status,
+ ExitStatus: exitStatus,
+ UserTime: userTime,
+ UserMem: userMem,
+ },
}, nil)
case wstatus.Signaled():
@@ -137,14 +142,18 @@ func (c *containerServer) handleExecve(cmd *Cmd, msg *unixsocket.Msg) error {
status = types.StatusRE
}
c.sendReply(&Reply{
- Status: status,
- UserTime: userTime,
- UserMem: userMem,
+ ExecReply: &ExecReply{
+ Status: status,
+ UserTime: userTime,
+ UserMem: userMem,
+ },
}, nil)
+
default:
c.sendErrorReply("execve: unknown status %v", wstatus)
}
}
+
// wait for kill msg and reply done for finish
<-killDone
return c.sendReply(&Reply{}, nil)
diff --git a/daemon/container_init.go b/daemon/container_init.go
index ae73818..082acc2 100644
--- a/daemon/container_init.go
+++ b/daemon/container_init.go
@@ -2,8 +2,9 @@ package daemon
import (
"fmt"
- "io"
"os"
+ "runtime"
+ "syscall"
"github.com/criyle/go-sandbox/pkg/unixsocket"
)
@@ -13,6 +14,7 @@ type containerServer struct {
containerConfig
}
+// ContainerConfig set the container config
type containerConfig struct {
Cred bool
}
@@ -46,6 +48,9 @@ func Init() (err error) {
os.Exit(0)
}()
+ // limit container resource usage
+ runtime.GOMAXPROCS(containerMaxProc)
+
// new_master shared the socket at fd 3 (marked close_exec)
const defaultFd = 3
soc, err := unixsocket.NewSocket(defaultFd)
@@ -76,22 +81,19 @@ func (c *containerServer) handleCmd(cmd *Cmd, msg *unixsocket.Msg) error {
return c.handlePing()
case cmdConf:
- return c.handleConf(cmd)
-
- case cmdCopyIn:
- return c.handleCopyIn(cmd, msg)
+ return c.handleConf(cmd.ConfCmd)
case cmdOpen:
- return c.handleOpen(cmd)
+ return c.handleOpen(cmd.OpenCmd)
case cmdDelete:
- return c.handleDelete(cmd)
+ return c.handleDelete(cmd.DeleteCmd)
case cmdReset:
return c.handleReset()
case cmdExecve:
- return c.handleExecve(cmd, msg)
+ return c.handleExecve(cmd.ExecCmd, msg)
}
return fmt.Errorf("unknown command: %v", cmd.Cmd)
}
@@ -100,51 +102,37 @@ func (c *containerServer) handlePing() error {
return c.sendReply(&Reply{}, nil)
}
-func (c *containerServer) handleConf(cmd *Cmd) error {
- if cmd.Conf != nil {
- c.containerConfig = *cmd.Conf
+func (c *containerServer) handleConf(conf *ConfCmd) error {
+ if conf != nil {
+ c.containerConfig = conf.Conf
}
return c.sendReply(&Reply{}, nil)
}
-func (c *containerServer) handleCopyIn(cmd *Cmd, msg *unixsocket.Msg) error {
- if len(msg.Fds) != 1 {
- closeFds(msg.Fds)
- return c.sendErrorReply("copyin: unexpected number of fds(%d)", len(msg.Fds))
+func (c *containerServer) handleOpen(open []OpenCmd) error {
+ if len(open) == 0 {
+ return c.sendErrorReply("open: no open parameter received")
}
- inf := os.NewFile(uintptr(msg.Fds[0]), cmd.Path)
- if inf == nil {
- return c.sendErrorReply("copyin: newfile failed %v", msg.Fds[0])
- }
- defer inf.Close()
- // have 0777 permission to be able copy in executables
- outf, err := os.OpenFile(cmd.Path, os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0777)
- if err != nil {
- return c.sendErrorReply("copyin: open write file %v", err)
+ // open files
+ fds := make([]int, 0, len(open))
+ for _, o := range open {
+ outFile, err := os.OpenFile(o.Path, o.Flag, o.Perm)
+ if err != nil {
+ return c.sendErrorReply("open: %v", err)
+ }
+ defer outFile.Close()
+ fds = append(fds, int(outFile.Fd()))
}
- defer outf.Close()
- if _, err = io.Copy(outf, inf); err != nil {
- return c.sendErrorReply("copyin: io.copy %v", err)
- }
- return c.sendReply(&Reply{}, nil)
+ return c.sendReply(&Reply{}, &unixsocket.Msg{Fds: fds})
}
-func (c *containerServer) handleOpen(cmd *Cmd) error {
- outf, err := os.Open(cmd.Path)
- if err != nil {
- return c.sendErrorReply("open: %v", err)
+func (c *containerServer) handleDelete(delete *DeleteCmd) error {
+ if delete == nil {
+ return c.sendErrorReply("delete: no parameter provided")
}
- defer outf.Close()
-
- return c.sendReply(&Reply{}, &unixsocket.Msg{
- Fds: []int{int(outf.Fd())},
- })
-}
-
-func (c *containerServer) handleDelete(cmd *Cmd) error {
- if err := os.Remove(cmd.Path); err != nil {
+ if err := os.Remove(delete.Path); err != nil {
return c.sendErrorReply("delete: %v", err)
}
return c.sendReply(&Reply{}, nil)
@@ -175,5 +163,14 @@ func (c *containerServer) sendReply(reply *Reply, msg *unixsocket.Msg) error {
// sendErrorReply sends error reply
func (c *containerServer) sendErrorReply(ft string, v ...interface{}) error {
- return c.sendReply(&Reply{Error: fmt.Sprintf(ft, v...)}, nil)
+ errorReply := &ErrorReply{
+ Msg: fmt.Sprintf(ft, v...),
+ }
+ // store errno
+ if len(v) == 1 {
+ if errno, ok := v[0].(syscall.Errno); ok {
+ errorReply.Errno = &errno
+ }
+ }
+ return c.sendReply(&Reply{Error: errorReply}, nil)
}
diff --git a/daemon/deamon.go b/daemon/deamon.go
index aaa391d..06edfaf 100644
--- a/daemon/deamon.go
+++ b/daemon/deamon.go
@@ -5,57 +5,97 @@
package daemon
/*
-Protocol between client and daemon (not thread safe):
+Container / Master Communication Protocol (single thread):
+
- ping (alive check):
- reply: pong
+ - reply: pong
- conf (set configuration):
- reply pong
-- copyin (copy file into container):
- send: path,
- reply: "finished" (after copy finished) / "error"
-- open (open file in read-only mode inside container):
- send: path
- reply: "success", / "error"
+ - reply pong
+- open (open files in given mode inside container):
+ - send: []OpenCmd
+ - reply: "success", file fds / "error"
- delete (unlink file / rmdir dir inside container):
- send: path
- reply: "finished" / "error"
+ - send: path
+ - reply: "finished" / "error"
- reset (clean up container for later use (clear workdir / tmp)):
- send:
- reply: "success"
+ - send:
+ - reply: "success"
- execve: (execute file inside container):
- send: argv, env, rLimits,
- reply:
- - success: "success",
+ - send: argv, env, rLimits, fds
+ - reply:
+ - success: "success", pid
- failed: "failed"
- send (success): "init_finished" (as cmd)
+ - send (success): "init_finished" (as cmd)
- reply: "finished" / send: "kill" (as cmd)
- send: "kill" (as cmd) / reply: "finished"
- reply:
+ - reply:
Any socket related error will cause the daemon exit (with all process inside container)
*/
import (
+ "os"
+ "syscall"
+
"github.com/criyle/go-sandbox/pkg/rlimit"
"github.com/criyle/go-sandbox/types"
)
// Cmd is the control message send into daemon
type Cmd struct {
- Cmd string // type of the cmd
- Path string // path (copyin / open)
- Argv []string // execve argv
- Env []string // execve env
- RLimits []rlimit.RLimit // execve posix rlimit
- FdExec bool // if use fexecve (fd[0] as exec)
- Conf *containerConfig // to set configuration
+ Cmd string // type of the cmd
+
+ OpenCmd []OpenCmd // open argument
+ DeleteCmd *DeleteCmd // delete argument
+ ExecCmd *ExecCmd // execve argument
+ ConfCmd *ConfCmd // to set configuration
+}
+
+// OpenCmd correspond to a single open syscall
+type OpenCmd struct {
+ Path string
+ Flag int
+ Perm os.FileMode
+}
+
+// DeleteCmd stores delete command
+type DeleteCmd struct {
+ Path string
+}
+
+// ExecCmd stores execve parameter
+type ExecCmd struct {
+ Argv []string // execve argv
+ Env []string // execve env
+ RLimits []rlimit.RLimit // execve posix rlimit
+ FdExec bool // if use fexecve (fd[0] as exec)
+}
+
+// ConfCmd stores conf parameter
+type ConfCmd struct {
+ Conf containerConfig
}
// Reply is the reply message send back to controller
type Reply struct {
- Error string // empty if no error
+ Error *ErrorReply // nil if no error
+ ExecReply *ExecReply
+}
+
+// ErrorReply stores error returned back from container
+type ErrorReply struct {
+ Msg string
+ Errno *syscall.Errno
+}
+
+// ExecReply stores execve result
+type ExecReply struct {
ExitStatus int // waitpid exit status
Status types.Status // return status
UserTime uint64 // waitpid user CPU (ms)
UserMem uint64 // waitpid user memory (kb)
}
+
+func (e *ErrorReply) Error() string {
+ return e.Msg
+}
diff --git a/daemon/master_cmd.go b/daemon/master_cmd.go
index 7b9bac8..ae9e3b2 100644
--- a/daemon/master_cmd.go
+++ b/daemon/master_cmd.go
@@ -35,8 +35,8 @@ func (m *Master) conf(conf *containerConfig) error {
defer m.mu.Unlock()
cmd := Cmd{
- Cmd: cmdConf,
- Conf: conf,
+ Cmd: cmdConf,
+ ConfCmd: &ConfCmd{Conf: *conf},
}
if err := m.sendCmd(&cmd, nil); err != nil {
return fmt.Errorf("conf: %v", err)
@@ -44,34 +44,15 @@ func (m *Master) conf(conf *containerConfig) error {
return m.recvAckReply("conf")
}
-// CopyIn copies file to container
-func (m *Master) CopyIn(f *os.File, p string) error {
+// Open open files in container
+func (m *Master) Open(p []OpenCmd) ([]*os.File, error) {
m.mu.Lock()
defer m.mu.Unlock()
// send copyin
cmd := Cmd{
- Cmd: cmdCopyIn,
- Path: p,
- }
- msg := unixsocket.Msg{
- Fds: []int{int(f.Fd())},
- }
- if err := m.sendCmd(&cmd, &msg); err != nil {
- return fmt.Errorf("copyin: %v", err)
- }
- return m.recvAckReply("copyin")
-}
-
-// Open open file in container
-func (m *Master) Open(p string) (*os.File, error) {
- m.mu.Lock()
- defer m.mu.Unlock()
-
- // send copyin
- cmd := Cmd{
- Cmd: cmdOpen,
- Path: p,
+ Cmd: cmdOpen,
+ OpenCmd: p,
}
if err := m.sendCmd(&cmd, nil); err != nil {
return nil, fmt.Errorf("open: %v", err)
@@ -80,19 +61,24 @@ func (m *Master) Open(p string) (*os.File, error) {
if err != nil {
return nil, fmt.Errorf("open: %v", err)
}
- if reply.Error != "" {
+ if reply.Error != nil {
return nil, fmt.Errorf("open: %v", reply.Error)
}
- if len(msg.Fds) != 1 {
+ if len(msg.Fds) != len(p) {
closeFds(msg.Fds)
- return nil, fmt.Errorf("open: unexpected number of fd %v", len(msg.Fds))
+ return nil, fmt.Errorf("open: unexpected number of fd %v / %v", len(msg.Fds), len(p))
}
- f := os.NewFile(uintptr(msg.Fds[0]), p)
- if f == nil {
- closeFds(msg.Fds)
- return nil, fmt.Errorf("open: failed %v", msg.Fds[0])
+
+ ret := make([]*os.File, 0, len(p))
+ for i, fd := range msg.Fds {
+ f := os.NewFile(uintptr(fd), p[i].Path)
+ if f == nil {
+ closeFds(msg.Fds)
+ return nil, fmt.Errorf("open: failed %v", msg.Fds[0])
+ }
+ ret = append(ret, f)
}
- return f, nil
+ return ret, nil
}
// Delete remove file from container
@@ -101,8 +87,8 @@ func (m *Master) Delete(p string) error {
defer m.mu.Unlock()
cmd := Cmd{
- Cmd: cmdDelete,
- Path: p,
+ Cmd: cmdDelete,
+ DeleteCmd: &DeleteCmd{Path: p},
}
if err := m.sendCmd(&cmd, nil); err != nil {
return fmt.Errorf("delete: %v", err)
@@ -129,7 +115,7 @@ func (m *Master) recvAckReply(name string) error {
if err != nil {
return fmt.Errorf("%v: recvAck %v", name, err)
}
- if reply.Error != "" {
+ if reply.Error != nil {
return fmt.Errorf("%v: container error %v", name, reply.Error)
}
return nil
diff --git a/daemon/master_cmd_execve.go b/daemon/master_cmd_execve.go
index bbba357..c138da4 100644
--- a/daemon/master_cmd_execve.go
+++ b/daemon/master_cmd_execve.go
@@ -41,13 +41,16 @@ func (m *Master) Execve(done <-chan struct{}, param *ExecveParam) (<-chan types.
msg := &unixsocket.Msg{
Fds: files,
}
- cmd := Cmd{
- Cmd: cmdExecve,
+ execCmd := &ExecCmd{
Argv: param.Args,
Env: param.Env,
RLimits: param.RLimits,
FdExec: param.ExecFile > 0,
}
+ cmd := Cmd{
+ Cmd: cmdExecve,
+ ExecCmd: execCmd,
+ }
if err := m.sendCmd(&cmd, msg); err != nil {
m.mu.Unlock()
return nil, fmt.Errorf("execve: sendCmd %v", err)
@@ -59,7 +62,7 @@ func (m *Master) Execve(done <-chan struct{}, param *ExecveParam) (<-chan types.
return nil, fmt.Errorf("execve: recvReply %v", err)
}
// if sync function did not involved
- if reply.Error != "" || msg == nil || msg.Cred == nil {
+ if reply.Error != nil || msg == nil || msg.Cred == nil {
// tell kill function to exit and sync
m.execveSyncKill()
m.mu.Unlock()
@@ -95,6 +98,7 @@ func (m *Master) Execve(done <-chan struct{}, param *ExecveParam) (<-chan types.
m.recvReply()
// unlock after last read / write
m.mu.Unlock()
+
// handle potential error
if err != nil {
result <- types.Result{
@@ -103,17 +107,27 @@ func (m *Master) Execve(done <-chan struct{}, param *ExecveParam) (<-chan types.
}
return
}
- // emit result after all communication finish
- status := reply2.Status
- if reply2.Error != "" {
- status = types.StatusFatal
+ if reply2.ExecReply == nil {
+ result <- types.Result{
+ Status: types.StatusFatal,
+ Error: "execve: no reply received",
+ }
+ return
}
+ // emit result after all communication finish
+ status := reply2.ExecReply.Status
+ errMsg := ""
+ if reply2.Error != nil {
+ status = types.StatusFatal
+ errMsg = reply2.Error.Error()
+ }
+
result <- types.Result{
Status: status,
- ExitStatus: reply2.ExitStatus,
- UserTime: reply2.UserTime,
- UserMem: reply2.UserMem,
- Error: reply2.Error,
+ ExitStatus: reply2.ExecReply.ExitStatus,
+ UserTime: reply2.ExecReply.UserTime,
+ UserMem: reply2.ExecReply.UserMem,
+ Error: errMsg,
SetUpTime: mTime.Sub(sTime),
RunningTime: time.Since(mTime),
}