use open syscall to copyin files

This commit is contained in:
criyle 2019-12-25 19:51:28 +08:00
parent 1cf901bbc2
commit 6b97e1f663
7 changed files with 199 additions and 124 deletions

View File

@ -44,6 +44,33 @@ Default file access syscall check:
1. Pre-fork container daemons to run programs inside 1. Pre-fork container daemons to run programs inside
2. Unix socket to pass fd inside / outside 2. Unix socket to pass fd inside / outside
Container / Master Communication Protocol (single thread):
- ping (alive check):
- reply: pong
- conf (set configuration):
- reply pong
- open (open files in given mode inside container):
- send: []OpenCmd
- reply: "success", file fds / "error"
- delete (unlink file / rmdir dir inside container):
- send: path
- reply: "finished" / "error"
- reset (clean up container for later use (clear workdir / tmp)):
- send:
- reply: "success"
- execve: (execute file inside container):
- send: argv, env, rLimits, fds
- reply:
- success: "success", pid
- failed: "failed"
- send (success): "init_finished" (as cmd)
- reply: "finished" / send: "kill" (as cmd)
- send: "kill" (as cmd) / reply: "finished"
- reply:
Any socket related error will cause the daemon exit (with all process inside container)
## Packages (/pkg) ## Packages (/pkg)
- seccomp: provides seccomp type definition - seccomp: provides seccomp type definition

View File

@ -20,4 +20,6 @@ const (
containerName = "daemon" containerName = "daemon"
containerWD = "/w" containerWD = "/w"
containerMaxProc = 1
) )

View File

@ -9,12 +9,15 @@ import (
"github.com/criyle/go-sandbox/types" "github.com/criyle/go-sandbox/types"
) )
func (c *containerServer) handleExecve(cmd *Cmd, msg *unixsocket.Msg) error { func (c *containerServer) handleExecve(cmd *ExecCmd, msg *unixsocket.Msg) error {
var ( var (
files []uintptr files []uintptr
execFile uintptr execFile uintptr
cred *syscall.Credential cred *syscall.Credential
) )
if cmd == nil {
return c.sendErrorReply("execve: no parameter provided")
}
if msg != nil { if msg != nil {
files = intSliceToUintptr(msg.Fds) files = intSliceToUintptr(msg.Fds)
// don't leak fds to child // don't leak fds to child
@ -118,10 +121,12 @@ func (c *containerServer) handleExecve(cmd *Cmd, msg *unixsocket.Msg) error {
case wstatus.Exited(): case wstatus.Exited():
exitStatus := wstatus.ExitStatus() exitStatus := wstatus.ExitStatus()
c.sendReply(&Reply{ c.sendReply(&Reply{
Status: status, ExecReply: &ExecReply{
ExitStatus: exitStatus, Status: status,
UserTime: userTime, ExitStatus: exitStatus,
UserMem: userMem, UserTime: userTime,
UserMem: userMem,
},
}, nil) }, nil)
case wstatus.Signaled(): case wstatus.Signaled():
@ -137,14 +142,18 @@ func (c *containerServer) handleExecve(cmd *Cmd, msg *unixsocket.Msg) error {
status = types.StatusRE status = types.StatusRE
} }
c.sendReply(&Reply{ c.sendReply(&Reply{
Status: status, ExecReply: &ExecReply{
UserTime: userTime, Status: status,
UserMem: userMem, UserTime: userTime,
UserMem: userMem,
},
}, nil) }, nil)
default: default:
c.sendErrorReply("execve: unknown status %v", wstatus) c.sendErrorReply("execve: unknown status %v", wstatus)
} }
} }
// wait for kill msg and reply done for finish // wait for kill msg and reply done for finish
<-killDone <-killDone
return c.sendReply(&Reply{}, nil) return c.sendReply(&Reply{}, nil)

View File

@ -2,8 +2,9 @@ package daemon
import ( import (
"fmt" "fmt"
"io"
"os" "os"
"runtime"
"syscall"
"github.com/criyle/go-sandbox/pkg/unixsocket" "github.com/criyle/go-sandbox/pkg/unixsocket"
) )
@ -13,6 +14,7 @@ type containerServer struct {
containerConfig containerConfig
} }
// ContainerConfig set the container config
type containerConfig struct { type containerConfig struct {
Cred bool Cred bool
} }
@ -46,6 +48,9 @@ func Init() (err error) {
os.Exit(0) os.Exit(0)
}() }()
// limit container resource usage
runtime.GOMAXPROCS(containerMaxProc)
// new_master shared the socket at fd 3 (marked close_exec) // new_master shared the socket at fd 3 (marked close_exec)
const defaultFd = 3 const defaultFd = 3
soc, err := unixsocket.NewSocket(defaultFd) soc, err := unixsocket.NewSocket(defaultFd)
@ -76,22 +81,19 @@ func (c *containerServer) handleCmd(cmd *Cmd, msg *unixsocket.Msg) error {
return c.handlePing() return c.handlePing()
case cmdConf: case cmdConf:
return c.handleConf(cmd) return c.handleConf(cmd.ConfCmd)
case cmdCopyIn:
return c.handleCopyIn(cmd, msg)
case cmdOpen: case cmdOpen:
return c.handleOpen(cmd) return c.handleOpen(cmd.OpenCmd)
case cmdDelete: case cmdDelete:
return c.handleDelete(cmd) return c.handleDelete(cmd.DeleteCmd)
case cmdReset: case cmdReset:
return c.handleReset() return c.handleReset()
case cmdExecve: case cmdExecve:
return c.handleExecve(cmd, msg) return c.handleExecve(cmd.ExecCmd, msg)
} }
return fmt.Errorf("unknown command: %v", cmd.Cmd) return fmt.Errorf("unknown command: %v", cmd.Cmd)
} }
@ -100,51 +102,37 @@ func (c *containerServer) handlePing() error {
return c.sendReply(&Reply{}, nil) return c.sendReply(&Reply{}, nil)
} }
func (c *containerServer) handleConf(cmd *Cmd) error { func (c *containerServer) handleConf(conf *ConfCmd) error {
if cmd.Conf != nil { if conf != nil {
c.containerConfig = *cmd.Conf c.containerConfig = conf.Conf
} }
return c.sendReply(&Reply{}, nil) return c.sendReply(&Reply{}, nil)
} }
func (c *containerServer) handleCopyIn(cmd *Cmd, msg *unixsocket.Msg) error { func (c *containerServer) handleOpen(open []OpenCmd) error {
if len(msg.Fds) != 1 { if len(open) == 0 {
closeFds(msg.Fds) return c.sendErrorReply("open: no open parameter received")
return c.sendErrorReply("copyin: unexpected number of fds(%d)", len(msg.Fds))
} }
inf := os.NewFile(uintptr(msg.Fds[0]), cmd.Path)
if inf == nil {
return c.sendErrorReply("copyin: newfile failed %v", msg.Fds[0])
}
defer inf.Close()
// have 0777 permission to be able copy in executables // open files
outf, err := os.OpenFile(cmd.Path, os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0777) fds := make([]int, 0, len(open))
if err != nil { for _, o := range open {
return c.sendErrorReply("copyin: open write file %v", err) outFile, err := os.OpenFile(o.Path, o.Flag, o.Perm)
if err != nil {
return c.sendErrorReply("open: %v", err)
}
defer outFile.Close()
fds = append(fds, int(outFile.Fd()))
} }
defer outf.Close()
if _, err = io.Copy(outf, inf); err != nil { return c.sendReply(&Reply{}, &unixsocket.Msg{Fds: fds})
return c.sendErrorReply("copyin: io.copy %v", err)
}
return c.sendReply(&Reply{}, nil)
} }
func (c *containerServer) handleOpen(cmd *Cmd) error { func (c *containerServer) handleDelete(delete *DeleteCmd) error {
outf, err := os.Open(cmd.Path) if delete == nil {
if err != nil { return c.sendErrorReply("delete: no parameter provided")
return c.sendErrorReply("open: %v", err)
} }
defer outf.Close() if err := os.Remove(delete.Path); err != nil {
return c.sendReply(&Reply{}, &unixsocket.Msg{
Fds: []int{int(outf.Fd())},
})
}
func (c *containerServer) handleDelete(cmd *Cmd) error {
if err := os.Remove(cmd.Path); err != nil {
return c.sendErrorReply("delete: %v", err) return c.sendErrorReply("delete: %v", err)
} }
return c.sendReply(&Reply{}, nil) return c.sendReply(&Reply{}, nil)
@ -175,5 +163,14 @@ func (c *containerServer) sendReply(reply *Reply, msg *unixsocket.Msg) error {
// sendErrorReply sends error reply // sendErrorReply sends error reply
func (c *containerServer) sendErrorReply(ft string, v ...interface{}) error { func (c *containerServer) sendErrorReply(ft string, v ...interface{}) error {
return c.sendReply(&Reply{Error: fmt.Sprintf(ft, v...)}, nil) errorReply := &ErrorReply{
Msg: fmt.Sprintf(ft, v...),
}
// store errno
if len(v) == 1 {
if errno, ok := v[0].(syscall.Errno); ok {
errorReply.Errno = &errno
}
}
return c.sendReply(&Reply{Error: errorReply}, nil)
} }

View File

@ -5,57 +5,97 @@
package daemon package daemon
/* /*
Protocol between client and daemon (not thread safe): Container / Master Communication Protocol (single thread):
- ping (alive check): - ping (alive check):
reply: pong - reply: pong
- conf (set configuration): - conf (set configuration):
reply pong - reply pong
- copyin (copy file into container): - open (open files in given mode inside container):
send: path, <input fd> - send: []OpenCmd
reply: "finished" (after copy finished) / "error" - reply: "success", file fds / "error"
- open (open file in read-only mode inside container):
send: path
reply: "success", <file fd> / "error"
- delete (unlink file / rmdir dir inside container): - delete (unlink file / rmdir dir inside container):
send: path - send: path
reply: "finished" / "error" - reply: "finished" / "error"
- reset (clean up container for later use (clear workdir / tmp)): - reset (clean up container for later use (clear workdir / tmp)):
send: - send:
reply: "success" - reply: "success"
- execve: (execute file inside container): - execve: (execute file inside container):
send: argv, env, rLimits, <fds> - send: argv, env, rLimits, fds
reply: - reply:
- success: "success", <pid> - success: "success", pid
- failed: "failed" - failed: "failed"
send (success): "init_finished" (as cmd) - send (success): "init_finished" (as cmd)
- reply: "finished" / send: "kill" (as cmd) - reply: "finished" / send: "kill" (as cmd)
- send: "kill" (as cmd) / reply: "finished" - send: "kill" (as cmd) / reply: "finished"
reply: - reply:
Any socket related error will cause the daemon exit (with all process inside container) Any socket related error will cause the daemon exit (with all process inside container)
*/ */
import ( import (
"os"
"syscall"
"github.com/criyle/go-sandbox/pkg/rlimit" "github.com/criyle/go-sandbox/pkg/rlimit"
"github.com/criyle/go-sandbox/types" "github.com/criyle/go-sandbox/types"
) )
// Cmd is the control message send into daemon // Cmd is the control message send into daemon
type Cmd struct { type Cmd struct {
Cmd string // type of the cmd Cmd string // type of the cmd
Path string // path (copyin / open)
Argv []string // execve argv OpenCmd []OpenCmd // open argument
Env []string // execve env DeleteCmd *DeleteCmd // delete argument
RLimits []rlimit.RLimit // execve posix rlimit ExecCmd *ExecCmd // execve argument
FdExec bool // if use fexecve (fd[0] as exec) ConfCmd *ConfCmd // to set configuration
Conf *containerConfig // to set configuration }
// OpenCmd correspond to a single open syscall
type OpenCmd struct {
Path string
Flag int
Perm os.FileMode
}
// DeleteCmd stores delete command
type DeleteCmd struct {
Path string
}
// ExecCmd stores execve parameter
type ExecCmd struct {
Argv []string // execve argv
Env []string // execve env
RLimits []rlimit.RLimit // execve posix rlimit
FdExec bool // if use fexecve (fd[0] as exec)
}
// ConfCmd stores conf parameter
type ConfCmd struct {
Conf containerConfig
} }
// Reply is the reply message send back to controller // Reply is the reply message send back to controller
type Reply struct { type Reply struct {
Error string // empty if no error Error *ErrorReply // nil if no error
ExecReply *ExecReply
}
// ErrorReply stores error returned back from container
type ErrorReply struct {
Msg string
Errno *syscall.Errno
}
// ExecReply stores execve result
type ExecReply struct {
ExitStatus int // waitpid exit status ExitStatus int // waitpid exit status
Status types.Status // return status Status types.Status // return status
UserTime uint64 // waitpid user CPU (ms) UserTime uint64 // waitpid user CPU (ms)
UserMem uint64 // waitpid user memory (kb) UserMem uint64 // waitpid user memory (kb)
} }
func (e *ErrorReply) Error() string {
return e.Msg
}

View File

@ -35,8 +35,8 @@ func (m *Master) conf(conf *containerConfig) error {
defer m.mu.Unlock() defer m.mu.Unlock()
cmd := Cmd{ cmd := Cmd{
Cmd: cmdConf, Cmd: cmdConf,
Conf: conf, ConfCmd: &ConfCmd{Conf: *conf},
} }
if err := m.sendCmd(&cmd, nil); err != nil { if err := m.sendCmd(&cmd, nil); err != nil {
return fmt.Errorf("conf: %v", err) return fmt.Errorf("conf: %v", err)
@ -44,34 +44,15 @@ func (m *Master) conf(conf *containerConfig) error {
return m.recvAckReply("conf") return m.recvAckReply("conf")
} }
// CopyIn copies file to container // Open open files in container
func (m *Master) CopyIn(f *os.File, p string) error { func (m *Master) Open(p []OpenCmd) ([]*os.File, error) {
m.mu.Lock() m.mu.Lock()
defer m.mu.Unlock() defer m.mu.Unlock()
// send copyin // send copyin
cmd := Cmd{ cmd := Cmd{
Cmd: cmdCopyIn, Cmd: cmdOpen,
Path: p, OpenCmd: p,
}
msg := unixsocket.Msg{
Fds: []int{int(f.Fd())},
}
if err := m.sendCmd(&cmd, &msg); err != nil {
return fmt.Errorf("copyin: %v", err)
}
return m.recvAckReply("copyin")
}
// Open open file in container
func (m *Master) Open(p string) (*os.File, error) {
m.mu.Lock()
defer m.mu.Unlock()
// send copyin
cmd := Cmd{
Cmd: cmdOpen,
Path: p,
} }
if err := m.sendCmd(&cmd, nil); err != nil { if err := m.sendCmd(&cmd, nil); err != nil {
return nil, fmt.Errorf("open: %v", err) return nil, fmt.Errorf("open: %v", err)
@ -80,19 +61,24 @@ func (m *Master) Open(p string) (*os.File, error) {
if err != nil { if err != nil {
return nil, fmt.Errorf("open: %v", err) return nil, fmt.Errorf("open: %v", err)
} }
if reply.Error != "" { if reply.Error != nil {
return nil, fmt.Errorf("open: %v", reply.Error) return nil, fmt.Errorf("open: %v", reply.Error)
} }
if len(msg.Fds) != 1 { if len(msg.Fds) != len(p) {
closeFds(msg.Fds) closeFds(msg.Fds)
return nil, fmt.Errorf("open: unexpected number of fd %v", len(msg.Fds)) return nil, fmt.Errorf("open: unexpected number of fd %v / %v", len(msg.Fds), len(p))
} }
f := os.NewFile(uintptr(msg.Fds[0]), p)
if f == nil { ret := make([]*os.File, 0, len(p))
closeFds(msg.Fds) for i, fd := range msg.Fds {
return nil, fmt.Errorf("open: failed %v", msg.Fds[0]) f := os.NewFile(uintptr(fd), p[i].Path)
if f == nil {
closeFds(msg.Fds)
return nil, fmt.Errorf("open: failed %v", msg.Fds[0])
}
ret = append(ret, f)
} }
return f, nil return ret, nil
} }
// Delete remove file from container // Delete remove file from container
@ -101,8 +87,8 @@ func (m *Master) Delete(p string) error {
defer m.mu.Unlock() defer m.mu.Unlock()
cmd := Cmd{ cmd := Cmd{
Cmd: cmdDelete, Cmd: cmdDelete,
Path: p, DeleteCmd: &DeleteCmd{Path: p},
} }
if err := m.sendCmd(&cmd, nil); err != nil { if err := m.sendCmd(&cmd, nil); err != nil {
return fmt.Errorf("delete: %v", err) return fmt.Errorf("delete: %v", err)
@ -129,7 +115,7 @@ func (m *Master) recvAckReply(name string) error {
if err != nil { if err != nil {
return fmt.Errorf("%v: recvAck %v", name, err) return fmt.Errorf("%v: recvAck %v", name, err)
} }
if reply.Error != "" { if reply.Error != nil {
return fmt.Errorf("%v: container error %v", name, reply.Error) return fmt.Errorf("%v: container error %v", name, reply.Error)
} }
return nil return nil

View File

@ -41,13 +41,16 @@ func (m *Master) Execve(done <-chan struct{}, param *ExecveParam) (<-chan types.
msg := &unixsocket.Msg{ msg := &unixsocket.Msg{
Fds: files, Fds: files,
} }
cmd := Cmd{ execCmd := &ExecCmd{
Cmd: cmdExecve,
Argv: param.Args, Argv: param.Args,
Env: param.Env, Env: param.Env,
RLimits: param.RLimits, RLimits: param.RLimits,
FdExec: param.ExecFile > 0, FdExec: param.ExecFile > 0,
} }
cmd := Cmd{
Cmd: cmdExecve,
ExecCmd: execCmd,
}
if err := m.sendCmd(&cmd, msg); err != nil { if err := m.sendCmd(&cmd, msg); err != nil {
m.mu.Unlock() m.mu.Unlock()
return nil, fmt.Errorf("execve: sendCmd %v", err) return nil, fmt.Errorf("execve: sendCmd %v", err)
@ -59,7 +62,7 @@ func (m *Master) Execve(done <-chan struct{}, param *ExecveParam) (<-chan types.
return nil, fmt.Errorf("execve: recvReply %v", err) return nil, fmt.Errorf("execve: recvReply %v", err)
} }
// if sync function did not involved // if sync function did not involved
if reply.Error != "" || msg == nil || msg.Cred == nil { if reply.Error != nil || msg == nil || msg.Cred == nil {
// tell kill function to exit and sync // tell kill function to exit and sync
m.execveSyncKill() m.execveSyncKill()
m.mu.Unlock() m.mu.Unlock()
@ -95,6 +98,7 @@ func (m *Master) Execve(done <-chan struct{}, param *ExecveParam) (<-chan types.
m.recvReply() m.recvReply()
// unlock after last read / write // unlock after last read / write
m.mu.Unlock() m.mu.Unlock()
// handle potential error // handle potential error
if err != nil { if err != nil {
result <- types.Result{ result <- types.Result{
@ -103,17 +107,27 @@ func (m *Master) Execve(done <-chan struct{}, param *ExecveParam) (<-chan types.
} }
return return
} }
// emit result after all communication finish if reply2.ExecReply == nil {
status := reply2.Status result <- types.Result{
if reply2.Error != "" { Status: types.StatusFatal,
status = types.StatusFatal Error: "execve: no reply received",
}
return
} }
// emit result after all communication finish
status := reply2.ExecReply.Status
errMsg := ""
if reply2.Error != nil {
status = types.StatusFatal
errMsg = reply2.Error.Error()
}
result <- types.Result{ result <- types.Result{
Status: status, Status: status,
ExitStatus: reply2.ExitStatus, ExitStatus: reply2.ExecReply.ExitStatus,
UserTime: reply2.UserTime, UserTime: reply2.ExecReply.UserTime,
UserMem: reply2.UserMem, UserMem: reply2.ExecReply.UserMem,
Error: reply2.Error, Error: errMsg,
SetUpTime: mTime.Sub(sTime), SetUpTime: mTime.Sub(sTime),
RunningTime: time.Since(mTime), RunningTime: time.Since(mTime),
} }