use open syscall to copyin files

This commit is contained in:
criyle 2019-12-25 19:51:28 +08:00
parent 1cf901bbc2
commit 6b97e1f663
7 changed files with 199 additions and 124 deletions

View File

@ -44,6 +44,33 @@ Default file access syscall check:
1. Pre-fork container daemons to run programs inside
2. Unix socket to pass fd inside / outside
Container / Master Communication Protocol (single thread):
- ping (alive check):
- reply: pong
- conf (set configuration):
- reply pong
- open (open files in given mode inside container):
- send: []OpenCmd
- reply: "success", file fds / "error"
- delete (unlink file / rmdir dir inside container):
- send: path
- reply: "finished" / "error"
- reset (clean up container for later use (clear workdir / tmp)):
- send:
- reply: "success"
- execve: (execute file inside container):
- send: argv, env, rLimits, fds
- reply:
- success: "success", pid
- failed: "failed"
- send (success): "init_finished" (as cmd)
- reply: "finished" / send: "kill" (as cmd)
- send: "kill" (as cmd) / reply: "finished"
- reply:
Any socket related error will cause the daemon exit (with all process inside container)
## Packages (/pkg)
- seccomp: provides seccomp type definition

View File

@ -20,4 +20,6 @@ const (
containerName = "daemon"
containerWD = "/w"
containerMaxProc = 1
)

View File

@ -9,12 +9,15 @@ import (
"github.com/criyle/go-sandbox/types"
)
func (c *containerServer) handleExecve(cmd *Cmd, msg *unixsocket.Msg) error {
func (c *containerServer) handleExecve(cmd *ExecCmd, msg *unixsocket.Msg) error {
var (
files []uintptr
execFile uintptr
cred *syscall.Credential
)
if cmd == nil {
return c.sendErrorReply("execve: no parameter provided")
}
if msg != nil {
files = intSliceToUintptr(msg.Fds)
// don't leak fds to child
@ -118,10 +121,12 @@ func (c *containerServer) handleExecve(cmd *Cmd, msg *unixsocket.Msg) error {
case wstatus.Exited():
exitStatus := wstatus.ExitStatus()
c.sendReply(&Reply{
ExecReply: &ExecReply{
Status: status,
ExitStatus: exitStatus,
UserTime: userTime,
UserMem: userMem,
},
}, nil)
case wstatus.Signaled():
@ -137,14 +142,18 @@ func (c *containerServer) handleExecve(cmd *Cmd, msg *unixsocket.Msg) error {
status = types.StatusRE
}
c.sendReply(&Reply{
ExecReply: &ExecReply{
Status: status,
UserTime: userTime,
UserMem: userMem,
},
}, nil)
default:
c.sendErrorReply("execve: unknown status %v", wstatus)
}
}
// wait for kill msg and reply done for finish
<-killDone
return c.sendReply(&Reply{}, nil)

View File

@ -2,8 +2,9 @@ package daemon
import (
"fmt"
"io"
"os"
"runtime"
"syscall"
"github.com/criyle/go-sandbox/pkg/unixsocket"
)
@ -13,6 +14,7 @@ type containerServer struct {
containerConfig
}
// ContainerConfig set the container config
type containerConfig struct {
Cred bool
}
@ -46,6 +48,9 @@ func Init() (err error) {
os.Exit(0)
}()
// limit container resource usage
runtime.GOMAXPROCS(containerMaxProc)
// new_master shared the socket at fd 3 (marked close_exec)
const defaultFd = 3
soc, err := unixsocket.NewSocket(defaultFd)
@ -76,22 +81,19 @@ func (c *containerServer) handleCmd(cmd *Cmd, msg *unixsocket.Msg) error {
return c.handlePing()
case cmdConf:
return c.handleConf(cmd)
case cmdCopyIn:
return c.handleCopyIn(cmd, msg)
return c.handleConf(cmd.ConfCmd)
case cmdOpen:
return c.handleOpen(cmd)
return c.handleOpen(cmd.OpenCmd)
case cmdDelete:
return c.handleDelete(cmd)
return c.handleDelete(cmd.DeleteCmd)
case cmdReset:
return c.handleReset()
case cmdExecve:
return c.handleExecve(cmd, msg)
return c.handleExecve(cmd.ExecCmd, msg)
}
return fmt.Errorf("unknown command: %v", cmd.Cmd)
}
@ -100,51 +102,37 @@ func (c *containerServer) handlePing() error {
return c.sendReply(&Reply{}, nil)
}
func (c *containerServer) handleConf(cmd *Cmd) error {
if cmd.Conf != nil {
c.containerConfig = *cmd.Conf
func (c *containerServer) handleConf(conf *ConfCmd) error {
if conf != nil {
c.containerConfig = conf.Conf
}
return c.sendReply(&Reply{}, nil)
}
func (c *containerServer) handleCopyIn(cmd *Cmd, msg *unixsocket.Msg) error {
if len(msg.Fds) != 1 {
closeFds(msg.Fds)
return c.sendErrorReply("copyin: unexpected number of fds(%d)", len(msg.Fds))
func (c *containerServer) handleOpen(open []OpenCmd) error {
if len(open) == 0 {
return c.sendErrorReply("open: no open parameter received")
}
inf := os.NewFile(uintptr(msg.Fds[0]), cmd.Path)
if inf == nil {
return c.sendErrorReply("copyin: newfile failed %v", msg.Fds[0])
}
defer inf.Close()
// have 0777 permission to be able copy in executables
outf, err := os.OpenFile(cmd.Path, os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0777)
if err != nil {
return c.sendErrorReply("copyin: open write file %v", err)
}
defer outf.Close()
if _, err = io.Copy(outf, inf); err != nil {
return c.sendErrorReply("copyin: io.copy %v", err)
}
return c.sendReply(&Reply{}, nil)
}
func (c *containerServer) handleOpen(cmd *Cmd) error {
outf, err := os.Open(cmd.Path)
// open files
fds := make([]int, 0, len(open))
for _, o := range open {
outFile, err := os.OpenFile(o.Path, o.Flag, o.Perm)
if err != nil {
return c.sendErrorReply("open: %v", err)
}
defer outf.Close()
defer outFile.Close()
fds = append(fds, int(outFile.Fd()))
}
return c.sendReply(&Reply{}, &unixsocket.Msg{
Fds: []int{int(outf.Fd())},
})
return c.sendReply(&Reply{}, &unixsocket.Msg{Fds: fds})
}
func (c *containerServer) handleDelete(cmd *Cmd) error {
if err := os.Remove(cmd.Path); err != nil {
func (c *containerServer) handleDelete(delete *DeleteCmd) error {
if delete == nil {
return c.sendErrorReply("delete: no parameter provided")
}
if err := os.Remove(delete.Path); err != nil {
return c.sendErrorReply("delete: %v", err)
}
return c.sendReply(&Reply{}, nil)
@ -175,5 +163,14 @@ func (c *containerServer) sendReply(reply *Reply, msg *unixsocket.Msg) error {
// sendErrorReply sends error reply
func (c *containerServer) sendErrorReply(ft string, v ...interface{}) error {
return c.sendReply(&Reply{Error: fmt.Sprintf(ft, v...)}, nil)
errorReply := &ErrorReply{
Msg: fmt.Sprintf(ft, v...),
}
// store errno
if len(v) == 1 {
if errno, ok := v[0].(syscall.Errno); ok {
errorReply.Errno = &errno
}
}
return c.sendReply(&Reply{Error: errorReply}, nil)
}

View File

@ -5,37 +5,38 @@
package daemon
/*
Protocol between client and daemon (not thread safe):
Container / Master Communication Protocol (single thread):
- ping (alive check):
reply: pong
- reply: pong
- conf (set configuration):
reply pong
- copyin (copy file into container):
send: path, <input fd>
reply: "finished" (after copy finished) / "error"
- open (open file in read-only mode inside container):
send: path
reply: "success", <file fd> / "error"
- reply pong
- open (open files in given mode inside container):
- send: []OpenCmd
- reply: "success", file fds / "error"
- delete (unlink file / rmdir dir inside container):
send: path
reply: "finished" / "error"
- send: path
- reply: "finished" / "error"
- reset (clean up container for later use (clear workdir / tmp)):
send:
reply: "success"
- send:
- reply: "success"
- execve: (execute file inside container):
send: argv, env, rLimits, <fds>
reply:
- success: "success", <pid>
- send: argv, env, rLimits, fds
- reply:
- success: "success", pid
- failed: "failed"
send (success): "init_finished" (as cmd)
- send (success): "init_finished" (as cmd)
- reply: "finished" / send: "kill" (as cmd)
- send: "kill" (as cmd) / reply: "finished"
reply:
- reply:
Any socket related error will cause the daemon exit (with all process inside container)
*/
import (
"os"
"syscall"
"github.com/criyle/go-sandbox/pkg/rlimit"
"github.com/criyle/go-sandbox/types"
)
@ -43,19 +44,58 @@ import (
// Cmd is the control message send into daemon
type Cmd struct {
Cmd string // type of the cmd
Path string // path (copyin / open)
OpenCmd []OpenCmd // open argument
DeleteCmd *DeleteCmd // delete argument
ExecCmd *ExecCmd // execve argument
ConfCmd *ConfCmd // to set configuration
}
// OpenCmd correspond to a single open syscall
type OpenCmd struct {
Path string
Flag int
Perm os.FileMode
}
// DeleteCmd stores delete command
type DeleteCmd struct {
Path string
}
// ExecCmd stores execve parameter
type ExecCmd struct {
Argv []string // execve argv
Env []string // execve env
RLimits []rlimit.RLimit // execve posix rlimit
FdExec bool // if use fexecve (fd[0] as exec)
Conf *containerConfig // to set configuration
}
// ConfCmd stores conf parameter
type ConfCmd struct {
Conf containerConfig
}
// Reply is the reply message send back to controller
type Reply struct {
Error string // empty if no error
Error *ErrorReply // nil if no error
ExecReply *ExecReply
}
// ErrorReply stores error returned back from container
type ErrorReply struct {
Msg string
Errno *syscall.Errno
}
// ExecReply stores execve result
type ExecReply struct {
ExitStatus int // waitpid exit status
Status types.Status // return status
UserTime uint64 // waitpid user CPU (ms)
UserMem uint64 // waitpid user memory (kb)
}
func (e *ErrorReply) Error() string {
return e.Msg
}

View File

@ -36,7 +36,7 @@ func (m *Master) conf(conf *containerConfig) error {
cmd := Cmd{
Cmd: cmdConf,
Conf: conf,
ConfCmd: &ConfCmd{Conf: *conf},
}
if err := m.sendCmd(&cmd, nil); err != nil {
return fmt.Errorf("conf: %v", err)
@ -44,34 +44,15 @@ func (m *Master) conf(conf *containerConfig) error {
return m.recvAckReply("conf")
}
// CopyIn copies file to container
func (m *Master) CopyIn(f *os.File, p string) error {
m.mu.Lock()
defer m.mu.Unlock()
// send copyin
cmd := Cmd{
Cmd: cmdCopyIn,
Path: p,
}
msg := unixsocket.Msg{
Fds: []int{int(f.Fd())},
}
if err := m.sendCmd(&cmd, &msg); err != nil {
return fmt.Errorf("copyin: %v", err)
}
return m.recvAckReply("copyin")
}
// Open open file in container
func (m *Master) Open(p string) (*os.File, error) {
// Open open files in container
func (m *Master) Open(p []OpenCmd) ([]*os.File, error) {
m.mu.Lock()
defer m.mu.Unlock()
// send copyin
cmd := Cmd{
Cmd: cmdOpen,
Path: p,
OpenCmd: p,
}
if err := m.sendCmd(&cmd, nil); err != nil {
return nil, fmt.Errorf("open: %v", err)
@ -80,19 +61,24 @@ func (m *Master) Open(p string) (*os.File, error) {
if err != nil {
return nil, fmt.Errorf("open: %v", err)
}
if reply.Error != "" {
if reply.Error != nil {
return nil, fmt.Errorf("open: %v", reply.Error)
}
if len(msg.Fds) != 1 {
if len(msg.Fds) != len(p) {
closeFds(msg.Fds)
return nil, fmt.Errorf("open: unexpected number of fd %v", len(msg.Fds))
return nil, fmt.Errorf("open: unexpected number of fd %v / %v", len(msg.Fds), len(p))
}
f := os.NewFile(uintptr(msg.Fds[0]), p)
ret := make([]*os.File, 0, len(p))
for i, fd := range msg.Fds {
f := os.NewFile(uintptr(fd), p[i].Path)
if f == nil {
closeFds(msg.Fds)
return nil, fmt.Errorf("open: failed %v", msg.Fds[0])
}
return f, nil
ret = append(ret, f)
}
return ret, nil
}
// Delete remove file from container
@ -102,7 +88,7 @@ func (m *Master) Delete(p string) error {
cmd := Cmd{
Cmd: cmdDelete,
Path: p,
DeleteCmd: &DeleteCmd{Path: p},
}
if err := m.sendCmd(&cmd, nil); err != nil {
return fmt.Errorf("delete: %v", err)
@ -129,7 +115,7 @@ func (m *Master) recvAckReply(name string) error {
if err != nil {
return fmt.Errorf("%v: recvAck %v", name, err)
}
if reply.Error != "" {
if reply.Error != nil {
return fmt.Errorf("%v: container error %v", name, reply.Error)
}
return nil

View File

@ -41,13 +41,16 @@ func (m *Master) Execve(done <-chan struct{}, param *ExecveParam) (<-chan types.
msg := &unixsocket.Msg{
Fds: files,
}
cmd := Cmd{
Cmd: cmdExecve,
execCmd := &ExecCmd{
Argv: param.Args,
Env: param.Env,
RLimits: param.RLimits,
FdExec: param.ExecFile > 0,
}
cmd := Cmd{
Cmd: cmdExecve,
ExecCmd: execCmd,
}
if err := m.sendCmd(&cmd, msg); err != nil {
m.mu.Unlock()
return nil, fmt.Errorf("execve: sendCmd %v", err)
@ -59,7 +62,7 @@ func (m *Master) Execve(done <-chan struct{}, param *ExecveParam) (<-chan types.
return nil, fmt.Errorf("execve: recvReply %v", err)
}
// if sync function did not involved
if reply.Error != "" || msg == nil || msg.Cred == nil {
if reply.Error != nil || msg == nil || msg.Cred == nil {
// tell kill function to exit and sync
m.execveSyncKill()
m.mu.Unlock()
@ -95,6 +98,7 @@ func (m *Master) Execve(done <-chan struct{}, param *ExecveParam) (<-chan types.
m.recvReply()
// unlock after last read / write
m.mu.Unlock()
// handle potential error
if err != nil {
result <- types.Result{
@ -103,17 +107,27 @@ func (m *Master) Execve(done <-chan struct{}, param *ExecveParam) (<-chan types.
}
return
}
// emit result after all communication finish
status := reply2.Status
if reply2.Error != "" {
status = types.StatusFatal
if reply2.ExecReply == nil {
result <- types.Result{
Status: types.StatusFatal,
Error: "execve: no reply received",
}
return
}
// emit result after all communication finish
status := reply2.ExecReply.Status
errMsg := ""
if reply2.Error != nil {
status = types.StatusFatal
errMsg = reply2.Error.Error()
}
result <- types.Result{
Status: status,
ExitStatus: reply2.ExitStatus,
UserTime: reply2.UserTime,
UserMem: reply2.UserMem,
Error: reply2.Error,
ExitStatus: reply2.ExecReply.ExitStatus,
UserTime: reply2.ExecReply.UserTime,
UserMem: reply2.ExecReply.UserMem,
Error: errMsg,
SetUpTime: mTime.Sub(sTime),
RunningTime: time.Since(mTime),
}