mirror of
https://github.com/criyle/go-sandbox.git
synced 2025-11-04 14:49:53 +08:00
use open syscall to copyin files
This commit is contained in:
parent
1cf901bbc2
commit
6b97e1f663
27
README.md
27
README.md
@ -44,6 +44,33 @@ Default file access syscall check:
|
||||
1. Pre-fork container daemons to run programs inside
|
||||
2. Unix socket to pass fd inside / outside
|
||||
|
||||
Container / Master Communication Protocol (single thread):
|
||||
|
||||
- ping (alive check):
|
||||
- reply: pong
|
||||
- conf (set configuration):
|
||||
- reply pong
|
||||
- open (open files in given mode inside container):
|
||||
- send: []OpenCmd
|
||||
- reply: "success", file fds / "error"
|
||||
- delete (unlink file / rmdir dir inside container):
|
||||
- send: path
|
||||
- reply: "finished" / "error"
|
||||
- reset (clean up container for later use (clear workdir / tmp)):
|
||||
- send:
|
||||
- reply: "success"
|
||||
- execve: (execute file inside container):
|
||||
- send: argv, env, rLimits, fds
|
||||
- reply:
|
||||
- success: "success", pid
|
||||
- failed: "failed"
|
||||
- send (success): "init_finished" (as cmd)
|
||||
- reply: "finished" / send: "kill" (as cmd)
|
||||
- send: "kill" (as cmd) / reply: "finished"
|
||||
- reply:
|
||||
|
||||
Any socket related error will cause the daemon exit (with all process inside container)
|
||||
|
||||
## Packages (/pkg)
|
||||
|
||||
- seccomp: provides seccomp type definition
|
||||
|
||||
@ -20,4 +20,6 @@ const (
|
||||
|
||||
containerName = "daemon"
|
||||
containerWD = "/w"
|
||||
|
||||
containerMaxProc = 1
|
||||
)
|
||||
|
||||
@ -9,12 +9,15 @@ import (
|
||||
"github.com/criyle/go-sandbox/types"
|
||||
)
|
||||
|
||||
func (c *containerServer) handleExecve(cmd *Cmd, msg *unixsocket.Msg) error {
|
||||
func (c *containerServer) handleExecve(cmd *ExecCmd, msg *unixsocket.Msg) error {
|
||||
var (
|
||||
files []uintptr
|
||||
execFile uintptr
|
||||
cred *syscall.Credential
|
||||
)
|
||||
if cmd == nil {
|
||||
return c.sendErrorReply("execve: no parameter provided")
|
||||
}
|
||||
if msg != nil {
|
||||
files = intSliceToUintptr(msg.Fds)
|
||||
// don't leak fds to child
|
||||
@ -118,10 +121,12 @@ func (c *containerServer) handleExecve(cmd *Cmd, msg *unixsocket.Msg) error {
|
||||
case wstatus.Exited():
|
||||
exitStatus := wstatus.ExitStatus()
|
||||
c.sendReply(&Reply{
|
||||
Status: status,
|
||||
ExitStatus: exitStatus,
|
||||
UserTime: userTime,
|
||||
UserMem: userMem,
|
||||
ExecReply: &ExecReply{
|
||||
Status: status,
|
||||
ExitStatus: exitStatus,
|
||||
UserTime: userTime,
|
||||
UserMem: userMem,
|
||||
},
|
||||
}, nil)
|
||||
|
||||
case wstatus.Signaled():
|
||||
@ -137,14 +142,18 @@ func (c *containerServer) handleExecve(cmd *Cmd, msg *unixsocket.Msg) error {
|
||||
status = types.StatusRE
|
||||
}
|
||||
c.sendReply(&Reply{
|
||||
Status: status,
|
||||
UserTime: userTime,
|
||||
UserMem: userMem,
|
||||
ExecReply: &ExecReply{
|
||||
Status: status,
|
||||
UserTime: userTime,
|
||||
UserMem: userMem,
|
||||
},
|
||||
}, nil)
|
||||
|
||||
default:
|
||||
c.sendErrorReply("execve: unknown status %v", wstatus)
|
||||
}
|
||||
}
|
||||
|
||||
// wait for kill msg and reply done for finish
|
||||
<-killDone
|
||||
return c.sendReply(&Reply{}, nil)
|
||||
|
||||
@ -2,8 +2,9 @@ package daemon
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"runtime"
|
||||
"syscall"
|
||||
|
||||
"github.com/criyle/go-sandbox/pkg/unixsocket"
|
||||
)
|
||||
@ -13,6 +14,7 @@ type containerServer struct {
|
||||
containerConfig
|
||||
}
|
||||
|
||||
// ContainerConfig set the container config
|
||||
type containerConfig struct {
|
||||
Cred bool
|
||||
}
|
||||
@ -46,6 +48,9 @@ func Init() (err error) {
|
||||
os.Exit(0)
|
||||
}()
|
||||
|
||||
// limit container resource usage
|
||||
runtime.GOMAXPROCS(containerMaxProc)
|
||||
|
||||
// new_master shared the socket at fd 3 (marked close_exec)
|
||||
const defaultFd = 3
|
||||
soc, err := unixsocket.NewSocket(defaultFd)
|
||||
@ -76,22 +81,19 @@ func (c *containerServer) handleCmd(cmd *Cmd, msg *unixsocket.Msg) error {
|
||||
return c.handlePing()
|
||||
|
||||
case cmdConf:
|
||||
return c.handleConf(cmd)
|
||||
|
||||
case cmdCopyIn:
|
||||
return c.handleCopyIn(cmd, msg)
|
||||
return c.handleConf(cmd.ConfCmd)
|
||||
|
||||
case cmdOpen:
|
||||
return c.handleOpen(cmd)
|
||||
return c.handleOpen(cmd.OpenCmd)
|
||||
|
||||
case cmdDelete:
|
||||
return c.handleDelete(cmd)
|
||||
return c.handleDelete(cmd.DeleteCmd)
|
||||
|
||||
case cmdReset:
|
||||
return c.handleReset()
|
||||
|
||||
case cmdExecve:
|
||||
return c.handleExecve(cmd, msg)
|
||||
return c.handleExecve(cmd.ExecCmd, msg)
|
||||
}
|
||||
return fmt.Errorf("unknown command: %v", cmd.Cmd)
|
||||
}
|
||||
@ -100,51 +102,37 @@ func (c *containerServer) handlePing() error {
|
||||
return c.sendReply(&Reply{}, nil)
|
||||
}
|
||||
|
||||
func (c *containerServer) handleConf(cmd *Cmd) error {
|
||||
if cmd.Conf != nil {
|
||||
c.containerConfig = *cmd.Conf
|
||||
func (c *containerServer) handleConf(conf *ConfCmd) error {
|
||||
if conf != nil {
|
||||
c.containerConfig = conf.Conf
|
||||
}
|
||||
return c.sendReply(&Reply{}, nil)
|
||||
}
|
||||
|
||||
func (c *containerServer) handleCopyIn(cmd *Cmd, msg *unixsocket.Msg) error {
|
||||
if len(msg.Fds) != 1 {
|
||||
closeFds(msg.Fds)
|
||||
return c.sendErrorReply("copyin: unexpected number of fds(%d)", len(msg.Fds))
|
||||
func (c *containerServer) handleOpen(open []OpenCmd) error {
|
||||
if len(open) == 0 {
|
||||
return c.sendErrorReply("open: no open parameter received")
|
||||
}
|
||||
inf := os.NewFile(uintptr(msg.Fds[0]), cmd.Path)
|
||||
if inf == nil {
|
||||
return c.sendErrorReply("copyin: newfile failed %v", msg.Fds[0])
|
||||
}
|
||||
defer inf.Close()
|
||||
|
||||
// have 0777 permission to be able copy in executables
|
||||
outf, err := os.OpenFile(cmd.Path, os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0777)
|
||||
if err != nil {
|
||||
return c.sendErrorReply("copyin: open write file %v", err)
|
||||
// open files
|
||||
fds := make([]int, 0, len(open))
|
||||
for _, o := range open {
|
||||
outFile, err := os.OpenFile(o.Path, o.Flag, o.Perm)
|
||||
if err != nil {
|
||||
return c.sendErrorReply("open: %v", err)
|
||||
}
|
||||
defer outFile.Close()
|
||||
fds = append(fds, int(outFile.Fd()))
|
||||
}
|
||||
defer outf.Close()
|
||||
|
||||
if _, err = io.Copy(outf, inf); err != nil {
|
||||
return c.sendErrorReply("copyin: io.copy %v", err)
|
||||
}
|
||||
return c.sendReply(&Reply{}, nil)
|
||||
return c.sendReply(&Reply{}, &unixsocket.Msg{Fds: fds})
|
||||
}
|
||||
|
||||
func (c *containerServer) handleOpen(cmd *Cmd) error {
|
||||
outf, err := os.Open(cmd.Path)
|
||||
if err != nil {
|
||||
return c.sendErrorReply("open: %v", err)
|
||||
func (c *containerServer) handleDelete(delete *DeleteCmd) error {
|
||||
if delete == nil {
|
||||
return c.sendErrorReply("delete: no parameter provided")
|
||||
}
|
||||
defer outf.Close()
|
||||
|
||||
return c.sendReply(&Reply{}, &unixsocket.Msg{
|
||||
Fds: []int{int(outf.Fd())},
|
||||
})
|
||||
}
|
||||
|
||||
func (c *containerServer) handleDelete(cmd *Cmd) error {
|
||||
if err := os.Remove(cmd.Path); err != nil {
|
||||
if err := os.Remove(delete.Path); err != nil {
|
||||
return c.sendErrorReply("delete: %v", err)
|
||||
}
|
||||
return c.sendReply(&Reply{}, nil)
|
||||
@ -175,5 +163,14 @@ func (c *containerServer) sendReply(reply *Reply, msg *unixsocket.Msg) error {
|
||||
|
||||
// sendErrorReply sends error reply
|
||||
func (c *containerServer) sendErrorReply(ft string, v ...interface{}) error {
|
||||
return c.sendReply(&Reply{Error: fmt.Sprintf(ft, v...)}, nil)
|
||||
errorReply := &ErrorReply{
|
||||
Msg: fmt.Sprintf(ft, v...),
|
||||
}
|
||||
// store errno
|
||||
if len(v) == 1 {
|
||||
if errno, ok := v[0].(syscall.Errno); ok {
|
||||
errorReply.Errno = &errno
|
||||
}
|
||||
}
|
||||
return c.sendReply(&Reply{Error: errorReply}, nil)
|
||||
}
|
||||
|
||||
@ -5,57 +5,97 @@
|
||||
package daemon
|
||||
|
||||
/*
|
||||
Protocol between client and daemon (not thread safe):
|
||||
Container / Master Communication Protocol (single thread):
|
||||
|
||||
- ping (alive check):
|
||||
reply: pong
|
||||
- reply: pong
|
||||
- conf (set configuration):
|
||||
reply pong
|
||||
- copyin (copy file into container):
|
||||
send: path, <input fd>
|
||||
reply: "finished" (after copy finished) / "error"
|
||||
- open (open file in read-only mode inside container):
|
||||
send: path
|
||||
reply: "success", <file fd> / "error"
|
||||
- reply pong
|
||||
- open (open files in given mode inside container):
|
||||
- send: []OpenCmd
|
||||
- reply: "success", file fds / "error"
|
||||
- delete (unlink file / rmdir dir inside container):
|
||||
send: path
|
||||
reply: "finished" / "error"
|
||||
- send: path
|
||||
- reply: "finished" / "error"
|
||||
- reset (clean up container for later use (clear workdir / tmp)):
|
||||
send:
|
||||
reply: "success"
|
||||
- send:
|
||||
- reply: "success"
|
||||
- execve: (execute file inside container):
|
||||
send: argv, env, rLimits, <fds>
|
||||
reply:
|
||||
- success: "success", <pid>
|
||||
- send: argv, env, rLimits, fds
|
||||
- reply:
|
||||
- success: "success", pid
|
||||
- failed: "failed"
|
||||
send (success): "init_finished" (as cmd)
|
||||
- send (success): "init_finished" (as cmd)
|
||||
- reply: "finished" / send: "kill" (as cmd)
|
||||
- send: "kill" (as cmd) / reply: "finished"
|
||||
reply:
|
||||
- reply:
|
||||
|
||||
Any socket related error will cause the daemon exit (with all process inside container)
|
||||
*/
|
||||
|
||||
import (
|
||||
"os"
|
||||
"syscall"
|
||||
|
||||
"github.com/criyle/go-sandbox/pkg/rlimit"
|
||||
"github.com/criyle/go-sandbox/types"
|
||||
)
|
||||
|
||||
// Cmd is the control message send into daemon
|
||||
type Cmd struct {
|
||||
Cmd string // type of the cmd
|
||||
Path string // path (copyin / open)
|
||||
Argv []string // execve argv
|
||||
Env []string // execve env
|
||||
RLimits []rlimit.RLimit // execve posix rlimit
|
||||
FdExec bool // if use fexecve (fd[0] as exec)
|
||||
Conf *containerConfig // to set configuration
|
||||
Cmd string // type of the cmd
|
||||
|
||||
OpenCmd []OpenCmd // open argument
|
||||
DeleteCmd *DeleteCmd // delete argument
|
||||
ExecCmd *ExecCmd // execve argument
|
||||
ConfCmd *ConfCmd // to set configuration
|
||||
}
|
||||
|
||||
// OpenCmd correspond to a single open syscall
|
||||
type OpenCmd struct {
|
||||
Path string
|
||||
Flag int
|
||||
Perm os.FileMode
|
||||
}
|
||||
|
||||
// DeleteCmd stores delete command
|
||||
type DeleteCmd struct {
|
||||
Path string
|
||||
}
|
||||
|
||||
// ExecCmd stores execve parameter
|
||||
type ExecCmd struct {
|
||||
Argv []string // execve argv
|
||||
Env []string // execve env
|
||||
RLimits []rlimit.RLimit // execve posix rlimit
|
||||
FdExec bool // if use fexecve (fd[0] as exec)
|
||||
}
|
||||
|
||||
// ConfCmd stores conf parameter
|
||||
type ConfCmd struct {
|
||||
Conf containerConfig
|
||||
}
|
||||
|
||||
// Reply is the reply message send back to controller
|
||||
type Reply struct {
|
||||
Error string // empty if no error
|
||||
Error *ErrorReply // nil if no error
|
||||
ExecReply *ExecReply
|
||||
}
|
||||
|
||||
// ErrorReply stores error returned back from container
|
||||
type ErrorReply struct {
|
||||
Msg string
|
||||
Errno *syscall.Errno
|
||||
}
|
||||
|
||||
// ExecReply stores execve result
|
||||
type ExecReply struct {
|
||||
ExitStatus int // waitpid exit status
|
||||
Status types.Status // return status
|
||||
UserTime uint64 // waitpid user CPU (ms)
|
||||
UserMem uint64 // waitpid user memory (kb)
|
||||
}
|
||||
|
||||
func (e *ErrorReply) Error() string {
|
||||
return e.Msg
|
||||
}
|
||||
|
||||
@ -35,8 +35,8 @@ func (m *Master) conf(conf *containerConfig) error {
|
||||
defer m.mu.Unlock()
|
||||
|
||||
cmd := Cmd{
|
||||
Cmd: cmdConf,
|
||||
Conf: conf,
|
||||
Cmd: cmdConf,
|
||||
ConfCmd: &ConfCmd{Conf: *conf},
|
||||
}
|
||||
if err := m.sendCmd(&cmd, nil); err != nil {
|
||||
return fmt.Errorf("conf: %v", err)
|
||||
@ -44,34 +44,15 @@ func (m *Master) conf(conf *containerConfig) error {
|
||||
return m.recvAckReply("conf")
|
||||
}
|
||||
|
||||
// CopyIn copies file to container
|
||||
func (m *Master) CopyIn(f *os.File, p string) error {
|
||||
// Open open files in container
|
||||
func (m *Master) Open(p []OpenCmd) ([]*os.File, error) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
|
||||
// send copyin
|
||||
cmd := Cmd{
|
||||
Cmd: cmdCopyIn,
|
||||
Path: p,
|
||||
}
|
||||
msg := unixsocket.Msg{
|
||||
Fds: []int{int(f.Fd())},
|
||||
}
|
||||
if err := m.sendCmd(&cmd, &msg); err != nil {
|
||||
return fmt.Errorf("copyin: %v", err)
|
||||
}
|
||||
return m.recvAckReply("copyin")
|
||||
}
|
||||
|
||||
// Open open file in container
|
||||
func (m *Master) Open(p string) (*os.File, error) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
|
||||
// send copyin
|
||||
cmd := Cmd{
|
||||
Cmd: cmdOpen,
|
||||
Path: p,
|
||||
Cmd: cmdOpen,
|
||||
OpenCmd: p,
|
||||
}
|
||||
if err := m.sendCmd(&cmd, nil); err != nil {
|
||||
return nil, fmt.Errorf("open: %v", err)
|
||||
@ -80,19 +61,24 @@ func (m *Master) Open(p string) (*os.File, error) {
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("open: %v", err)
|
||||
}
|
||||
if reply.Error != "" {
|
||||
if reply.Error != nil {
|
||||
return nil, fmt.Errorf("open: %v", reply.Error)
|
||||
}
|
||||
if len(msg.Fds) != 1 {
|
||||
if len(msg.Fds) != len(p) {
|
||||
closeFds(msg.Fds)
|
||||
return nil, fmt.Errorf("open: unexpected number of fd %v", len(msg.Fds))
|
||||
return nil, fmt.Errorf("open: unexpected number of fd %v / %v", len(msg.Fds), len(p))
|
||||
}
|
||||
f := os.NewFile(uintptr(msg.Fds[0]), p)
|
||||
if f == nil {
|
||||
closeFds(msg.Fds)
|
||||
return nil, fmt.Errorf("open: failed %v", msg.Fds[0])
|
||||
|
||||
ret := make([]*os.File, 0, len(p))
|
||||
for i, fd := range msg.Fds {
|
||||
f := os.NewFile(uintptr(fd), p[i].Path)
|
||||
if f == nil {
|
||||
closeFds(msg.Fds)
|
||||
return nil, fmt.Errorf("open: failed %v", msg.Fds[0])
|
||||
}
|
||||
ret = append(ret, f)
|
||||
}
|
||||
return f, nil
|
||||
return ret, nil
|
||||
}
|
||||
|
||||
// Delete remove file from container
|
||||
@ -101,8 +87,8 @@ func (m *Master) Delete(p string) error {
|
||||
defer m.mu.Unlock()
|
||||
|
||||
cmd := Cmd{
|
||||
Cmd: cmdDelete,
|
||||
Path: p,
|
||||
Cmd: cmdDelete,
|
||||
DeleteCmd: &DeleteCmd{Path: p},
|
||||
}
|
||||
if err := m.sendCmd(&cmd, nil); err != nil {
|
||||
return fmt.Errorf("delete: %v", err)
|
||||
@ -129,7 +115,7 @@ func (m *Master) recvAckReply(name string) error {
|
||||
if err != nil {
|
||||
return fmt.Errorf("%v: recvAck %v", name, err)
|
||||
}
|
||||
if reply.Error != "" {
|
||||
if reply.Error != nil {
|
||||
return fmt.Errorf("%v: container error %v", name, reply.Error)
|
||||
}
|
||||
return nil
|
||||
|
||||
@ -41,13 +41,16 @@ func (m *Master) Execve(done <-chan struct{}, param *ExecveParam) (<-chan types.
|
||||
msg := &unixsocket.Msg{
|
||||
Fds: files,
|
||||
}
|
||||
cmd := Cmd{
|
||||
Cmd: cmdExecve,
|
||||
execCmd := &ExecCmd{
|
||||
Argv: param.Args,
|
||||
Env: param.Env,
|
||||
RLimits: param.RLimits,
|
||||
FdExec: param.ExecFile > 0,
|
||||
}
|
||||
cmd := Cmd{
|
||||
Cmd: cmdExecve,
|
||||
ExecCmd: execCmd,
|
||||
}
|
||||
if err := m.sendCmd(&cmd, msg); err != nil {
|
||||
m.mu.Unlock()
|
||||
return nil, fmt.Errorf("execve: sendCmd %v", err)
|
||||
@ -59,7 +62,7 @@ func (m *Master) Execve(done <-chan struct{}, param *ExecveParam) (<-chan types.
|
||||
return nil, fmt.Errorf("execve: recvReply %v", err)
|
||||
}
|
||||
// if sync function did not involved
|
||||
if reply.Error != "" || msg == nil || msg.Cred == nil {
|
||||
if reply.Error != nil || msg == nil || msg.Cred == nil {
|
||||
// tell kill function to exit and sync
|
||||
m.execveSyncKill()
|
||||
m.mu.Unlock()
|
||||
@ -95,6 +98,7 @@ func (m *Master) Execve(done <-chan struct{}, param *ExecveParam) (<-chan types.
|
||||
m.recvReply()
|
||||
// unlock after last read / write
|
||||
m.mu.Unlock()
|
||||
|
||||
// handle potential error
|
||||
if err != nil {
|
||||
result <- types.Result{
|
||||
@ -103,17 +107,27 @@ func (m *Master) Execve(done <-chan struct{}, param *ExecveParam) (<-chan types.
|
||||
}
|
||||
return
|
||||
}
|
||||
// emit result after all communication finish
|
||||
status := reply2.Status
|
||||
if reply2.Error != "" {
|
||||
status = types.StatusFatal
|
||||
if reply2.ExecReply == nil {
|
||||
result <- types.Result{
|
||||
Status: types.StatusFatal,
|
||||
Error: "execve: no reply received",
|
||||
}
|
||||
return
|
||||
}
|
||||
// emit result after all communication finish
|
||||
status := reply2.ExecReply.Status
|
||||
errMsg := ""
|
||||
if reply2.Error != nil {
|
||||
status = types.StatusFatal
|
||||
errMsg = reply2.Error.Error()
|
||||
}
|
||||
|
||||
result <- types.Result{
|
||||
Status: status,
|
||||
ExitStatus: reply2.ExitStatus,
|
||||
UserTime: reply2.UserTime,
|
||||
UserMem: reply2.UserMem,
|
||||
Error: reply2.Error,
|
||||
ExitStatus: reply2.ExecReply.ExitStatus,
|
||||
UserTime: reply2.ExecReply.UserTime,
|
||||
UserMem: reply2.ExecReply.UserMem,
|
||||
Error: errMsg,
|
||||
SetUpTime: mTime.Sub(sTime),
|
||||
RunningTime: time.Since(mTime),
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user