allow customize mount points for container

This commit is contained in:
criyle 2019-09-25 01:28:04 -07:00
parent a6c330b612
commit 31edacf69e
9 changed files with 195 additions and 91 deletions

View File

@ -44,9 +44,12 @@ func printUsage() {
os.Exit(2)
}
func main() {
daemon.ContainerInit()
func init() {
// container init
daemon.Init()
}
func main() {
flag.Usage = printUsage
flag.Uint64Var(&timeLimit, "tl", 1, "Set time limit (in second)")
flag.Uint64Var(&realTimeLimit, "rtl", 0, "Set real time limit (in second)")
@ -222,7 +225,11 @@ func start() (*types.Result, error) {
}
defer os.RemoveAll(root)
m, err := daemon.New(root)
b := daemon.Builder{
Root: root,
}
m, err := b.Build()
if err != nil {
return nil, fmt.Errorf("failed to new master: %v", err)
}

View File

@ -11,4 +11,6 @@ const (
cmdKill = "kill"
initArg = "init"
currentExec = "/proc/self/exe"
)

View File

@ -13,11 +13,11 @@ import (
"github.com/criyle/go-sandbox/types"
)
// ContainerInit is called for container init process
// Init is called for container init process
// it will check if pid == 1, otherwise it is noop
// ContainerInit will do infinite loop on socket commands,
// and exits when at socket close
func ContainerInit() (err error) {
// Init will do infinite loop on socket commands,
// and exits when at socket close, use it in init function
func Init() (err error) {
// noop if self is not container init process
// Notice: docker init is also 1, additional check for args[1] == init
if os.Getpid() != 1 || len(os.Args) != 2 || os.Args[1] != initArg {
@ -26,6 +26,10 @@ func ContainerInit() (err error) {
// exit process (with whole container) upon exit this function
defer func() {
if err2 := recover(); err2 != nil {
fmt.Fprintf(os.Stderr, "container_panic: %v", err)
os.Exit(1)
}
if err != nil {
fmt.Fprintf(os.Stderr, "container_exit: %v", err)
os.Exit(1)
@ -71,7 +75,7 @@ func handleCmd(s *unixsocket.Socket, cmd *Cmd, msg *unixsocket.Msg) error {
case cmdExecve:
return handleExecve(s, cmd, msg)
}
return nil
return fmt.Errorf("Unknown command: %v", cmd.Cmd)
}
func handlePing(s *unixsocket.Socket) error {
@ -79,8 +83,9 @@ func handlePing(s *unixsocket.Socket) error {
}
func handleCopyIn(s *unixsocket.Socket, cmd *Cmd, msg *unixsocket.Msg) error {
if len(msg.Fds) == 0 {
return sendErrorReply(s, "copyin: did not receive fd")
if len(msg.Fds) != 1 {
closeFds(msg.Fds)
return sendErrorReply(s, "copyin: unexpected number of fds(%d)", len(msg.Fds))
}
inf := os.NewFile(uintptr(msg.Fds[0]), cmd.Path)
if inf == nil {
@ -109,27 +114,23 @@ func handleOpen(s *unixsocket.Socket, cmd *Cmd) error {
}
defer outf.Close()
msg := unixsocket.Msg{
return sendReply(s, &Reply{}, &unixsocket.Msg{
Fds: []int{int(outf.Fd())},
}
return sendReply(s, &Reply{}, &msg)
})
}
func handleDelete(s *unixsocket.Socket, cmd *Cmd) error {
err := os.Remove(cmd.Path)
if err != nil {
if err := os.Remove(cmd.Path); err != nil {
return sendErrorReply(s, "delete: %v", err)
}
return sendReply(s, &Reply{}, nil)
}
func handleReset(s *unixsocket.Socket) error {
err := removeContents("/tmp")
if err != nil {
if err := removeContents("/tmp"); err != nil {
return sendErrorReply(s, "reset: /tmp %v", err)
}
err = removeContents("/w")
if err != nil {
if err := removeContents("/w"); err != nil {
return sendErrorReply(s, "reset: /w %v", err)
}
return sendReply(s, &Reply{}, nil)
@ -142,13 +143,17 @@ func handleExecve(s *unixsocket.Socket, cmd *Cmd, msg *unixsocket.Msg) error {
)
if msg != nil {
files = intSliceToUintptr(msg.Fds)
// don't leak fds to child
closeOnExecFds(msg.Fds)
// release files after execve
defer closeFds(msg.Fds)
}
// don't leak fds to child
closeOnExecFds(msg.Fds)
// release files after execve
defer closeFds(msg.Fds)
if cmd.FdExec && len(files) > 0 {
// if fexecve, then the first fd must be executable
if cmd.FdExec {
if len(files) == 0 {
return fmt.Errorf("execve: expected fexecve fd")
}
execFile = files[0]
files = files[1:]
}
@ -168,8 +173,7 @@ func handleExecve(s *unixsocket.Socket, cmd *Cmd, msg *unixsocket.Msg) error {
if err2 != nil {
return fmt.Errorf("syncFunc: recvCmd(%v)", err2)
}
switch cmd2.Cmd {
case cmdKill:
if cmd2.Cmd == cmdKill {
return fmt.Errorf("syncFunc: recved kill")
}
return nil
@ -185,7 +189,7 @@ func handleExecve(s *unixsocket.Socket, cmd *Cmd, msg *unixsocket.Msg) error {
DropCaps: true,
SyncFunc: syncFunc,
}
// starts the runner, error is handled same as wait4 to make api simple
// starts the runner, error is handled same as wait4 to make communication equal
pid, err := r.Start()
// done is to signal kill goroutine exits
@ -205,8 +209,7 @@ func handleExecve(s *unixsocket.Socket, cmd *Cmd, msg *unixsocket.Msg) error {
<-waitDone
// collect zombies
for {
pid, err := syscall.Wait4(-1, nil, syscall.WNOHANG, nil)
if err != nil || pid <= 0 {
if pid, err := syscall.Wait4(-1, nil, syscall.WNOHANG, nil); err != nil || pid <= 0 {
break
}
}
@ -219,13 +222,13 @@ func handleExecve(s *unixsocket.Socket, cmd *Cmd, msg *unixsocket.Msg) error {
}
// sync with kill goroutine
close(waitDone)
if err != nil {
sendErrorReply(s, "execve: wait4 %v", err)
} else {
switch {
case wstatus.Exited():
reply := Reply{ExitStatus: wstatus.ExitStatus()}
sendReply(s, &reply, nil)
sendReply(s, &Reply{ExitStatus: wstatus.ExitStatus()}, nil)
case wstatus.Signaled():
var status types.Status
@ -240,8 +243,7 @@ func handleExecve(s *unixsocket.Socket, cmd *Cmd, msg *unixsocket.Msg) error {
default:
status = types.StatusRE
}
reply := Reply{Status: status}
sendReply(s, &reply, nil)
sendReply(s, &Reply{Status: status}, nil)
default:
sendErrorReply(s, "execve: unknown status %v", wstatus)
}

View File

@ -7,71 +7,100 @@ import (
"github.com/criyle/go-sandbox/pkg/forkexec"
"github.com/criyle/go-sandbox/pkg/memfd"
"github.com/criyle/go-sandbox/pkg/mount"
"github.com/criyle/go-sandbox/pkg/pipe"
"github.com/criyle/go-sandbox/pkg/unixsocket"
"golang.org/x/sys/unix"
)
// Builder builds instance of container masters
type Builder struct {
// Root is container root mount path, empty uses current work path
Root string
// Mounts defines container mount points, empty uses default mounts
Mounts []mount.SyscallParams
// Stderr defines whether to collect container stderr output for debug
Stderr bool
// ExecFile defines executable that called Init, otherwise defer current
// executable (/proc/self/exe)
ExecFile *os.File
}
// Master manages single pre-forked container
type Master struct {
pid int // underlying container init pid
socket *unixsocket.Socket // master - container communication
buff *pipe.Buffer
}
// New creates new master with underlying container
func New(root string) (*Master, error) {
// dummy stdin / stdout / stderr
fnull, err := os.OpenFile(os.DevNull, os.O_RDWR, 0666)
func (b *Builder) Build() (*Master, error) {
var (
err error
buff *pipe.Buffer
)
// container mount points
mounts := b.Mounts
if len(mounts) == 0 {
if mounts, err = mount.NewBuilder().WithMounts(DefaultMounts).Build(true); err != nil {
return nil, fmt.Errorf("daemon: failed to build rootfs mount %v", err)
}
}
root := b.Root
if root == "" {
if root, err = os.Getwd(); err != nil {
return nil, fmt.Errorf("daemon: failed to get work directory %v", err)
}
}
// prepare stdin / stdout / stderr
fnull, err := os.OpenFile(os.DevNull, os.O_RDWR, os.ModePerm)
if err != nil {
return nil, fmt.Errorf("daemon: failed to open devNull(%v)", err)
}
defer fnull.Close()
// prepare self memfd
self, err := os.Open("/proc/self/exe")
if err != nil {
return nil, fmt.Errorf("daemon: failed to open /proc/self/exe(%v)", err)
files := make([]uintptr, 0, 4)
files = append(files, []uintptr{fnull.Fd(), fnull.Fd()}...)
if b.Stderr {
buff, err = pipe.NewBuffer(bufferSize)
if err != nil {
return nil, fmt.Errorf("daemon: failed to open stderr pipe: %v", err)
}
defer buff.W.Close()
files = append(files, buff.W.Fd())
} else {
files = append(files, fnull.Fd())
}
defer self.Close()
execFile, err := memfd.DupToMemfd("daemon", self)
// prepare self memfd
execFile, err := b.exec()
if err != nil {
return nil, fmt.Errorf("daemon: failed to create memfd(%v)", err)
return nil, fmt.Errorf("deamon: %v", err)
}
defer execFile.Close()
// prepare socket
ins, outs, err := unixsocket.NewSocketPair()
ins, outs, err := newPassCredSocketPair()
if err != nil {
return nil, fmt.Errorf("daemon: failed to create socket(%v)", err)
return nil, fmt.Errorf("daemon: failed to create socket: %v", err)
}
defer outs.Conn.Close()
defer outs.Close()
outf, err := outs.Conn.File()
outf, err := outs.File()
if err != nil {
ins.Conn.Close()
ins.Close()
return nil, fmt.Errorf("daemon: failed to dup file outs(%v)", err)
}
defer outf.Close()
if err = ins.SetPassCred(1); err != nil {
ins.Conn.Close()
return nil, fmt.Errorf("daemon: failed to set pass_cred ins(%v)", err)
}
if err = outs.SetPassCred(1); err != nil {
ins.Conn.Close()
return nil, fmt.Errorf("daemon: failed to set pass_cred outs(%v)", err)
}
mounts, err := mount.NewBuilder().WithMounts(DefaultMounts).Build(true)
if err != nil {
ins.Conn.Close()
return nil, fmt.Errorf("daemon: failed to build rootfs mount %v", err)
}
files = append(files, uintptr(outf.Fd()))
r := &forkexec.Runner{
Args: []string{os.Args[0], initArg},
Env: []string{DefaultPath},
ExecFile: execFile.Fd(),
Files: []uintptr{fnull.Fd(), fnull.Fd(), fnull.Fd(), uintptr(outf.Fd())},
Files: files,
WorkDir: "/w",
UnshareFlags: forkexec.UnshareFlags,
Mounts: mounts,
@ -81,16 +110,64 @@ func New(root string) (*Master, error) {
}
pid, err := r.Start()
if err != nil {
ins.Conn.Close()
ins.Close()
return nil, fmt.Errorf("daemon: failed to execve(%v)", err)
}
return &Master{pid, ins}, nil
return &Master{pid, ins, buff}, nil
}
// Destroy kill the daemon process (with container)
// if stderr enabled, collect the output as error
func (m *Master) Destroy() error {
var wstatus unix.WaitStatus
unix.Kill(m.pid, unix.SIGKILL)
_, err := unix.Wait4(m.pid, &wstatus, 0, nil)
return err
if _, err := unix.Wait4(m.pid, &wstatus, 0, nil); err != nil {
return err
}
if m.buff != nil {
<-m.buff.Done
return fmt.Errorf("destroy: %s", m.buff.Buffer.Bytes())
}
return nil
}
// exec prepares executable
func (b *Builder) exec() (*os.File, error) {
if b.ExecFile != nil {
return b.ExecFile, nil
}
return OpenCurrentExec()
}
// OpenCurrentExec opens current executable and dup it to memfd
func OpenCurrentExec() (*os.File, error) {
self, err := os.Open(currentExec)
if err != nil {
return nil, fmt.Errorf("failed to open %v: %v", currentExec, err)
}
defer self.Close()
execFile, err := memfd.DupToMemfd("daemon", self)
if err != nil {
return nil, fmt.Errorf("failed to create memfd: %v", err)
}
return execFile, nil
}
func newPassCredSocketPair() (*unixsocket.Socket, *unixsocket.Socket, error) {
ins, outs, err := unixsocket.NewSocketPair()
if err != nil {
return nil, nil, err
}
if err = ins.SetPassCred(1); err != nil {
ins.Close()
outs.Close()
return nil, nil, err
}
if err = outs.SetPassCred(1); err != nil {
ins.Close()
outs.Close()
return nil, nil, err
}
return ins, outs, nil
}

View File

@ -55,11 +55,13 @@ func (m *Master) Open(p string) (*os.File, error) {
if reply.Error != "" {
return nil, fmt.Errorf("open: %v", reply.Error)
}
if len(msg.Fds) == 0 {
return nil, fmt.Errorf("open: did not receive fd")
if len(msg.Fds) != 1 {
closeFds(msg.Fds)
return nil, fmt.Errorf("open: unexpected number of fd %v", len(msg.Fds))
}
f := os.NewFile(uintptr(msg.Fds[0]), p)
if f == nil {
closeFds(msg.Fds)
return nil, fmt.Errorf("open: failed %v", msg.Fds[0])
}
return f, nil
@ -91,31 +93,36 @@ func (m *Master) Reset() error {
func (m *Master) recvAckReply(name string) error {
reply, _, err := m.recvReply()
if err != nil {
return fmt.Errorf(name+": %v", err)
return fmt.Errorf("%v: recvAck %v", name, err)
}
if reply.Error != "" {
return fmt.Errorf(name+": %v", reply.Error)
return fmt.Errorf("%v: container error %v", name, reply.Error)
}
return nil
}
func (m *Master) recvReply() (*Reply, *unixsocket.Msg, error) {
var reply Reply
buff := GetBuffer()
defer PutBuffer(buff)
n, msg, err := m.socket.RecvMsg(buff)
if err != nil {
return nil, nil, fmt.Errorf("failed to RecvMsg(%v)", err)
}
if err := gob.NewDecoder(bytes.NewReader(buff[:n])).Decode(&reply); err != nil {
reply := new(Reply)
if err := gob.NewDecoder(bytes.NewReader(buff[:n])).Decode(reply); err != nil {
return nil, nil, fmt.Errorf("failed to decode(%v)", err)
}
return &reply, msg, nil
return reply, msg, nil
}
func (m *Master) sendCmd(cmd *Cmd, msg *unixsocket.Msg) error {
var buff bytes.Buffer
if err := gob.NewEncoder(&buff).Encode(cmd); err != nil {
buf := GetBuffer()
defer PutBuffer(buf)
// use buf pool to reduce allocation
buff := bytes.NewBuffer(buf[:0])
if err := gob.NewEncoder(buff).Encode(cmd); err != nil {
return fmt.Errorf("failed to encode(%v)", err)
}
if err := m.socket.SendMsg(buff.Bytes(), msg); err != nil {

View File

@ -44,27 +44,27 @@ func (m *Master) Execve(done <-chan struct{}, param *ExecveParam) (<-chan types.
}
reply, msg, err := m.recvReply()
if err != nil {
return nil, fmt.Errorf("execve: RecvReply %v", err)
return nil, fmt.Errorf("execve: recvReply %v", err)
}
if reply.Error != "" || msg == nil || msg.Cred == nil {
m.sendCmd(&Cmd{Cmd: cmdKill}, nil)
m.execveSyncKill()
return nil, fmt.Errorf("execve: no pid recved or error(%v)", reply.Error)
}
if param.SyncFunc != nil {
if err := param.SyncFunc(int(msg.Cred.Pid)); err != nil {
m.sendCmd(&Cmd{Cmd: cmdKill}, nil)
m.execveSyncKill()
return nil, fmt.Errorf("execve: syncfunc failed(%v)", err)
}
}
// send to syncFunc ack ok
if err := m.sendCmd(&Cmd{Cmd: cmdOk}, nil); err != nil {
return nil, fmt.Errorf("execve: ok failed(%v)", err)
}
// make sure goroutine not leaked (blocked) even if result is not consumed
wait := make(chan types.Result, 1)
result := make(chan types.Result, 1)
waitDone := make(chan struct{})
// Wait
go func() {
defer close(wait)
reply2, _, _ := m.recvReply()
close(waitDone)
// done signal (should recv after kill)
@ -74,7 +74,7 @@ func (m *Master) Execve(done <-chan struct{}, param *ExecveParam) (<-chan types.
if reply2.Error != "" {
status = types.StatusFatal
}
wait <- types.Result{
result <- types.Result{
ExitStatus: reply2.ExitStatus,
Status: status,
Error: reply2.Error,
@ -88,5 +88,14 @@ func (m *Master) Execve(done <-chan struct{}, param *ExecveParam) (<-chan types.
}
m.sendCmd(&Cmd{Cmd: cmdKill}, nil)
}()
return wait, nil
return result, nil
}
// handleSyncKill will send kill (for sync func), kill (for wait to kill all) and
// then recv error and finish signal when terminated by syncFunc error
func (m *Master) execveSyncKill() {
m.sendCmd(&Cmd{Cmd: cmdKill}, nil)
m.sendCmd(&Cmd{Cmd: cmdKill}, nil)
m.recvReply()
m.recvReply()
}

2
go.mod
View File

@ -4,5 +4,5 @@ go 1.12
require (
github.com/seccomp/libseccomp-golang v0.9.1
golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3
golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe
)

4
go.sum
View File

@ -1,4 +1,4 @@
github.com/seccomp/libseccomp-golang v0.9.1 h1:NJjM5DNFOs0s3kYE1WUOr6G8V97sdt46rlXTMfXGWBo=
github.com/seccomp/libseccomp-golang v0.9.1/go.mod h1:GbW5+tmTXfcxTToHLXlScSlAvWlF4P2Ca7zGrPiEpWo=
golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3 h1:7TYNF4UdlohbFwpNH04CoPMp1cHUZgO1Ebq5r2hIjfo=
golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe h1:6fAMxZRR6sl1Uq8U61gxU+kPTs2tR8uOySCbBP7BN/M=
golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=

View File

@ -20,7 +20,7 @@ var oobPool = sync.Pool{
// Socket represents a unix socket
type Socket struct {
Conn *net.UnixConn
*net.UnixConn
}
// Msg is the oob msg with the message
@ -67,7 +67,7 @@ func NewSocketPair() (*Socket, *Socket, error) {
}
outs, err := NewSocket(fd[1])
if err != nil {
ins.Conn.Close()
ins.Close()
syscall.Close(fd[1])
return nil, nil, fmt.Errorf("NewSocketPair: failed to call NewSocket outs(%v)", err)
}
@ -76,7 +76,7 @@ func NewSocketPair() (*Socket, *Socket, error) {
// SetPassCred set sockopt for pass cred for unix socket
func (s *Socket) SetPassCred(option int) error {
sysconn, err := s.Conn.SyscallConn()
sysconn, err := s.SyscallConn()
if err != nil {
return err
}
@ -96,7 +96,7 @@ func (s *Socket) SendMsg(b []byte, m *Msg) error {
oob = append(oob, syscall.UnixCredentials(m.Cred)...)
}
}
_, _, err := s.Conn.WriteMsgUnix(b, oob, nil)
_, _, err := s.WriteMsgUnix(b, oob, nil)
if err != nil {
return err
}
@ -107,7 +107,7 @@ func (s *Socket) SendMsg(b []byte, m *Msg) error {
func (s *Socket) RecvMsg(b []byte) (int, *Msg, error) {
oob := oobPool.Get().([]byte)
defer oobPool.Put(oob)
n, oobn, _, _, err := s.Conn.ReadMsgUnix(b, oob)
n, oobn, _, _, err := s.ReadMsgUnix(b, oob)
if err != nil {
return 0, nil, err
}