rename daemon -> container

This commit is contained in:
criyle 2020-02-19 05:36:23 -05:00
parent a24933ac82
commit f2dacb5752
16 changed files with 475 additions and 450 deletions

View File

@ -92,7 +92,7 @@ type Runner interface {
### Pre-forked Container Protocol ### Pre-forked Container Protocol
1. Pre-fork container daemons to run programs inside 1. Pre-fork container to run programs inside
2. Unix socket to pass fd inside / outside 2. Unix socket to pass fd inside / outside
Container / Master Communication Protocol (single thread): Container / Master Communication Protocol (single thread):
@ -120,7 +120,7 @@ Container / Master Communication Protocol (single thread):
- send: "kill" (as cmd) / reply: "finished" - send: "kill" (as cmd) / reply: "finished"
- reply: - reply:
Any socket related error will cause the daemon exit (with all process inside container) Any socket related error will cause the container exit (with all process inside container)
## Packages (/pkg) ## Packages (/pkg)
@ -137,7 +137,7 @@ Any socket related error will cause the daemon exit (with all process inside con
## Packages ## Packages
- config: defines arch & language specified trace condition for ptrace runner from UOJ - config: defines arch & language specified trace condition for ptrace runner from UOJ
- daemon: creates pre-forked container to run programs inside - container: creates pre-forked container to run programs inside
- runner: interface to run program - runner: interface to run program
- ptrace: wrapper to call forkexec and ptracer - ptrace: wrapper to call forkexec and ptracer
- filehandler: an example implementation of UOJ file set - filehandler: an example implementation of UOJ file set

View File

@ -10,7 +10,7 @@ import (
"time" "time"
"github.com/criyle/go-sandbox/config" "github.com/criyle/go-sandbox/config"
"github.com/criyle/go-sandbox/daemon" "github.com/criyle/go-sandbox/container"
"github.com/criyle/go-sandbox/pkg/cgroup" "github.com/criyle/go-sandbox/pkg/cgroup"
"github.com/criyle/go-sandbox/pkg/memfd" "github.com/criyle/go-sandbox/pkg/memfd"
"github.com/criyle/go-sandbox/pkg/mount" "github.com/criyle/go-sandbox/pkg/mount"
@ -44,9 +44,9 @@ func printUsage() {
os.Exit(2) os.Exit(2)
} }
// container init
func init() { func init() {
// container init container.Init()
daemon.Init()
} }
func main() { func main() {
@ -71,7 +71,7 @@ func main() {
flag.Var(&addRawWritable, "add-writable-raw", "Add a writable file (don't transform to its real path)") flag.Var(&addRawWritable, "add-writable-raw", "Add a writable file (don't transform to its real path)")
flag.BoolVar(&useCGroup, "cgroup", false, "Use cgroup to colloct resource usage") flag.BoolVar(&useCGroup, "cgroup", false, "Use cgroup to colloct resource usage")
flag.BoolVar(&memfile, "memfd", false, "Use memfd as exec file") flag.BoolVar(&memfile, "memfd", false, "Use memfd as exec file")
flag.StringVar(&runt, "runner", "ptrace", "Runner for the program (ptrace, ns, daemon)") flag.StringVar(&runt, "runner", "ptrace", "Runner for the program (ptrace, ns, container)")
flag.Parse() flag.Parse()
args = flag.Args() args = flag.Args()
@ -133,13 +133,13 @@ func main() {
} }
} }
type daemonRunner struct { type containerRunner struct {
*daemon.Master container.Environment
*daemon.ExecveParam container.ExecveParam
} }
func (r *daemonRunner) Run(c context.Context) <-chan types.Result { func (r *containerRunner) Run(c context.Context) <-chan types.Result {
return r.Master.Execve(c, r.ExecveParam) return r.Environment.Execve(c, r.ExecveParam)
} }
func start() (*types.Result, error) { func start() (*types.Result, error) {
@ -231,29 +231,29 @@ func start() (*types.Result, error) {
MemoryLimit: types.Size(memoryLimit << 20), MemoryLimit: types.Size(memoryLimit << 20),
} }
if runt == "daemon" { if runt == "container" {
root, err := ioutil.TempDir("", "dm") root, err := ioutil.TempDir("", "dm")
if err != nil { if err != nil {
return nil, fmt.Errorf("cannot make temp root for daemon namespace: %v", err) return nil, fmt.Errorf("cannot make temp root for container namespace: %v", err)
} }
defer os.RemoveAll(root) defer os.RemoveAll(root)
b := daemon.Builder{ b := container.Builder{
Root: root, Root: root,
} }
m, err := b.Build() m, err := b.Build()
if err != nil { if err != nil {
return nil, fmt.Errorf("failed to new master: %v", err) return nil, fmt.Errorf("failed to new container: %v", err)
} }
defer m.Destroy() defer m.Destroy()
err = m.Ping() err = m.Ping()
if err != nil { if err != nil {
return nil, fmt.Errorf("failed to ping daemon: %v", err) return nil, fmt.Errorf("failed to ping container: %v", err)
} }
runner = &daemonRunner{ runner = &containerRunner{
Master: m, Environment: m,
ExecveParam: &daemon.ExecveParam{ ExecveParam: container.ExecveParam{
Args: args, Args: args,
Env: []string{pathEnv}, Env: []string{pathEnv},
Fds: fds, Fds: fds,

View File

@ -1,4 +1,4 @@
package daemon package container
const ( const (
cmdPing = "ping" cmdPing = "ping"
@ -18,7 +18,7 @@ const (
containerUID = 1000 containerUID = 1000
containerGID = 1000 containerGID = 1000
containerName = "daemon" containerName = "go-sandbox"
containerWD = "/w" containerWD = "/w"
containerMaxProc = 1 containerMaxProc = 1

View File

@ -0,0 +1,86 @@
package container
import (
"fmt"
"os"
"syscall"
"github.com/criyle/go-sandbox/pkg/unixsocket"
)
func (c *containerServer) handlePing() error {
return c.sendReply(&reply{}, nil)
}
func (c *containerServer) handleConf(conf *confCmd) error {
if conf != nil {
c.containerConfig = conf.Conf
}
return c.sendReply(&reply{}, nil)
}
func (c *containerServer) handleOpen(open []OpenCmd) error {
if len(open) == 0 {
return c.sendErrorReply("open: no open parameter received")
}
// open files
fds := make([]int, 0, len(open))
for _, o := range open {
outFile, err := os.OpenFile(o.Path, o.Flag, o.Perm)
if err != nil {
return c.sendErrorReply("open: %v", err)
}
defer outFile.Close()
fds = append(fds, int(outFile.Fd()))
}
return c.sendReply(&reply{}, &unixsocket.Msg{Fds: fds})
}
func (c *containerServer) handleDelete(delete *deleteCmd) error {
if delete == nil {
return c.sendErrorReply("delete: no parameter provided")
}
if err := os.Remove(delete.Path); err != nil {
return c.sendErrorReply("delete: %v", err)
}
return c.sendReply(&reply{}, nil)
}
func (c *containerServer) handleReset() error {
if err := removeContents("/tmp"); err != nil {
return c.sendErrorReply("reset: /tmp %v", err)
}
if err := removeContents("/w"); err != nil {
return c.sendErrorReply("reset: /w %v", err)
}
return c.sendReply(&reply{}, nil)
}
func (c *containerServer) recvCmd() (*cmd, *unixsocket.Msg, error) {
cm := new(cmd)
msg, err := (*socket)(c.socket).RecvMsg(cm)
if err != nil {
return nil, nil, err
}
return cm, msg, nil
}
func (c *containerServer) sendReply(rep *reply, msg *unixsocket.Msg) error {
return (*socket)(c.socket).SendMsg(rep, msg)
}
// sendErrorReply sends error reply
func (c *containerServer) sendErrorReply(ft string, v ...interface{}) error {
errRep := &errorReply{
Msg: fmt.Sprintf(ft, v...),
}
// store errno
if len(v) == 1 {
if errno, ok := v[0].(syscall.Errno); ok {
errRep.Errno = &errno
}
}
return c.sendReply(&reply{Error: errRep}, nil)
}

View File

@ -1,4 +1,4 @@
package daemon package container
import ( import (
"fmt" "fmt"
@ -10,7 +10,7 @@ import (
"github.com/criyle/go-sandbox/types" "github.com/criyle/go-sandbox/types"
) )
func (c *containerServer) handleExecve(cmd *ExecCmd, msg *unixsocket.Msg) error { func (c *containerServer) handleExecve(cmd *execCmd, msg *unixsocket.Msg) error {
var ( var (
files []uintptr files []uintptr
execFile uintptr execFile uintptr
@ -37,21 +37,21 @@ func (c *containerServer) handleExecve(cmd *ExecCmd, msg *unixsocket.Msg) error
} }
syncFunc := func(pid int) error { syncFunc := func(pid int) error {
msg2 := &unixsocket.Msg{ msg := &unixsocket.Msg{
Cred: &syscall.Ucred{ Cred: &syscall.Ucred{
Pid: int32(pid), Pid: int32(pid),
Uid: uint32(syscall.Getuid()), Uid: uint32(syscall.Getuid()),
Gid: uint32(syscall.Getgid()), Gid: uint32(syscall.Getgid()),
}, },
} }
if err2 := c.sendReply(&Reply{}, msg2); err2 != nil { if err := c.sendReply(&reply{}, msg); err != nil {
return fmt.Errorf("syncFunc: sendReply(%v)", err2) return fmt.Errorf("syncFunc: sendReply %w", err)
} }
cmd2, _, err2 := c.recvCmd() cmd, _, err := c.recvCmd()
if err2 != nil { if err != nil {
return fmt.Errorf("syncFunc: recvCmd(%v)", err2) return fmt.Errorf("syncFunc: recvCmd %w", err)
} }
if cmd2.Cmd == cmdKill { if cmd.Cmd == cmdKill {
return fmt.Errorf("syncFunc: received kill") return fmt.Errorf("syncFunc: received kill")
} }
return nil return nil
@ -121,8 +121,11 @@ func (c *containerServer) handleExecve(cmd *ExecCmd, msg *unixsocket.Msg) error
switch { switch {
case wstatus.Exited(): case wstatus.Exited():
exitStatus := wstatus.ExitStatus() exitStatus := wstatus.ExitStatus()
c.sendReply(&Reply{ if exitStatus != 0 {
ExecReply: &ExecReply{ status = types.StatusNonzeroExitStatus
}
c.sendReply(&reply{
ExecReply: &execReply{
Status: status, Status: status,
ExitStatus: exitStatus, ExitStatus: exitStatus,
Time: userTime, Time: userTime,
@ -142,8 +145,8 @@ func (c *containerServer) handleExecve(cmd *ExecCmd, msg *unixsocket.Msg) error
default: default:
status = types.StatusSignalled status = types.StatusSignalled
} }
c.sendReply(&Reply{ c.sendReply(&reply{
ExecReply: &ExecReply{ ExecReply: &execReply{
ExitStatus: int(wstatus.Signal()), ExitStatus: int(wstatus.Signal()),
Status: status, Status: status,
Time: userTime, Time: userTime,
@ -158,5 +161,5 @@ func (c *containerServer) handleExecve(cmd *ExecCmd, msg *unixsocket.Msg) error
// wait for kill msg and reply done for finish // wait for kill msg and reply done for finish
<-killDone <-killDone
return c.sendReply(&Reply{}, nil) return c.sendReply(&reply{}, nil)
} }

View File

@ -0,0 +1,93 @@
package container
import (
"fmt"
"os"
"runtime"
"github.com/criyle/go-sandbox/pkg/unixsocket"
)
type containerServer struct {
socket *unixsocket.Socket
containerConfig
}
// Init is called for container init process
// it will check if pid == 1, otherwise it is noop
// Init will do infinite loop on socket commands,
// and exits when at socket close, use it in init function
func Init() (err error) {
// noop if self is not container init process
// Notice: docker init is also 1, additional check for args[1] == init
if os.Getpid() != 1 || len(os.Args) != 2 || os.Args[1] != initArg {
return nil
}
// exit process (with whole container) upon exit this function
// possible reason:
// 1. socket broken (parent exit)
// 2. panic
// 3. undefined cmd (possible race condition)
defer func() {
if err := recover(); err != nil {
fmt.Fprintf(os.Stderr, "container_exit: panic: %v\n", err)
os.Exit(1)
}
if err != nil {
fmt.Fprintf(os.Stderr, "container_exit: %v\n", err)
os.Exit(1)
}
fmt.Fprintf(os.Stderr, "container_exit\n")
os.Exit(0)
}()
// limit container resource usage
runtime.GOMAXPROCS(containerMaxProc)
// new_master shared the socket at fd 3 (marked close_exec)
const defaultFd = 3
soc, err := unixsocket.NewSocket(defaultFd)
if err != nil {
return fmt.Errorf("container_init: failed to new socket %w", err)
}
// serve forever
cs := &containerServer{socket: soc}
return cs.serve()
}
func (c *containerServer) serve() error {
for {
cmd, msg, err := c.recvCmd()
if err != nil {
return fmt.Errorf("serve: recvCmd %w", err)
}
if err := c.handleCmd(cmd, msg); err != nil {
return fmt.Errorf("serve: failed to execute cmd %w", err)
}
}
}
func (c *containerServer) handleCmd(cmd *cmd, msg *unixsocket.Msg) error {
switch cmd.Cmd {
case cmdPing:
return c.handlePing()
case cmdConf:
return c.handleConf(cmd.ConfCmd)
case cmdOpen:
return c.handleOpen(cmd.OpenCmd)
case cmdDelete:
return c.handleDelete(cmd.DeleteCmd)
case cmdReset:
return c.handleReset()
case cmdExecve:
return c.handleExecve(cmd.ExecCmd, msg)
}
return fmt.Errorf("unknown command: %s", cmd.Cmd)
}

44
container/doc.go Normal file
View File

@ -0,0 +1,44 @@
// Package container provides pre-forked container environment to
// run programs in isolated Linux namespaces.
//
// Overview
//
// It creates container within unshared container and communicate
// with host process using unix socket with
// oob for fd / pid and commands encoded by gob.
//
// Protocol
//
// Host to container communication protocol is single threaded and always initiated by
// the host:
//
// - ping (alive check):
// - reply: pong
//
// - conf (set configuration):
// - reply pong
//
// - open (open files in given mode inside container):
// - send: []OpenCmd
// - reply: "success", file fds / "error"
//
// - delete (unlink file / rmdir dir inside container):
// - send: path
// - reply: "finished" / "error"
//
// - reset (clean up container for later use (clear workdir / tmp)):
// - send:
// - reply: "success"
//
// - execve: (execute file inside container):
// - send: argv, env, rLimits, fds
// - reply:
// - success: "success", pid
// - failed: "failed"
// - send (success): "init_finished" (as cmd)
// - reply: "finished" / send: "kill" (as cmd)
// - send: "kill" (as cmd) / reply: "finished"
// - reply:
//
// Any socket related error will cause the container exit with all process inside container
package container

View File

@ -1,6 +1,7 @@
package daemon package container
import ( import (
"context"
"fmt" "fmt"
"os" "os"
"sync" "sync"
@ -10,13 +11,14 @@ import (
"github.com/criyle/go-sandbox/pkg/memfd" "github.com/criyle/go-sandbox/pkg/memfd"
"github.com/criyle/go-sandbox/pkg/mount" "github.com/criyle/go-sandbox/pkg/mount"
"github.com/criyle/go-sandbox/pkg/unixsocket" "github.com/criyle/go-sandbox/pkg/unixsocket"
"github.com/criyle/go-sandbox/types"
"golang.org/x/sys/unix" "golang.org/x/sys/unix"
) )
// DefaultPath defines path evironment variable for container init process // PathEnv defines path environment variable for the container init process
const DefaultPath = "PATH=/usr/local/bin:/usr/bin:/bin" const PathEnv = "PATH=/usr/local/bin:/usr/bin:/bin"
// Builder builds instance of container masters // Builder builds instance of container environment
type Builder struct { type Builder struct {
// Root is container root mount path, empty uses current work path // Root is container root mount path, empty uses current work path
Root string Root string
@ -41,15 +43,25 @@ type CredGenerator interface {
Get() syscall.Credential Get() syscall.Credential
} }
// Master manages single pre-forked container // Environment holds single progrem containered environment
type Master struct { type Environment interface {
Ping() error
Open([]OpenCmd) ([]*os.File, error)
Delete(p string) error
Reset() error
Execve(context.Context, ExecveParam) <-chan types.Result
Destroy() error
}
// container manages single pre-forked container environment
type container struct {
pid int // underlying container init pid pid int // underlying container init pid
socket *unixsocket.Socket // master - container communication socket *unixsocket.Socket // master - container communication
mu sync.Mutex // lock to avoid race condition mu sync.Mutex // lock to avoid race condition
} }
// Build creates new master with underlying container // Build creates new environment with underlying container
func (b *Builder) Build() (*Master, error) { func (b *Builder) Build() (Environment, error) {
var ( var (
err error err error
cred syscall.Credential cred syscall.Credential
@ -64,19 +76,22 @@ func (b *Builder) Build() (*Master, error) {
WithTmpfs("w", ""). // work dir WithTmpfs("w", ""). // work dir
WithTmpfs("tmp", ""). // tmp WithTmpfs("tmp", ""). // tmp
Build(true); err != nil { Build(true); err != nil {
return nil, fmt.Errorf("daemon: failed to build rootfs mount %v", err) return nil, fmt.Errorf("container: failed to build rootfs mount %w", err)
} }
} }
// container root directory on the host
root := b.Root root := b.Root
if root == "" { if root == "" {
if root, err = os.Getwd(); err != nil { if root, err = os.Getwd(); err != nil {
return nil, fmt.Errorf("daemon: failed to get work directory %v", err) return nil, fmt.Errorf("container: failed to get work directory %w", err)
} }
} }
// prepare stdin / stdout / stderr // prepare stdin / stdout / stderr
devNull, err := os.OpenFile(os.DevNull, os.O_RDWR, os.ModePerm) devNull, err := os.OpenFile(os.DevNull, os.O_RDWR, os.ModePerm)
if err != nil { if err != nil {
return nil, fmt.Errorf("daemon: failed to open devNull(%v)", err) return nil, fmt.Errorf("container: failed to open devNull %w", err)
} }
defer devNull.Close() defer devNull.Close()
@ -88,30 +103,30 @@ func (b *Builder) Build() (*Master, error) {
files = append(files, devNull.Fd()) files = append(files, devNull.Fd())
} }
// prepare self memfd // prepare container exec file
execFile, err := b.exec() execFile, err := b.exec()
if err != nil { if err != nil {
return nil, fmt.Errorf("deamon: %v", err) return nil, fmt.Errorf("container: prepare exec %w", err)
} }
defer execFile.Close() defer execFile.Close()
// prepare socket // prepare host <-> container unix socket
ins, outs, err := newPassCredSocketPair() ins, outs, err := newPassCredSocketPair()
if err != nil { if err != nil {
return nil, fmt.Errorf("daemon: failed to create socket: %v", err) return nil, fmt.Errorf("container: failed to create socket: %w", err)
} }
defer outs.Close() defer outs.Close()
outf, err := outs.File() outf, err := outs.File()
if err != nil { if err != nil {
ins.Close() ins.Close()
return nil, fmt.Errorf("daemon: failed to dup file outs(%v)", err) return nil, fmt.Errorf("container: failed to dup container socket fd %w", err)
} }
defer outf.Close() defer outf.Close()
files = append(files, uintptr(outf.Fd())) files = append(files, uintptr(outf.Fd()))
// prepare credential // prepare container running credential
if b.CredGenerator != nil { if b.CredGenerator != nil {
cred = b.CredGenerator.Get() cred = b.CredGenerator.Get()
uidMap, gidMap = getIDMapping(&cred) uidMap, gidMap = getIDMapping(&cred)
@ -119,7 +134,7 @@ func (b *Builder) Build() (*Master, error) {
r := &forkexec.Runner{ r := &forkexec.Runner{
Args: []string{os.Args[0], initArg}, Args: []string{os.Args[0], initArg},
Env: []string{DefaultPath}, Env: []string{PathEnv},
ExecFile: execFile.Fd(), ExecFile: execFile.Fd(),
Files: files, Files: files,
WorkDir: containerWD, WorkDir: containerWD,
@ -134,41 +149,42 @@ func (b *Builder) Build() (*Master, error) {
pid, err := r.Start() pid, err := r.Start()
if err != nil { if err != nil {
ins.Close() ins.Close()
return nil, fmt.Errorf("daemon: failed to execve(%v)", err) return nil, fmt.Errorf("container: failed to start container %w", err)
} }
m := &Master{
c := &container{
pid: pid, pid: pid,
socket: ins, socket: ins,
} }
// set configuration and check if container creation successful // set configuration and check if container creation successful
if err = m.conf(&containerConfig{ if err = c.conf(&containerConfig{
Cred: b.CredGenerator != nil, Cred: b.CredGenerator != nil,
}); err != nil { }); err != nil {
m.Destroy() c.Destroy()
return nil, err return nil, err
} }
return m, nil return c, nil
} }
// Destroy kill the daemon process (with container) // Destroy kill the container process (with its children)
// if stderr enabled, collect the output as error // if stderr enabled, collect the output as error
func (m *Master) Destroy() error { func (c *container) Destroy() error {
// close socket (abort any ongoing command) // close socket (abort any ongoing command)
m.socket.Close() c.socket.Close()
// wait commands terminates // wait commands terminates
m.mu.Lock() c.mu.Lock()
defer m.mu.Unlock() defer c.mu.Unlock()
// kill process // kill process
var wstatus unix.WaitStatus var wstatus unix.WaitStatus
unix.Kill(m.pid, unix.SIGKILL) unix.Kill(c.pid, unix.SIGKILL)
// wait for container process to exit // wait for container process to exit
_, err := unix.Wait4(m.pid, &wstatus, 0, nil) _, err := unix.Wait4(c.pid, &wstatus, 0, nil)
for err == unix.EINTR { for err == unix.EINTR {
_, err = unix.Wait4(m.pid, &wstatus, 0, nil) _, err = unix.Wait4(c.pid, &wstatus, 0, nil)
} }
return err return err
} }
@ -181,7 +197,7 @@ func (b *Builder) exec() (*os.File, error) {
return OpenCurrentExec() return OpenCurrentExec()
} }
// OpenCurrentExec opens current executable and dup it to memfd // OpenCurrentExec opens current executable and dup and seal it to memfd
func OpenCurrentExec() (*os.File, error) { func OpenCurrentExec() (*os.File, error) {
self, err := os.Open(currentExec) self, err := os.Open(currentExec)
if err != nil { if err != nil {
@ -189,7 +205,7 @@ func OpenCurrentExec() (*os.File, error) {
} }
defer self.Close() defer self.Close()
execFile, err := memfd.DupToMemfd("daemon", self) execFile, err := memfd.DupToMemfd("init", self)
if err != nil { if err != nil {
return nil, fmt.Errorf("failed to create memfd: %v", err) return nil, fmt.Errorf("failed to create memfd: %v", err)
} }

View File

@ -1,4 +1,4 @@
package daemon package container
import ( import (
"fmt" "fmt"
@ -9,55 +9,55 @@ import (
) )
// Ping send ping message to container // Ping send ping message to container
func (m *Master) Ping() error { func (c *container) Ping() error {
m.mu.Lock() c.mu.Lock()
defer m.mu.Unlock() defer c.mu.Unlock()
// avoid infinite wait (max 3s) // avoid infinite wait (max 3s)
const pingWait = 3 * time.Second const pingWait = 3 * time.Second
m.socket.SetDeadline(time.Now().Add(pingWait)) c.socket.SetDeadline(time.Now().Add(pingWait))
defer m.socket.SetDeadline(time.Time{}) defer c.socket.SetDeadline(time.Time{})
// send ping // send ping
cmd := Cmd{ cmd := cmd{
Cmd: cmdPing, Cmd: cmdPing,
} }
if err := m.sendCmd(&cmd, nil); err != nil { if err := c.sendCmd(&cmd, nil); err != nil {
return fmt.Errorf("ping: %v", err) return fmt.Errorf("ping: %v", err)
} }
// receive no error // receive no error
return m.recvAckReply("ping") return c.recvAckReply("ping")
} }
// conf send configuration to container (used by builder only) // conf send configuration to container (used by builder only)
func (m *Master) conf(conf *containerConfig) error { func (c *container) conf(conf *containerConfig) error {
m.mu.Lock() c.mu.Lock()
defer m.mu.Unlock() defer c.mu.Unlock()
cmd := Cmd{ cmd := cmd{
Cmd: cmdConf, Cmd: cmdConf,
ConfCmd: &ConfCmd{Conf: *conf}, ConfCmd: &confCmd{Conf: *conf},
} }
if err := m.sendCmd(&cmd, nil); err != nil { if err := c.sendCmd(&cmd, nil); err != nil {
return fmt.Errorf("conf: %v", err) return fmt.Errorf("conf: %v", err)
} }
return m.recvAckReply("conf") return c.recvAckReply("conf")
} }
// Open open files in container // Open open files in container
func (m *Master) Open(p []OpenCmd) ([]*os.File, error) { func (c *container) Open(p []OpenCmd) ([]*os.File, error) {
m.mu.Lock() c.mu.Lock()
defer m.mu.Unlock() defer c.mu.Unlock()
// send copyin // send copyin
cmd := Cmd{ cmd := cmd{
Cmd: cmdOpen, Cmd: cmdOpen,
OpenCmd: p, OpenCmd: p,
} }
if err := m.sendCmd(&cmd, nil); err != nil { if err := c.sendCmd(&cmd, nil); err != nil {
return nil, fmt.Errorf("open: %v", err) return nil, fmt.Errorf("open: %v", err)
} }
reply, msg, err := m.recvReply() reply, msg, err := c.recvReply()
if err != nil { if err != nil {
return nil, fmt.Errorf("open: %v", err) return nil, fmt.Errorf("open: %v", err)
} }
@ -82,36 +82,36 @@ func (m *Master) Open(p []OpenCmd) ([]*os.File, error) {
} }
// Delete remove file from container // Delete remove file from container
func (m *Master) Delete(p string) error { func (c *container) Delete(p string) error {
m.mu.Lock() c.mu.Lock()
defer m.mu.Unlock() defer c.mu.Unlock()
cmd := Cmd{ cmd := cmd{
Cmd: cmdDelete, Cmd: cmdDelete,
DeleteCmd: &DeleteCmd{Path: p}, DeleteCmd: &deleteCmd{Path: p},
} }
if err := m.sendCmd(&cmd, nil); err != nil { if err := c.sendCmd(&cmd, nil); err != nil {
return fmt.Errorf("delete: %v", err) return fmt.Errorf("delete: %v", err)
} }
return m.recvAckReply("delete") return c.recvAckReply("delete")
} }
// Reset remove all from /tmp and /w // Reset remove all from /tmp and /w
func (m *Master) Reset() error { func (c *container) Reset() error {
m.mu.Lock() c.mu.Lock()
defer m.mu.Unlock() defer c.mu.Unlock()
cmd := Cmd{ cmd := cmd{
Cmd: cmdReset, Cmd: cmdReset,
} }
if err := m.sendCmd(&cmd, nil); err != nil { if err := c.sendCmd(&cmd, nil); err != nil {
return fmt.Errorf("reset: %v", err) return fmt.Errorf("reset: %v", err)
} }
return m.recvAckReply("reset") return c.recvAckReply("reset")
} }
func (m *Master) recvAckReply(name string) error { func (c *container) recvAckReply(name string) error {
reply, _, err := m.recvReply() reply, _, err := c.recvReply()
if err != nil { if err != nil {
return fmt.Errorf("%v: recvAck %v", name, err) return fmt.Errorf("%v: recvAck %v", name, err)
} }
@ -121,15 +121,15 @@ func (m *Master) recvAckReply(name string) error {
return nil return nil
} }
func (m *Master) recvReply() (*Reply, *unixsocket.Msg, error) { func (c *container) recvReply() (*reply, *unixsocket.Msg, error) {
reply := new(Reply) reply := new(reply)
msg, err := (*socket)(m.socket).RecvMsg(reply) msg, err := (*socket)(c.socket).RecvMsg(reply)
if err != nil { if err != nil {
return nil, nil, err return nil, nil, err
} }
return reply, msg, nil return reply, msg, nil
} }
func (m *Master) sendCmd(cmd *Cmd, msg *unixsocket.Msg) error { func (c *container) sendCmd(cmd *cmd, msg *unixsocket.Msg) error {
return (*socket)(m.socket).SendMsg(cmd, msg) return (*socket)(c.socket).SendMsg(cmd, msg)
} }

View File

@ -1,4 +1,4 @@
package daemon package container
import ( import (
"context" "context"
@ -16,20 +16,19 @@ type ExecveParam struct {
Env []string Env []string
Fds []uintptr Fds []uintptr
// fexecve fd // fd parameter of fexecve
ExecFile uintptr ExecFile uintptr
// POSIX Resource limit set by set rlimit // POSIX Resource limit set by set rlimit
RLimits []rlimit.RLimit RLimits []rlimit.RLimit
// SyncFunc called with pid before execve (e.g. add to cgroup) // SyncFunc called with pid before execve (for adding the process to cgroups)
SyncFunc func(pid int) error SyncFunc func(pid int) error
} }
// Execve runs process inside container // Execve runs process inside container. It accepts context cancelation as time limit exceeded.
// accepts context for cancelation func (c *container) Execve(ctx context.Context, param ExecveParam) <-chan types.Result {
func (m *Master) Execve(c context.Context, param *ExecveParam) <-chan types.Result { c.mu.Lock()
m.mu.Lock()
sTime := time.Now() sTime := time.Now()
@ -53,46 +52,46 @@ func (m *Master) Execve(c context.Context, param *ExecveParam) <-chan types.Resu
msg := &unixsocket.Msg{ msg := &unixsocket.Msg{
Fds: files, Fds: files,
} }
execCmd := &ExecCmd{ execCmd := &execCmd{
Argv: param.Args, Argv: param.Args,
Env: param.Env, Env: param.Env,
RLimits: param.RLimits, RLimits: param.RLimits,
FdExec: param.ExecFile > 0, FdExec: param.ExecFile > 0,
} }
cmd := Cmd{ cm := cmd{
Cmd: cmdExecve, Cmd: cmdExecve,
ExecCmd: execCmd, ExecCmd: execCmd,
} }
if err := m.sendCmd(&cmd, msg); err != nil { if err := c.sendCmd(&cm, msg); err != nil {
m.mu.Unlock() c.mu.Unlock()
return errResult("execve: sendCmd %v", err) return errResult("execve: sendCmd %v", err)
} }
// sync function // sync function
reply, msg, err := m.recvReply() reply, msg, err := c.recvReply()
if err != nil { if err != nil {
m.mu.Unlock() c.mu.Unlock()
return errResult("execve: recvReply %v", err) return errResult("execve: recvReply %v", err)
} }
// if sync function did not involved // if sync function did not involved
if reply.Error != nil || msg == nil || msg.Cred == nil { if reply.Error != nil || msg == nil || msg.Cred == nil {
// tell kill function to exit and sync // tell kill function to exit and sync
m.execveSyncKill() c.execveSyncKill()
m.mu.Unlock() c.mu.Unlock()
return errResult("execve: no pid received or error %v", reply.Error) return errResult("execve: no pid received or error %v", reply.Error)
} }
if param.SyncFunc != nil { if param.SyncFunc != nil {
if err := param.SyncFunc(int(msg.Cred.Pid)); err != nil { if err := param.SyncFunc(int(msg.Cred.Pid)); err != nil {
// tell sync function to exit and recv error // tell sync function to exit and recv error
m.execveSyncKill() c.execveSyncKill()
// tell kill function to exit and sync // tell kill function to exit and sync
m.execveSyncKill() c.execveSyncKill()
m.mu.Unlock() c.mu.Unlock()
return errResult("execve: syncfunc failed %v", err) return errResult("execve: syncfunc failed %v", err)
} }
} }
// send to syncFunc ack ok // send to syncFunc ack ok
if err := m.sendCmd(&Cmd{Cmd: cmdOk}, nil); err != nil { if err := c.sendCmd(&cmd{Cmd: cmdOk}, nil); err != nil {
m.mu.Unlock() c.mu.Unlock()
return errResult("execve: ack failed %v", err) return errResult("execve: ack failed %v", err)
} }
@ -102,12 +101,12 @@ func (m *Master) Execve(c context.Context, param *ExecveParam) <-chan types.Resu
// Wait // Wait
go func() { go func() {
reply2, _, err := m.recvReply() reply2, _, err := c.recvReply()
close(waitDone) close(waitDone)
// done signal (should recv after kill) // done signal (should recv after kill)
m.recvReply() c.recvReply()
// unlock after last read / write // unlock after last read / write
m.mu.Unlock() c.mu.Unlock()
// handle potential error // handle potential error
if err != nil { if err != nil {
@ -145,17 +144,17 @@ func (m *Master) Execve(c context.Context, param *ExecveParam) <-chan types.Resu
// Kill (if wait is done, a kill message need to be send to collect zombies) // Kill (if wait is done, a kill message need to be send to collect zombies)
go func() { go func() {
select { select {
case <-c.Done(): case <-ctx.Done():
case <-waitDone: case <-waitDone:
} }
m.sendCmd(&Cmd{Cmd: cmdKill}, nil) c.sendCmd(&cmd{Cmd: cmdKill}, nil)
}() }()
return result return result
} }
// execveSyncKill will send kill and recv reply // execveSyncKill will send kill and recv reply
func (m *Master) execveSyncKill() { func (c *container) execveSyncKill() {
m.sendCmd(&Cmd{Cmd: cmdKill}, nil) c.sendCmd(&cmd{Cmd: cmdKill}, nil)
m.recvReply() c.recvReply()
} }

74
container/protocol.go Normal file
View File

@ -0,0 +1,74 @@
package container
import (
"os"
"syscall"
"time"
"github.com/criyle/go-sandbox/pkg/rlimit"
"github.com/criyle/go-sandbox/types"
)
// cmd is the control message send into container
type cmd struct {
Cmd string // type of the cmd
OpenCmd []OpenCmd // open argument
DeleteCmd *deleteCmd // delete argument
ExecCmd *execCmd // execve argument
ConfCmd *confCmd // to set configuration
}
// OpenCmd correspond to a single open syscall
type OpenCmd struct {
Path string
Flag int
Perm os.FileMode
}
// deleteCmd stores delete command
type deleteCmd struct {
Path string
}
// execCmd stores execve parameter
type execCmd struct {
Argv []string // execve argv
Env []string // execve env
RLimits []rlimit.RLimit // execve posix rlimit
FdExec bool // if use fexecve (fd[0] as exec)
}
// confCmd stores conf parameter
type confCmd struct {
Conf containerConfig
}
// ContainerConfig set the container config
type containerConfig struct {
Cred bool
}
// reply is the reply message send back to controller
type reply struct {
Error *errorReply // nil if no error
ExecReply *execReply
}
// errorReply stores error returned back from container
type errorReply struct {
Msg string
Errno *syscall.Errno
}
// execReply stores execve result
type execReply struct {
ExitStatus int // waitpid exit status
Status types.Status // return status
Time time.Duration // waitpid user CPU (ns)
Memory types.Size // waitpid user memory (byte)
}
func (e *errorReply) Error() string {
return e.Msg
}

View File

@ -1,19 +1,30 @@
package daemon package container
import ( import (
"bytes" "bytes"
"encoding/gob" "encoding/gob"
"fmt" "fmt"
"sync"
"github.com/criyle/go-sandbox/pkg/unixsocket" "github.com/criyle/go-sandbox/pkg/unixsocket"
) )
// 16k buffsize
const bufferSize = 16 << 10
var bufferPool = sync.Pool{
New: func() interface{} {
return make([]byte, bufferSize)
},
}
type socket unixsocket.Socket type socket unixsocket.Socket
func (s *socket) RecvMsg(e interface{}) (*unixsocket.Msg, error) { func (s *socket) RecvMsg(e interface{}) (*unixsocket.Msg, error) {
soc := (*unixsocket.Socket)(s) soc := (*unixsocket.Socket)(s)
buff := GetBuffer()
defer PutBuffer(buff) buff := bufferPool.Get().([]byte)
defer bufferPool.Put(buff)
n, msg, err := soc.RecvMsg(buff) n, msg, err := soc.RecvMsg(buff)
if err != nil { if err != nil {
@ -21,24 +32,25 @@ func (s *socket) RecvMsg(e interface{}) (*unixsocket.Msg, error) {
} }
if err := gob.NewDecoder(bytes.NewReader(buff[:n])).Decode(e); err != nil { if err := gob.NewDecoder(bytes.NewReader(buff[:n])).Decode(e); err != nil {
return nil, fmt.Errorf("RecvMsg: failed to decode %v", err) return nil, fmt.Errorf("RecvMsg: failed to decode %w", err)
} }
return msg, nil return msg, nil
} }
func (s *socket) SendMsg(e interface{}, msg *unixsocket.Msg) error { func (s *socket) SendMsg(e interface{}, msg *unixsocket.Msg) error {
soc := (*unixsocket.Socket)(s) soc := (*unixsocket.Socket)(s)
buf := GetBuffer()
defer PutBuffer(buf) buf := bufferPool.Get().([]byte)
defer bufferPool.Put(buf)
// use buf pool to reduce allocation // use buf pool to reduce allocation
buff := bytes.NewBuffer(buf[:0]) buff := bytes.NewBuffer(buf[:0])
if err := gob.NewEncoder(buff).Encode(e); err != nil { if err := gob.NewEncoder(buff).Encode(e); err != nil {
return fmt.Errorf("SendMsg: failed to encode %v", err) return fmt.Errorf("SendMsg: failed to encode %w", err)
} }
if err := soc.SendMsg(buff.Bytes(), msg); err != nil { if err := soc.SendMsg(buff.Bytes(), msg); err != nil {
return fmt.Errorf("SendMsg: failed to SendMsg %v", err) return fmt.Errorf("SendMsg: failed to SendMsg %w", err)
} }
return nil return nil
} }

View File

@ -1,4 +1,4 @@
package daemon package container
import ( import (
"os" "os"

View File

@ -1,176 +0,0 @@
package daemon
import (
"fmt"
"os"
"runtime"
"syscall"
"github.com/criyle/go-sandbox/pkg/unixsocket"
)
type containerServer struct {
socket *unixsocket.Socket
containerConfig
}
// ContainerConfig set the container config
type containerConfig struct {
Cred bool
}
// Init is called for container init process
// it will check if pid == 1, otherwise it is noop
// Init will do infinite loop on socket commands,
// and exits when at socket close, use it in init function
func Init() (err error) {
// noop if self is not container init process
// Notice: docker init is also 1, additional check for args[1] == init
if os.Getpid() != 1 || len(os.Args) != 2 || os.Args[1] != initArg {
return nil
}
// exit process (with whole container) upon exit this function
// possible reason:
// 1. socket broken (parent exit)
// 2. panic
// 3. undefined cmd (possible race condition)
defer func() {
if err2 := recover(); err2 != nil {
fmt.Fprintf(os.Stderr, "container_panic: %v\n", err)
os.Exit(1)
}
if err != nil {
fmt.Fprintf(os.Stderr, "container_exit: %v\n", err)
os.Exit(1)
}
fmt.Fprintf(os.Stderr, "container_exit\n")
os.Exit(0)
}()
// limit container resource usage
runtime.GOMAXPROCS(containerMaxProc)
// new_master shared the socket at fd 3 (marked close_exec)
const defaultFd = 3
soc, err := unixsocket.NewSocket(defaultFd)
if err != nil {
return fmt.Errorf("container_init: failed to new socket(%v)", err)
}
// serve forever
cs := &containerServer{socket: soc}
return cs.serve()
}
func (c *containerServer) serve() error {
for {
cmd, msg, err := c.recvCmd()
if err != nil {
return fmt.Errorf("serve: %v", err)
}
if err := c.handleCmd(cmd, msg); err != nil {
return fmt.Errorf("serve: failed to execute cmd %v", err)
}
}
}
func (c *containerServer) handleCmd(cmd *Cmd, msg *unixsocket.Msg) error {
switch cmd.Cmd {
case cmdPing:
return c.handlePing()
case cmdConf:
return c.handleConf(cmd.ConfCmd)
case cmdOpen:
return c.handleOpen(cmd.OpenCmd)
case cmdDelete:
return c.handleDelete(cmd.DeleteCmd)
case cmdReset:
return c.handleReset()
case cmdExecve:
return c.handleExecve(cmd.ExecCmd, msg)
}
return fmt.Errorf("unknown command: %v", cmd.Cmd)
}
func (c *containerServer) handlePing() error {
return c.sendReply(&Reply{}, nil)
}
func (c *containerServer) handleConf(conf *ConfCmd) error {
if conf != nil {
c.containerConfig = conf.Conf
}
return c.sendReply(&Reply{}, nil)
}
func (c *containerServer) handleOpen(open []OpenCmd) error {
if len(open) == 0 {
return c.sendErrorReply("open: no open parameter received")
}
// open files
fds := make([]int, 0, len(open))
for _, o := range open {
outFile, err := os.OpenFile(o.Path, o.Flag, o.Perm)
if err != nil {
return c.sendErrorReply("open: %v", err)
}
defer outFile.Close()
fds = append(fds, int(outFile.Fd()))
}
return c.sendReply(&Reply{}, &unixsocket.Msg{Fds: fds})
}
func (c *containerServer) handleDelete(delete *DeleteCmd) error {
if delete == nil {
return c.sendErrorReply("delete: no parameter provided")
}
if err := os.Remove(delete.Path); err != nil {
return c.sendErrorReply("delete: %v", err)
}
return c.sendReply(&Reply{}, nil)
}
func (c *containerServer) handleReset() error {
if err := removeContents("/tmp"); err != nil {
return c.sendErrorReply("reset: /tmp %v", err)
}
if err := removeContents("/w"); err != nil {
return c.sendErrorReply("reset: /w %v", err)
}
return c.sendReply(&Reply{}, nil)
}
func (c *containerServer) recvCmd() (*Cmd, *unixsocket.Msg, error) {
cmd := new(Cmd)
msg, err := (*socket)(c.socket).RecvMsg(cmd)
if err != nil {
return nil, nil, err
}
return cmd, msg, nil
}
func (c *containerServer) sendReply(reply *Reply, msg *unixsocket.Msg) error {
return (*socket)(c.socket).SendMsg(reply, msg)
}
// sendErrorReply sends error reply
func (c *containerServer) sendErrorReply(ft string, v ...interface{}) error {
errorReply := &ErrorReply{
Msg: fmt.Sprintf(ft, v...),
}
// store errno
if len(v) == 1 {
if errno, ok := v[0].(syscall.Errno); ok {
errorReply.Errno = &errno
}
}
return c.sendReply(&Reply{Error: errorReply}, nil)
}

View File

@ -1,102 +0,0 @@
// Package daemon provides pre-forked container to reduce the container
// create / destroy costs (about 160ms). It creates daemon within unshared
// container and communicate with original process using unix socket with
// oob for fd / pid and struct are encoded in gob format.
package daemon
/*
Container / Master Communication Protocol (single thread):
- ping (alive check):
- reply: pong
- conf (set configuration):
- reply pong
- open (open files in given mode inside container):
- send: []OpenCmd
- reply: "success", file fds / "error"
- delete (unlink file / rmdir dir inside container):
- send: path
- reply: "finished" / "error"
- reset (clean up container for later use (clear workdir / tmp)):
- send:
- reply: "success"
- execve: (execute file inside container):
- send: argv, env, rLimits, fds
- reply:
- success: "success", pid
- failed: "failed"
- send (success): "init_finished" (as cmd)
- reply: "finished" / send: "kill" (as cmd)
- send: "kill" (as cmd) / reply: "finished"
- reply:
Any socket related error will cause the daemon exit (with all process inside container)
*/
import (
"os"
"syscall"
"time"
"github.com/criyle/go-sandbox/pkg/rlimit"
"github.com/criyle/go-sandbox/types"
)
// Cmd is the control message send into daemon
type Cmd struct {
Cmd string // type of the cmd
OpenCmd []OpenCmd // open argument
DeleteCmd *DeleteCmd // delete argument
ExecCmd *ExecCmd // execve argument
ConfCmd *ConfCmd // to set configuration
}
// OpenCmd correspond to a single open syscall
type OpenCmd struct {
Path string
Flag int
Perm os.FileMode
}
// DeleteCmd stores delete command
type DeleteCmd struct {
Path string
}
// ExecCmd stores execve parameter
type ExecCmd struct {
Argv []string // execve argv
Env []string // execve env
RLimits []rlimit.RLimit // execve posix rlimit
FdExec bool // if use fexecve (fd[0] as exec)
}
// ConfCmd stores conf parameter
type ConfCmd struct {
Conf containerConfig
}
// Reply is the reply message send back to controller
type Reply struct {
Error *ErrorReply // nil if no error
ExecReply *ExecReply
}
// ErrorReply stores error returned back from container
type ErrorReply struct {
Msg string
Errno *syscall.Errno
}
// ExecReply stores execve result
type ExecReply struct {
ExitStatus int // waitpid exit status
Status types.Status // return status
Time time.Duration // waitpid user CPU (ns)
Memory types.Size // waitpid user memory (byte)
}
func (e *ErrorReply) Error() string {
return e.Msg
}

View File

@ -1,24 +0,0 @@
package daemon
import (
"sync"
)
// 16k buffsize
const bufferSize = 16384
var bufferPool = sync.Pool{
New: func() interface{} {
return make([]byte, bufferSize)
},
}
// GetBuffer get buffer from pool
func GetBuffer() []byte {
return bufferPool.Get().([]byte)
}
// PutBuffer return buffer to the pool
func PutBuffer(x []byte) {
bufferPool.Put(x)
}