rename daemon -> container

This commit is contained in:
criyle 2020-02-19 05:36:23 -05:00
parent a24933ac82
commit f2dacb5752
16 changed files with 475 additions and 450 deletions

View File

@ -92,7 +92,7 @@ type Runner interface {
### Pre-forked Container Protocol
1. Pre-fork container daemons to run programs inside
1. Pre-fork container to run programs inside
2. Unix socket to pass fd inside / outside
Container / Master Communication Protocol (single thread):
@ -120,7 +120,7 @@ Container / Master Communication Protocol (single thread):
- send: "kill" (as cmd) / reply: "finished"
- reply:
Any socket related error will cause the daemon exit (with all process inside container)
Any socket related error will cause the container exit (with all process inside container)
## Packages (/pkg)
@ -137,7 +137,7 @@ Any socket related error will cause the daemon exit (with all process inside con
## Packages
- config: defines arch & language specified trace condition for ptrace runner from UOJ
- daemon: creates pre-forked container to run programs inside
- container: creates pre-forked container to run programs inside
- runner: interface to run program
- ptrace: wrapper to call forkexec and ptracer
- filehandler: an example implementation of UOJ file set

View File

@ -10,7 +10,7 @@ import (
"time"
"github.com/criyle/go-sandbox/config"
"github.com/criyle/go-sandbox/daemon"
"github.com/criyle/go-sandbox/container"
"github.com/criyle/go-sandbox/pkg/cgroup"
"github.com/criyle/go-sandbox/pkg/memfd"
"github.com/criyle/go-sandbox/pkg/mount"
@ -44,9 +44,9 @@ func printUsage() {
os.Exit(2)
}
// container init
func init() {
// container init
daemon.Init()
container.Init()
}
func main() {
@ -71,7 +71,7 @@ func main() {
flag.Var(&addRawWritable, "add-writable-raw", "Add a writable file (don't transform to its real path)")
flag.BoolVar(&useCGroup, "cgroup", false, "Use cgroup to colloct resource usage")
flag.BoolVar(&memfile, "memfd", false, "Use memfd as exec file")
flag.StringVar(&runt, "runner", "ptrace", "Runner for the program (ptrace, ns, daemon)")
flag.StringVar(&runt, "runner", "ptrace", "Runner for the program (ptrace, ns, container)")
flag.Parse()
args = flag.Args()
@ -133,13 +133,13 @@ func main() {
}
}
type daemonRunner struct {
*daemon.Master
*daemon.ExecveParam
type containerRunner struct {
container.Environment
container.ExecveParam
}
func (r *daemonRunner) Run(c context.Context) <-chan types.Result {
return r.Master.Execve(c, r.ExecveParam)
func (r *containerRunner) Run(c context.Context) <-chan types.Result {
return r.Environment.Execve(c, r.ExecveParam)
}
func start() (*types.Result, error) {
@ -231,29 +231,29 @@ func start() (*types.Result, error) {
MemoryLimit: types.Size(memoryLimit << 20),
}
if runt == "daemon" {
if runt == "container" {
root, err := ioutil.TempDir("", "dm")
if err != nil {
return nil, fmt.Errorf("cannot make temp root for daemon namespace: %v", err)
return nil, fmt.Errorf("cannot make temp root for container namespace: %v", err)
}
defer os.RemoveAll(root)
b := daemon.Builder{
b := container.Builder{
Root: root,
}
m, err := b.Build()
if err != nil {
return nil, fmt.Errorf("failed to new master: %v", err)
return nil, fmt.Errorf("failed to new container: %v", err)
}
defer m.Destroy()
err = m.Ping()
if err != nil {
return nil, fmt.Errorf("failed to ping daemon: %v", err)
return nil, fmt.Errorf("failed to ping container: %v", err)
}
runner = &daemonRunner{
Master: m,
ExecveParam: &daemon.ExecveParam{
runner = &containerRunner{
Environment: m,
ExecveParam: container.ExecveParam{
Args: args,
Env: []string{pathEnv},
Fds: fds,

View File

@ -1,4 +1,4 @@
package daemon
package container
const (
cmdPing = "ping"
@ -18,7 +18,7 @@ const (
containerUID = 1000
containerGID = 1000
containerName = "daemon"
containerName = "go-sandbox"
containerWD = "/w"
containerMaxProc = 1

View File

@ -0,0 +1,86 @@
package container
import (
"fmt"
"os"
"syscall"
"github.com/criyle/go-sandbox/pkg/unixsocket"
)
func (c *containerServer) handlePing() error {
return c.sendReply(&reply{}, nil)
}
func (c *containerServer) handleConf(conf *confCmd) error {
if conf != nil {
c.containerConfig = conf.Conf
}
return c.sendReply(&reply{}, nil)
}
func (c *containerServer) handleOpen(open []OpenCmd) error {
if len(open) == 0 {
return c.sendErrorReply("open: no open parameter received")
}
// open files
fds := make([]int, 0, len(open))
for _, o := range open {
outFile, err := os.OpenFile(o.Path, o.Flag, o.Perm)
if err != nil {
return c.sendErrorReply("open: %v", err)
}
defer outFile.Close()
fds = append(fds, int(outFile.Fd()))
}
return c.sendReply(&reply{}, &unixsocket.Msg{Fds: fds})
}
func (c *containerServer) handleDelete(delete *deleteCmd) error {
if delete == nil {
return c.sendErrorReply("delete: no parameter provided")
}
if err := os.Remove(delete.Path); err != nil {
return c.sendErrorReply("delete: %v", err)
}
return c.sendReply(&reply{}, nil)
}
func (c *containerServer) handleReset() error {
if err := removeContents("/tmp"); err != nil {
return c.sendErrorReply("reset: /tmp %v", err)
}
if err := removeContents("/w"); err != nil {
return c.sendErrorReply("reset: /w %v", err)
}
return c.sendReply(&reply{}, nil)
}
func (c *containerServer) recvCmd() (*cmd, *unixsocket.Msg, error) {
cm := new(cmd)
msg, err := (*socket)(c.socket).RecvMsg(cm)
if err != nil {
return nil, nil, err
}
return cm, msg, nil
}
func (c *containerServer) sendReply(rep *reply, msg *unixsocket.Msg) error {
return (*socket)(c.socket).SendMsg(rep, msg)
}
// sendErrorReply sends error reply
func (c *containerServer) sendErrorReply(ft string, v ...interface{}) error {
errRep := &errorReply{
Msg: fmt.Sprintf(ft, v...),
}
// store errno
if len(v) == 1 {
if errno, ok := v[0].(syscall.Errno); ok {
errRep.Errno = &errno
}
}
return c.sendReply(&reply{Error: errRep}, nil)
}

View File

@ -1,4 +1,4 @@
package daemon
package container
import (
"fmt"
@ -10,7 +10,7 @@ import (
"github.com/criyle/go-sandbox/types"
)
func (c *containerServer) handleExecve(cmd *ExecCmd, msg *unixsocket.Msg) error {
func (c *containerServer) handleExecve(cmd *execCmd, msg *unixsocket.Msg) error {
var (
files []uintptr
execFile uintptr
@ -37,21 +37,21 @@ func (c *containerServer) handleExecve(cmd *ExecCmd, msg *unixsocket.Msg) error
}
syncFunc := func(pid int) error {
msg2 := &unixsocket.Msg{
msg := &unixsocket.Msg{
Cred: &syscall.Ucred{
Pid: int32(pid),
Uid: uint32(syscall.Getuid()),
Gid: uint32(syscall.Getgid()),
},
}
if err2 := c.sendReply(&Reply{}, msg2); err2 != nil {
return fmt.Errorf("syncFunc: sendReply(%v)", err2)
if err := c.sendReply(&reply{}, msg); err != nil {
return fmt.Errorf("syncFunc: sendReply %w", err)
}
cmd2, _, err2 := c.recvCmd()
if err2 != nil {
return fmt.Errorf("syncFunc: recvCmd(%v)", err2)
cmd, _, err := c.recvCmd()
if err != nil {
return fmt.Errorf("syncFunc: recvCmd %w", err)
}
if cmd2.Cmd == cmdKill {
if cmd.Cmd == cmdKill {
return fmt.Errorf("syncFunc: received kill")
}
return nil
@ -121,8 +121,11 @@ func (c *containerServer) handleExecve(cmd *ExecCmd, msg *unixsocket.Msg) error
switch {
case wstatus.Exited():
exitStatus := wstatus.ExitStatus()
c.sendReply(&Reply{
ExecReply: &ExecReply{
if exitStatus != 0 {
status = types.StatusNonzeroExitStatus
}
c.sendReply(&reply{
ExecReply: &execReply{
Status: status,
ExitStatus: exitStatus,
Time: userTime,
@ -142,8 +145,8 @@ func (c *containerServer) handleExecve(cmd *ExecCmd, msg *unixsocket.Msg) error
default:
status = types.StatusSignalled
}
c.sendReply(&Reply{
ExecReply: &ExecReply{
c.sendReply(&reply{
ExecReply: &execReply{
ExitStatus: int(wstatus.Signal()),
Status: status,
Time: userTime,
@ -158,5 +161,5 @@ func (c *containerServer) handleExecve(cmd *ExecCmd, msg *unixsocket.Msg) error
// wait for kill msg and reply done for finish
<-killDone
return c.sendReply(&Reply{}, nil)
return c.sendReply(&reply{}, nil)
}

View File

@ -0,0 +1,93 @@
package container
import (
"fmt"
"os"
"runtime"
"github.com/criyle/go-sandbox/pkg/unixsocket"
)
type containerServer struct {
socket *unixsocket.Socket
containerConfig
}
// Init is called for container init process
// it will check if pid == 1, otherwise it is noop
// Init will do infinite loop on socket commands,
// and exits when at socket close, use it in init function
func Init() (err error) {
// noop if self is not container init process
// Notice: docker init is also 1, additional check for args[1] == init
if os.Getpid() != 1 || len(os.Args) != 2 || os.Args[1] != initArg {
return nil
}
// exit process (with whole container) upon exit this function
// possible reason:
// 1. socket broken (parent exit)
// 2. panic
// 3. undefined cmd (possible race condition)
defer func() {
if err := recover(); err != nil {
fmt.Fprintf(os.Stderr, "container_exit: panic: %v\n", err)
os.Exit(1)
}
if err != nil {
fmt.Fprintf(os.Stderr, "container_exit: %v\n", err)
os.Exit(1)
}
fmt.Fprintf(os.Stderr, "container_exit\n")
os.Exit(0)
}()
// limit container resource usage
runtime.GOMAXPROCS(containerMaxProc)
// new_master shared the socket at fd 3 (marked close_exec)
const defaultFd = 3
soc, err := unixsocket.NewSocket(defaultFd)
if err != nil {
return fmt.Errorf("container_init: failed to new socket %w", err)
}
// serve forever
cs := &containerServer{socket: soc}
return cs.serve()
}
func (c *containerServer) serve() error {
for {
cmd, msg, err := c.recvCmd()
if err != nil {
return fmt.Errorf("serve: recvCmd %w", err)
}
if err := c.handleCmd(cmd, msg); err != nil {
return fmt.Errorf("serve: failed to execute cmd %w", err)
}
}
}
func (c *containerServer) handleCmd(cmd *cmd, msg *unixsocket.Msg) error {
switch cmd.Cmd {
case cmdPing:
return c.handlePing()
case cmdConf:
return c.handleConf(cmd.ConfCmd)
case cmdOpen:
return c.handleOpen(cmd.OpenCmd)
case cmdDelete:
return c.handleDelete(cmd.DeleteCmd)
case cmdReset:
return c.handleReset()
case cmdExecve:
return c.handleExecve(cmd.ExecCmd, msg)
}
return fmt.Errorf("unknown command: %s", cmd.Cmd)
}

44
container/doc.go Normal file
View File

@ -0,0 +1,44 @@
// Package container provides pre-forked container environment to
// run programs in isolated Linux namespaces.
//
// Overview
//
// It creates container within unshared container and communicate
// with host process using unix socket with
// oob for fd / pid and commands encoded by gob.
//
// Protocol
//
// Host to container communication protocol is single threaded and always initiated by
// the host:
//
// - ping (alive check):
// - reply: pong
//
// - conf (set configuration):
// - reply pong
//
// - open (open files in given mode inside container):
// - send: []OpenCmd
// - reply: "success", file fds / "error"
//
// - delete (unlink file / rmdir dir inside container):
// - send: path
// - reply: "finished" / "error"
//
// - reset (clean up container for later use (clear workdir / tmp)):
// - send:
// - reply: "success"
//
// - execve: (execute file inside container):
// - send: argv, env, rLimits, fds
// - reply:
// - success: "success", pid
// - failed: "failed"
// - send (success): "init_finished" (as cmd)
// - reply: "finished" / send: "kill" (as cmd)
// - send: "kill" (as cmd) / reply: "finished"
// - reply:
//
// Any socket related error will cause the container exit with all process inside container
package container

View File

@ -1,6 +1,7 @@
package daemon
package container
import (
"context"
"fmt"
"os"
"sync"
@ -10,13 +11,14 @@ import (
"github.com/criyle/go-sandbox/pkg/memfd"
"github.com/criyle/go-sandbox/pkg/mount"
"github.com/criyle/go-sandbox/pkg/unixsocket"
"github.com/criyle/go-sandbox/types"
"golang.org/x/sys/unix"
)
// DefaultPath defines path evironment variable for container init process
const DefaultPath = "PATH=/usr/local/bin:/usr/bin:/bin"
// PathEnv defines path environment variable for the container init process
const PathEnv = "PATH=/usr/local/bin:/usr/bin:/bin"
// Builder builds instance of container masters
// Builder builds instance of container environment
type Builder struct {
// Root is container root mount path, empty uses current work path
Root string
@ -41,15 +43,25 @@ type CredGenerator interface {
Get() syscall.Credential
}
// Master manages single pre-forked container
type Master struct {
// Environment holds single progrem containered environment
type Environment interface {
Ping() error
Open([]OpenCmd) ([]*os.File, error)
Delete(p string) error
Reset() error
Execve(context.Context, ExecveParam) <-chan types.Result
Destroy() error
}
// container manages single pre-forked container environment
type container struct {
pid int // underlying container init pid
socket *unixsocket.Socket // master - container communication
mu sync.Mutex // lock to avoid race condition
}
// Build creates new master with underlying container
func (b *Builder) Build() (*Master, error) {
// Build creates new environment with underlying container
func (b *Builder) Build() (Environment, error) {
var (
err error
cred syscall.Credential
@ -64,19 +76,22 @@ func (b *Builder) Build() (*Master, error) {
WithTmpfs("w", ""). // work dir
WithTmpfs("tmp", ""). // tmp
Build(true); err != nil {
return nil, fmt.Errorf("daemon: failed to build rootfs mount %v", err)
return nil, fmt.Errorf("container: failed to build rootfs mount %w", err)
}
}
// container root directory on the host
root := b.Root
if root == "" {
if root, err = os.Getwd(); err != nil {
return nil, fmt.Errorf("daemon: failed to get work directory %v", err)
return nil, fmt.Errorf("container: failed to get work directory %w", err)
}
}
// prepare stdin / stdout / stderr
devNull, err := os.OpenFile(os.DevNull, os.O_RDWR, os.ModePerm)
if err != nil {
return nil, fmt.Errorf("daemon: failed to open devNull(%v)", err)
return nil, fmt.Errorf("container: failed to open devNull %w", err)
}
defer devNull.Close()
@ -88,30 +103,30 @@ func (b *Builder) Build() (*Master, error) {
files = append(files, devNull.Fd())
}
// prepare self memfd
// prepare container exec file
execFile, err := b.exec()
if err != nil {
return nil, fmt.Errorf("deamon: %v", err)
return nil, fmt.Errorf("container: prepare exec %w", err)
}
defer execFile.Close()
// prepare socket
// prepare host <-> container unix socket
ins, outs, err := newPassCredSocketPair()
if err != nil {
return nil, fmt.Errorf("daemon: failed to create socket: %v", err)
return nil, fmt.Errorf("container: failed to create socket: %w", err)
}
defer outs.Close()
outf, err := outs.File()
if err != nil {
ins.Close()
return nil, fmt.Errorf("daemon: failed to dup file outs(%v)", err)
return nil, fmt.Errorf("container: failed to dup container socket fd %w", err)
}
defer outf.Close()
files = append(files, uintptr(outf.Fd()))
// prepare credential
// prepare container running credential
if b.CredGenerator != nil {
cred = b.CredGenerator.Get()
uidMap, gidMap = getIDMapping(&cred)
@ -119,7 +134,7 @@ func (b *Builder) Build() (*Master, error) {
r := &forkexec.Runner{
Args: []string{os.Args[0], initArg},
Env: []string{DefaultPath},
Env: []string{PathEnv},
ExecFile: execFile.Fd(),
Files: files,
WorkDir: containerWD,
@ -134,41 +149,42 @@ func (b *Builder) Build() (*Master, error) {
pid, err := r.Start()
if err != nil {
ins.Close()
return nil, fmt.Errorf("daemon: failed to execve(%v)", err)
return nil, fmt.Errorf("container: failed to start container %w", err)
}
m := &Master{
c := &container{
pid: pid,
socket: ins,
}
// set configuration and check if container creation successful
if err = m.conf(&containerConfig{
if err = c.conf(&containerConfig{
Cred: b.CredGenerator != nil,
}); err != nil {
m.Destroy()
c.Destroy()
return nil, err
}
return m, nil
return c, nil
}
// Destroy kill the daemon process (with container)
// Destroy kill the container process (with its children)
// if stderr enabled, collect the output as error
func (m *Master) Destroy() error {
func (c *container) Destroy() error {
// close socket (abort any ongoing command)
m.socket.Close()
c.socket.Close()
// wait commands terminates
m.mu.Lock()
defer m.mu.Unlock()
c.mu.Lock()
defer c.mu.Unlock()
// kill process
var wstatus unix.WaitStatus
unix.Kill(m.pid, unix.SIGKILL)
unix.Kill(c.pid, unix.SIGKILL)
// wait for container process to exit
_, err := unix.Wait4(m.pid, &wstatus, 0, nil)
_, err := unix.Wait4(c.pid, &wstatus, 0, nil)
for err == unix.EINTR {
_, err = unix.Wait4(m.pid, &wstatus, 0, nil)
_, err = unix.Wait4(c.pid, &wstatus, 0, nil)
}
return err
}
@ -181,7 +197,7 @@ func (b *Builder) exec() (*os.File, error) {
return OpenCurrentExec()
}
// OpenCurrentExec opens current executable and dup it to memfd
// OpenCurrentExec opens current executable and dup and seal it to memfd
func OpenCurrentExec() (*os.File, error) {
self, err := os.Open(currentExec)
if err != nil {
@ -189,7 +205,7 @@ func OpenCurrentExec() (*os.File, error) {
}
defer self.Close()
execFile, err := memfd.DupToMemfd("daemon", self)
execFile, err := memfd.DupToMemfd("init", self)
if err != nil {
return nil, fmt.Errorf("failed to create memfd: %v", err)
}

View File

@ -1,4 +1,4 @@
package daemon
package container
import (
"fmt"
@ -9,55 +9,55 @@ import (
)
// Ping send ping message to container
func (m *Master) Ping() error {
m.mu.Lock()
defer m.mu.Unlock()
func (c *container) Ping() error {
c.mu.Lock()
defer c.mu.Unlock()
// avoid infinite wait (max 3s)
const pingWait = 3 * time.Second
m.socket.SetDeadline(time.Now().Add(pingWait))
defer m.socket.SetDeadline(time.Time{})
c.socket.SetDeadline(time.Now().Add(pingWait))
defer c.socket.SetDeadline(time.Time{})
// send ping
cmd := Cmd{
cmd := cmd{
Cmd: cmdPing,
}
if err := m.sendCmd(&cmd, nil); err != nil {
if err := c.sendCmd(&cmd, nil); err != nil {
return fmt.Errorf("ping: %v", err)
}
// receive no error
return m.recvAckReply("ping")
return c.recvAckReply("ping")
}
// conf send configuration to container (used by builder only)
func (m *Master) conf(conf *containerConfig) error {
m.mu.Lock()
defer m.mu.Unlock()
func (c *container) conf(conf *containerConfig) error {
c.mu.Lock()
defer c.mu.Unlock()
cmd := Cmd{
cmd := cmd{
Cmd: cmdConf,
ConfCmd: &ConfCmd{Conf: *conf},
ConfCmd: &confCmd{Conf: *conf},
}
if err := m.sendCmd(&cmd, nil); err != nil {
if err := c.sendCmd(&cmd, nil); err != nil {
return fmt.Errorf("conf: %v", err)
}
return m.recvAckReply("conf")
return c.recvAckReply("conf")
}
// Open open files in container
func (m *Master) Open(p []OpenCmd) ([]*os.File, error) {
m.mu.Lock()
defer m.mu.Unlock()
func (c *container) Open(p []OpenCmd) ([]*os.File, error) {
c.mu.Lock()
defer c.mu.Unlock()
// send copyin
cmd := Cmd{
cmd := cmd{
Cmd: cmdOpen,
OpenCmd: p,
}
if err := m.sendCmd(&cmd, nil); err != nil {
if err := c.sendCmd(&cmd, nil); err != nil {
return nil, fmt.Errorf("open: %v", err)
}
reply, msg, err := m.recvReply()
reply, msg, err := c.recvReply()
if err != nil {
return nil, fmt.Errorf("open: %v", err)
}
@ -82,36 +82,36 @@ func (m *Master) Open(p []OpenCmd) ([]*os.File, error) {
}
// Delete remove file from container
func (m *Master) Delete(p string) error {
m.mu.Lock()
defer m.mu.Unlock()
func (c *container) Delete(p string) error {
c.mu.Lock()
defer c.mu.Unlock()
cmd := Cmd{
cmd := cmd{
Cmd: cmdDelete,
DeleteCmd: &DeleteCmd{Path: p},
DeleteCmd: &deleteCmd{Path: p},
}
if err := m.sendCmd(&cmd, nil); err != nil {
if err := c.sendCmd(&cmd, nil); err != nil {
return fmt.Errorf("delete: %v", err)
}
return m.recvAckReply("delete")
return c.recvAckReply("delete")
}
// Reset remove all from /tmp and /w
func (m *Master) Reset() error {
m.mu.Lock()
defer m.mu.Unlock()
func (c *container) Reset() error {
c.mu.Lock()
defer c.mu.Unlock()
cmd := Cmd{
cmd := cmd{
Cmd: cmdReset,
}
if err := m.sendCmd(&cmd, nil); err != nil {
if err := c.sendCmd(&cmd, nil); err != nil {
return fmt.Errorf("reset: %v", err)
}
return m.recvAckReply("reset")
return c.recvAckReply("reset")
}
func (m *Master) recvAckReply(name string) error {
reply, _, err := m.recvReply()
func (c *container) recvAckReply(name string) error {
reply, _, err := c.recvReply()
if err != nil {
return fmt.Errorf("%v: recvAck %v", name, err)
}
@ -121,15 +121,15 @@ func (m *Master) recvAckReply(name string) error {
return nil
}
func (m *Master) recvReply() (*Reply, *unixsocket.Msg, error) {
reply := new(Reply)
msg, err := (*socket)(m.socket).RecvMsg(reply)
func (c *container) recvReply() (*reply, *unixsocket.Msg, error) {
reply := new(reply)
msg, err := (*socket)(c.socket).RecvMsg(reply)
if err != nil {
return nil, nil, err
}
return reply, msg, nil
}
func (m *Master) sendCmd(cmd *Cmd, msg *unixsocket.Msg) error {
return (*socket)(m.socket).SendMsg(cmd, msg)
func (c *container) sendCmd(cmd *cmd, msg *unixsocket.Msg) error {
return (*socket)(c.socket).SendMsg(cmd, msg)
}

View File

@ -1,4 +1,4 @@
package daemon
package container
import (
"context"
@ -16,20 +16,19 @@ type ExecveParam struct {
Env []string
Fds []uintptr
// fexecve fd
// fd parameter of fexecve
ExecFile uintptr
// POSIX Resource limit set by set rlimit
RLimits []rlimit.RLimit
// SyncFunc called with pid before execve (e.g. add to cgroup)
// SyncFunc called with pid before execve (for adding the process to cgroups)
SyncFunc func(pid int) error
}
// Execve runs process inside container
// accepts context for cancelation
func (m *Master) Execve(c context.Context, param *ExecveParam) <-chan types.Result {
m.mu.Lock()
// Execve runs process inside container. It accepts context cancelation as time limit exceeded.
func (c *container) Execve(ctx context.Context, param ExecveParam) <-chan types.Result {
c.mu.Lock()
sTime := time.Now()
@ -53,46 +52,46 @@ func (m *Master) Execve(c context.Context, param *ExecveParam) <-chan types.Resu
msg := &unixsocket.Msg{
Fds: files,
}
execCmd := &ExecCmd{
execCmd := &execCmd{
Argv: param.Args,
Env: param.Env,
RLimits: param.RLimits,
FdExec: param.ExecFile > 0,
}
cmd := Cmd{
cm := cmd{
Cmd: cmdExecve,
ExecCmd: execCmd,
}
if err := m.sendCmd(&cmd, msg); err != nil {
m.mu.Unlock()
if err := c.sendCmd(&cm, msg); err != nil {
c.mu.Unlock()
return errResult("execve: sendCmd %v", err)
}
// sync function
reply, msg, err := m.recvReply()
reply, msg, err := c.recvReply()
if err != nil {
m.mu.Unlock()
c.mu.Unlock()
return errResult("execve: recvReply %v", err)
}
// if sync function did not involved
if reply.Error != nil || msg == nil || msg.Cred == nil {
// tell kill function to exit and sync
m.execveSyncKill()
m.mu.Unlock()
c.execveSyncKill()
c.mu.Unlock()
return errResult("execve: no pid received or error %v", reply.Error)
}
if param.SyncFunc != nil {
if err := param.SyncFunc(int(msg.Cred.Pid)); err != nil {
// tell sync function to exit and recv error
m.execveSyncKill()
c.execveSyncKill()
// tell kill function to exit and sync
m.execveSyncKill()
m.mu.Unlock()
c.execveSyncKill()
c.mu.Unlock()
return errResult("execve: syncfunc failed %v", err)
}
}
// send to syncFunc ack ok
if err := m.sendCmd(&Cmd{Cmd: cmdOk}, nil); err != nil {
m.mu.Unlock()
if err := c.sendCmd(&cmd{Cmd: cmdOk}, nil); err != nil {
c.mu.Unlock()
return errResult("execve: ack failed %v", err)
}
@ -102,12 +101,12 @@ func (m *Master) Execve(c context.Context, param *ExecveParam) <-chan types.Resu
// Wait
go func() {
reply2, _, err := m.recvReply()
reply2, _, err := c.recvReply()
close(waitDone)
// done signal (should recv after kill)
m.recvReply()
c.recvReply()
// unlock after last read / write
m.mu.Unlock()
c.mu.Unlock()
// handle potential error
if err != nil {
@ -145,17 +144,17 @@ func (m *Master) Execve(c context.Context, param *ExecveParam) <-chan types.Resu
// Kill (if wait is done, a kill message need to be send to collect zombies)
go func() {
select {
case <-c.Done():
case <-ctx.Done():
case <-waitDone:
}
m.sendCmd(&Cmd{Cmd: cmdKill}, nil)
c.sendCmd(&cmd{Cmd: cmdKill}, nil)
}()
return result
}
// execveSyncKill will send kill and recv reply
func (m *Master) execveSyncKill() {
m.sendCmd(&Cmd{Cmd: cmdKill}, nil)
m.recvReply()
func (c *container) execveSyncKill() {
c.sendCmd(&cmd{Cmd: cmdKill}, nil)
c.recvReply()
}

74
container/protocol.go Normal file
View File

@ -0,0 +1,74 @@
package container
import (
"os"
"syscall"
"time"
"github.com/criyle/go-sandbox/pkg/rlimit"
"github.com/criyle/go-sandbox/types"
)
// cmd is the control message send into container
type cmd struct {
Cmd string // type of the cmd
OpenCmd []OpenCmd // open argument
DeleteCmd *deleteCmd // delete argument
ExecCmd *execCmd // execve argument
ConfCmd *confCmd // to set configuration
}
// OpenCmd correspond to a single open syscall
type OpenCmd struct {
Path string
Flag int
Perm os.FileMode
}
// deleteCmd stores delete command
type deleteCmd struct {
Path string
}
// execCmd stores execve parameter
type execCmd struct {
Argv []string // execve argv
Env []string // execve env
RLimits []rlimit.RLimit // execve posix rlimit
FdExec bool // if use fexecve (fd[0] as exec)
}
// confCmd stores conf parameter
type confCmd struct {
Conf containerConfig
}
// ContainerConfig set the container config
type containerConfig struct {
Cred bool
}
// reply is the reply message send back to controller
type reply struct {
Error *errorReply // nil if no error
ExecReply *execReply
}
// errorReply stores error returned back from container
type errorReply struct {
Msg string
Errno *syscall.Errno
}
// execReply stores execve result
type execReply struct {
ExitStatus int // waitpid exit status
Status types.Status // return status
Time time.Duration // waitpid user CPU (ns)
Memory types.Size // waitpid user memory (byte)
}
func (e *errorReply) Error() string {
return e.Msg
}

View File

@ -1,19 +1,30 @@
package daemon
package container
import (
"bytes"
"encoding/gob"
"fmt"
"sync"
"github.com/criyle/go-sandbox/pkg/unixsocket"
)
// 16k buffsize
const bufferSize = 16 << 10
var bufferPool = sync.Pool{
New: func() interface{} {
return make([]byte, bufferSize)
},
}
type socket unixsocket.Socket
func (s *socket) RecvMsg(e interface{}) (*unixsocket.Msg, error) {
soc := (*unixsocket.Socket)(s)
buff := GetBuffer()
defer PutBuffer(buff)
buff := bufferPool.Get().([]byte)
defer bufferPool.Put(buff)
n, msg, err := soc.RecvMsg(buff)
if err != nil {
@ -21,24 +32,25 @@ func (s *socket) RecvMsg(e interface{}) (*unixsocket.Msg, error) {
}
if err := gob.NewDecoder(bytes.NewReader(buff[:n])).Decode(e); err != nil {
return nil, fmt.Errorf("RecvMsg: failed to decode %v", err)
return nil, fmt.Errorf("RecvMsg: failed to decode %w", err)
}
return msg, nil
}
func (s *socket) SendMsg(e interface{}, msg *unixsocket.Msg) error {
soc := (*unixsocket.Socket)(s)
buf := GetBuffer()
defer PutBuffer(buf)
buf := bufferPool.Get().([]byte)
defer bufferPool.Put(buf)
// use buf pool to reduce allocation
buff := bytes.NewBuffer(buf[:0])
if err := gob.NewEncoder(buff).Encode(e); err != nil {
return fmt.Errorf("SendMsg: failed to encode %v", err)
return fmt.Errorf("SendMsg: failed to encode %w", err)
}
if err := soc.SendMsg(buff.Bytes(), msg); err != nil {
return fmt.Errorf("SendMsg: failed to SendMsg %v", err)
return fmt.Errorf("SendMsg: failed to SendMsg %w", err)
}
return nil
}

View File

@ -1,4 +1,4 @@
package daemon
package container
import (
"os"

View File

@ -1,176 +0,0 @@
package daemon
import (
"fmt"
"os"
"runtime"
"syscall"
"github.com/criyle/go-sandbox/pkg/unixsocket"
)
type containerServer struct {
socket *unixsocket.Socket
containerConfig
}
// ContainerConfig set the container config
type containerConfig struct {
Cred bool
}
// Init is called for container init process
// it will check if pid == 1, otherwise it is noop
// Init will do infinite loop on socket commands,
// and exits when at socket close, use it in init function
func Init() (err error) {
// noop if self is not container init process
// Notice: docker init is also 1, additional check for args[1] == init
if os.Getpid() != 1 || len(os.Args) != 2 || os.Args[1] != initArg {
return nil
}
// exit process (with whole container) upon exit this function
// possible reason:
// 1. socket broken (parent exit)
// 2. panic
// 3. undefined cmd (possible race condition)
defer func() {
if err2 := recover(); err2 != nil {
fmt.Fprintf(os.Stderr, "container_panic: %v\n", err)
os.Exit(1)
}
if err != nil {
fmt.Fprintf(os.Stderr, "container_exit: %v\n", err)
os.Exit(1)
}
fmt.Fprintf(os.Stderr, "container_exit\n")
os.Exit(0)
}()
// limit container resource usage
runtime.GOMAXPROCS(containerMaxProc)
// new_master shared the socket at fd 3 (marked close_exec)
const defaultFd = 3
soc, err := unixsocket.NewSocket(defaultFd)
if err != nil {
return fmt.Errorf("container_init: failed to new socket(%v)", err)
}
// serve forever
cs := &containerServer{socket: soc}
return cs.serve()
}
func (c *containerServer) serve() error {
for {
cmd, msg, err := c.recvCmd()
if err != nil {
return fmt.Errorf("serve: %v", err)
}
if err := c.handleCmd(cmd, msg); err != nil {
return fmt.Errorf("serve: failed to execute cmd %v", err)
}
}
}
func (c *containerServer) handleCmd(cmd *Cmd, msg *unixsocket.Msg) error {
switch cmd.Cmd {
case cmdPing:
return c.handlePing()
case cmdConf:
return c.handleConf(cmd.ConfCmd)
case cmdOpen:
return c.handleOpen(cmd.OpenCmd)
case cmdDelete:
return c.handleDelete(cmd.DeleteCmd)
case cmdReset:
return c.handleReset()
case cmdExecve:
return c.handleExecve(cmd.ExecCmd, msg)
}
return fmt.Errorf("unknown command: %v", cmd.Cmd)
}
func (c *containerServer) handlePing() error {
return c.sendReply(&Reply{}, nil)
}
func (c *containerServer) handleConf(conf *ConfCmd) error {
if conf != nil {
c.containerConfig = conf.Conf
}
return c.sendReply(&Reply{}, nil)
}
func (c *containerServer) handleOpen(open []OpenCmd) error {
if len(open) == 0 {
return c.sendErrorReply("open: no open parameter received")
}
// open files
fds := make([]int, 0, len(open))
for _, o := range open {
outFile, err := os.OpenFile(o.Path, o.Flag, o.Perm)
if err != nil {
return c.sendErrorReply("open: %v", err)
}
defer outFile.Close()
fds = append(fds, int(outFile.Fd()))
}
return c.sendReply(&Reply{}, &unixsocket.Msg{Fds: fds})
}
func (c *containerServer) handleDelete(delete *DeleteCmd) error {
if delete == nil {
return c.sendErrorReply("delete: no parameter provided")
}
if err := os.Remove(delete.Path); err != nil {
return c.sendErrorReply("delete: %v", err)
}
return c.sendReply(&Reply{}, nil)
}
func (c *containerServer) handleReset() error {
if err := removeContents("/tmp"); err != nil {
return c.sendErrorReply("reset: /tmp %v", err)
}
if err := removeContents("/w"); err != nil {
return c.sendErrorReply("reset: /w %v", err)
}
return c.sendReply(&Reply{}, nil)
}
func (c *containerServer) recvCmd() (*Cmd, *unixsocket.Msg, error) {
cmd := new(Cmd)
msg, err := (*socket)(c.socket).RecvMsg(cmd)
if err != nil {
return nil, nil, err
}
return cmd, msg, nil
}
func (c *containerServer) sendReply(reply *Reply, msg *unixsocket.Msg) error {
return (*socket)(c.socket).SendMsg(reply, msg)
}
// sendErrorReply sends error reply
func (c *containerServer) sendErrorReply(ft string, v ...interface{}) error {
errorReply := &ErrorReply{
Msg: fmt.Sprintf(ft, v...),
}
// store errno
if len(v) == 1 {
if errno, ok := v[0].(syscall.Errno); ok {
errorReply.Errno = &errno
}
}
return c.sendReply(&Reply{Error: errorReply}, nil)
}

View File

@ -1,102 +0,0 @@
// Package daemon provides pre-forked container to reduce the container
// create / destroy costs (about 160ms). It creates daemon within unshared
// container and communicate with original process using unix socket with
// oob for fd / pid and struct are encoded in gob format.
package daemon
/*
Container / Master Communication Protocol (single thread):
- ping (alive check):
- reply: pong
- conf (set configuration):
- reply pong
- open (open files in given mode inside container):
- send: []OpenCmd
- reply: "success", file fds / "error"
- delete (unlink file / rmdir dir inside container):
- send: path
- reply: "finished" / "error"
- reset (clean up container for later use (clear workdir / tmp)):
- send:
- reply: "success"
- execve: (execute file inside container):
- send: argv, env, rLimits, fds
- reply:
- success: "success", pid
- failed: "failed"
- send (success): "init_finished" (as cmd)
- reply: "finished" / send: "kill" (as cmd)
- send: "kill" (as cmd) / reply: "finished"
- reply:
Any socket related error will cause the daemon exit (with all process inside container)
*/
import (
"os"
"syscall"
"time"
"github.com/criyle/go-sandbox/pkg/rlimit"
"github.com/criyle/go-sandbox/types"
)
// Cmd is the control message send into daemon
type Cmd struct {
Cmd string // type of the cmd
OpenCmd []OpenCmd // open argument
DeleteCmd *DeleteCmd // delete argument
ExecCmd *ExecCmd // execve argument
ConfCmd *ConfCmd // to set configuration
}
// OpenCmd correspond to a single open syscall
type OpenCmd struct {
Path string
Flag int
Perm os.FileMode
}
// DeleteCmd stores delete command
type DeleteCmd struct {
Path string
}
// ExecCmd stores execve parameter
type ExecCmd struct {
Argv []string // execve argv
Env []string // execve env
RLimits []rlimit.RLimit // execve posix rlimit
FdExec bool // if use fexecve (fd[0] as exec)
}
// ConfCmd stores conf parameter
type ConfCmd struct {
Conf containerConfig
}
// Reply is the reply message send back to controller
type Reply struct {
Error *ErrorReply // nil if no error
ExecReply *ExecReply
}
// ErrorReply stores error returned back from container
type ErrorReply struct {
Msg string
Errno *syscall.Errno
}
// ExecReply stores execve result
type ExecReply struct {
ExitStatus int // waitpid exit status
Status types.Status // return status
Time time.Duration // waitpid user CPU (ns)
Memory types.Size // waitpid user memory (byte)
}
func (e *ErrorReply) Error() string {
return e.Msg
}

View File

@ -1,24 +0,0 @@
package daemon
import (
"sync"
)
// 16k buffsize
const bufferSize = 16384
var bufferPool = sync.Pool{
New: func() interface{} {
return make([]byte, bufferSize)
},
}
// GetBuffer get buffer from pool
func GetBuffer() []byte {
return bufferPool.Get().([]byte)
}
// PutBuffer return buffer to the pool
func PutBuffer(x []byte) {
bufferPool.Put(x)
}