diff --git a/cmd/runprog/main_linux.go b/cmd/runprog/main_linux.go index e59d3bd..4b6e84b 100644 --- a/cmd/runprog/main_linux.go +++ b/cmd/runprog/main_linux.go @@ -5,6 +5,7 @@ import ( "context" "flag" "fmt" + "io" "io/ioutil" "os" "os/signal" @@ -171,9 +172,10 @@ func start() (*runner.Result, error) { // work dir WithTmpfs("w", "size=8m,nr_inodes=4k"). // tmp dir - WithTmpfs("tmp", "size=8m,nr_inodes=4k") + WithTmpfs("tmp", "size=8m,nr_inodes=4k"). + FilterNotExist() - mt, err := mb.Build(true) + mt, err := mb.FilterNotExist().Build() if err != nil { return nil, err } @@ -264,10 +266,15 @@ func start() (*runner.Result, error) { if cred { credG = newCredGen() } + var stderr io.Writer + if showDetails { + stderr = os.Stderr + } b := container.Builder{ Root: root, - Mounts: mt, + Mounts: mb.Mounts, + Stderr: stderr, CredGenerator: credG, CloneFlags: forkexec.UnshareFlags, } diff --git a/container/benchmark_linux_test.go b/container/benchmark_linux_test.go index 7fca6b1..a89ffb2 100644 --- a/container/benchmark_linux_test.go +++ b/container/benchmark_linux_test.go @@ -22,7 +22,8 @@ func BenchmarkContainer(b *testing.B) { b.Error(err) } builder := &Builder{ - Root: tmpDir, + Root: tmpDir, + Stderr: os.Stderr, } n := runtime.GOMAXPROCS(0) ch := make(chan Environment, n) @@ -61,7 +62,7 @@ func TestContainerSuccess(t *testing.T) { }) r := <-rt if r.Status != runner.StatusNormal { - t.Error(r.Status, r.Error) + t.Fatal(r.Status, r.Error) } } @@ -86,7 +87,7 @@ func TestContainerSetCred(t *testing.T) { }) r := <-rt if r.Status != runner.StatusNormal { - t.Error(r.Status, r.Error) + t.Fatal(r.Status, r.Error) } } @@ -99,7 +100,7 @@ func TestContainerNotExists(t *testing.T) { }) r := <-rt if r.Status != runner.StatusRunnerError { - t.Error(r.Status, r.Error) + t.Fatal(r.Status, r.Error) } } @@ -116,14 +117,14 @@ func TestContainerSyncFuncFail(t *testing.T) { }) r := <-rt if r.Status != runner.StatusRunnerError { - t.Error(r.Status, r.Error) + t.Fatal(r.Status, r.Error) } } func getEnv(t *testing.T, credGen CredGenerator) Environment { tmpDir, err := ioutil.TempDir("", "") if err != nil { - t.Error(err) + t.Fatal(err) } t.Cleanup(func() { os.Remove(tmpDir) @@ -131,10 +132,11 @@ func getEnv(t *testing.T, credGen CredGenerator) Environment { builder := &Builder{ Root: tmpDir, CredGenerator: credGen, + Stderr: os.Stderr, } m, err := builder.Build() if err != nil { - t.Error(err) + t.Fatal(err) } t.Cleanup(func() { m.Destroy() diff --git a/container/consts.go b/container/consts.go index 7107e4a..77b65e1 100644 --- a/container/consts.go +++ b/container/consts.go @@ -11,7 +11,7 @@ const ( cmdKill = "kill" cmdConf = "conf" - initArg = "init" + initArg = "container_init" currentExec = "/proc/self/exe" diff --git a/container/container_cmd_linux.go b/container/container_cmd_linux.go index eb42583..0af2478 100644 --- a/container/container_cmd_linux.go +++ b/container/container_cmd_linux.go @@ -15,6 +15,9 @@ func (c *containerServer) handlePing() error { func (c *containerServer) handleConf(conf *confCmd) error { if conf != nil { c.containerConfig = conf.Conf + if err := initContainer(conf.Conf); err != nil { + return err + } } return c.sendReply(&reply{}, nil) } diff --git a/container/container_init_linux.go b/container/container_init_linux.go index 2e16bc8..124ba51 100644 --- a/container/container_init_linux.go +++ b/container/container_init_linux.go @@ -4,6 +4,7 @@ import ( "fmt" "os" "runtime" + "syscall" "github.com/criyle/go-sandbox/pkg/unixsocket" ) @@ -20,7 +21,7 @@ type containerServer struct { func Init() (err error) { // noop if self is not container init process // Notice: docker init is also 1, additional check for args[1] == init - if os.Getpid() != 1 || len(os.Args) != 2 || os.Args[1] != initArg { + if os.Getpid() != 1 || len(os.Args) < 2 || os.Args[1] != initArg { return nil } @@ -91,3 +92,54 @@ func (c *containerServer) handleCmd(cmd *cmd, msg *unixsocket.Msg) error { } return fmt.Errorf("unknown command: %s", cmd.Cmd) } + +func initContainer(c containerConfig) error { + if err := initFileSystem(c); err != nil { + return err + } + if err := syscall.Setdomainname([]byte(c.DomainName)); err != nil { + return err + } + if err := syscall.Sethostname([]byte(c.HostName)); err != nil { + return err + } + return os.Chdir(c.WorkDir) +} + +func initFileSystem(c containerConfig) error { + // mount tmpfs as root + const tmpfs = "tmpfs" + if err := syscall.Mount(tmpfs, c.ContainerRoot, tmpfs, 0, ""); err != nil { + return fmt.Errorf("init_fs: mount / %v", err) + } + // change dir to container root + if err := syscall.Chdir(c.ContainerRoot); err != nil { + return fmt.Errorf("init_fs: chdir %v", err) + } + // performing mounts + for _, m := range c.Mounts { + if err := m.Mount(); err != nil { + return fmt.Errorf("init_fs: mount %v %v", m, err) + } + } + // pivot root + const oldRoot = "old_root" + if err := os.Mkdir(oldRoot, 0755); err != nil { + return fmt.Errorf("init_fs: mkdir(old_root) %v", err) + } + if err := syscall.PivotRoot(c.ContainerRoot, oldRoot); err != nil { + return fmt.Errorf("init_fs: pivot_root(%s, %s) %v", c.ContainerRoot, oldRoot, err) + } + if err := syscall.Unmount(oldRoot, syscall.MNT_DETACH); err != nil { + return fmt.Errorf("init_fs: unmount(old_root) %v", err) + } + if err := os.Remove(oldRoot); err != nil { + return fmt.Errorf("init_fs: unlink(old_root) %v", err) + } + // readonly root + const remountFlag = syscall.MS_BIND | syscall.MS_REMOUNT | syscall.MS_RDONLY | syscall.MS_NOATIME | syscall.MS_NOSUID + if err := syscall.Mount(tmpfs, "/", tmpfs, remountFlag, ""); err != nil { + return fmt.Errorf("init_fs: readonly remount / %v", err) + } + return nil +} diff --git a/container/environment_linux.go b/container/environment_linux.go index 9602a40..ff20b37 100644 --- a/container/environment_linux.go +++ b/container/environment_linux.go @@ -3,7 +3,9 @@ package container import ( "context" "fmt" + "io" "os" + "os/exec" "sync" "syscall" @@ -23,10 +25,13 @@ type Builder struct { Root string // Mounts defines container mount points, empty uses default mounts - Mounts []mount.SyscallParams + Mounts []mount.Mount + + // WorkDir defines container default work directory (default: /w) + WorkDir string // Stderr defines whether to dup container stderr to stderr for debug - Stderr bool + Stderr io.Writer // ExecFile defines executable that called Init, otherwise defer current // executable (/proc/self/exe) @@ -37,6 +42,12 @@ type Builder struct { // Clone flags defines unshare clone flag to create container CloneFlags uintptr + + // HostName set container hostname (default: go-sandbox) + HostName string + + // DomainName set container domainname (default: go-sandbox) + DomainName string } // CredGenerator generates uid / gid credential used by container @@ -57,28 +68,31 @@ type Environment interface { // container manages single pre-forked container environment type container struct { - pid int // underlying container init pid - socket *socket // host - container communication - mu sync.Mutex // lock to avoid race condition + process *os.Process // underlying container init pid + socket *socket // host - container communication + mu sync.Mutex // lock to avoid race condition } // Build creates new environment with underlying container func (b *Builder) Build() (Environment, error) { - var ( - err error - cred syscall.Credential - uidMap, gidMap []syscall.SysProcIDMap - ) + c, err := b.startContainer() + if err != nil { + return nil, err + } + + // avoid non cinit enabled executable running as container init process + if err = c.Ping(); err != nil { + c.Destroy() + return nil, fmt.Errorf("container: container init not responding to ping %v", err) + } // container mount points mounts := b.Mounts if len(mounts) == 0 { - if mounts, err = mount.NewDefaultBuilder(). + mounts = mount.NewDefaultBuilder(). WithTmpfs("w", ""). // work dir WithTmpfs("tmp", ""). // tmp - Build(true); err != nil { - return nil, fmt.Errorf("container: failed to build rootfs mount %v", err) - } + FilterNotExist().Mounts } // container root directory on the host @@ -88,29 +102,40 @@ func (b *Builder) Build() (Environment, error) { return nil, fmt.Errorf("container: failed to get work directory %v", err) } } - - // prepare stdin / stdout / stderr - devNull, err := os.OpenFile(os.DevNull, os.O_RDWR, os.ModePerm) - if err != nil { - return nil, fmt.Errorf("container: failed to open devNull %v", err) + workDir := containerWD + if b.WorkDir != "" { + workDir = b.WorkDir } - defer devNull.Close() - - files := make([]uintptr, 0, 4) - files = append(files, devNull.Fd(), devNull.Fd()) - if b.Stderr { - files = append(files, os.Stderr.Fd()) - } else { - files = append(files, devNull.Fd()) + hostName := containerName + if b.HostName != "" { + hostName = b.HostName + } + domainName := containerName + if b.DomainName != "" { + domainName = b.DomainName } - // prepare container exec file - execFile, err := b.exec() - if err != nil { - return nil, fmt.Errorf("container: prepare exec %v", err) + // set configuration and check if container creation successful + if err = c.conf(&containerConfig{ + WorkDir: workDir, + HostName: hostName, + DomainName: domainName, + ContainerRoot: root, + Mounts: mounts, + Cred: b.CredGenerator != nil, + }); err != nil { + c.Destroy() + return nil, err } - defer execFile.Close() + return c, nil +} +func (b *Builder) startContainer() (*container, error) { + var ( + err error + cred syscall.Credential + uidMap, gidMap []syscall.SysProcIDMap + ) // prepare host <-> container unix socket ins, outs, err := newPassCredSocketPair() if err != nil { @@ -125,12 +150,13 @@ func (b *Builder) Build() (Environment, error) { } defer outf.Close() - files = append(files, uintptr(outf.Fd())) - // prepare container running credential if b.CredGenerator != nil { cred = b.CredGenerator.Get() uidMap, gidMap = getIDMapping(&cred) + } else { + uidMap = []syscall.SysProcIDMap{{HostID: os.Geteuid(), Size: 1}} + gidMap = []syscall.SysProcIDMap{{HostID: os.Getegid(), Size: 1}} } var cloneFlag uintptr @@ -140,45 +166,34 @@ func (b *Builder) Build() (Environment, error) { cloneFlag = b.CloneFlags & forkexec.UnshareFlags } - r := &forkexec.Runner{ - Args: []string{os.Args[0], initArg}, - Env: []string{PathEnv}, - ExecFile: execFile.Fd(), - Files: files, - WorkDir: containerWD, - CloneFlags: cloneFlag, - Mounts: mounts, - HostName: containerName, - DomainName: containerName, - PivotRoot: root, - UIDMappings: uidMap, - GIDMappings: gidMap, + args := []string{os.Args[0], initArg} + if b.ExecFile != "" { + args[0] = b.ExecFile } - pid, err := r.Start() - if err != nil { + + r := exec.Cmd{ + Path: args[0], + Args: args, + Env: []string{PathEnv}, + Stderr: b.Stderr, + ExtraFiles: []*os.File{outf}, + SysProcAttr: &syscall.SysProcAttr{ + Cloneflags: cloneFlag, + UidMappings: uidMap, + GidMappings: gidMap, + AmbientCaps: []uintptr{ + unix.CAP_SYS_ADMIN, + }, + }, + } + if err = r.Start(); err != nil { ins.Close() return nil, fmt.Errorf("container: failed to start container %v", err) } - - c := &container{ - pid: pid, - socket: newSocket(ins), - } - // avoid non cinit enabled executable running as container init process - if err = c.Ping(); err != nil { - c.Destroy() - return nil, fmt.Errorf("container: container init not responding to ping %v", err) - } - - // set configuration and check if container creation successful - if err = c.conf(&containerConfig{ - Cred: b.CredGenerator != nil, - }); err != nil { - c.Destroy() - return nil, err - } - - return c, nil + return &container{ + process: r.Process, + socket: newSocket(ins), + }, nil } // Destroy kill the container process (with its children) @@ -192,13 +207,8 @@ func (c *container) Destroy() error { defer c.mu.Unlock() // kill process - var wstatus unix.WaitStatus - unix.Kill(c.pid, unix.SIGKILL) - // wait for container process to exit - _, err := unix.Wait4(c.pid, &wstatus, 0, nil) - for err == unix.EINTR { - _, err = unix.Wait4(c.pid, &wstatus, 0, nil) - } + c.process.Kill() + _, err := c.process.Wait() return err } diff --git a/container/host_cmd_linux.go b/container/host_cmd_linux.go index a7f9bb1..12bfbf9 100644 --- a/container/host_cmd_linux.go +++ b/container/host_cmd_linux.go @@ -8,7 +8,7 @@ import ( "github.com/criyle/go-sandbox/pkg/unixsocket" ) -// Ping send ping message to container +// Ping send ping message to container, wait for 3 second before timeout func (c *container) Ping() error { c.mu.Lock() defer c.mu.Unlock() diff --git a/container/protocol_linux.go b/container/protocol_linux.go index b8c7218..665eb1e 100644 --- a/container/protocol_linux.go +++ b/container/protocol_linux.go @@ -5,6 +5,7 @@ import ( "syscall" "time" + "github.com/criyle/go-sandbox/pkg/mount" "github.com/criyle/go-sandbox/pkg/rlimit" "github.com/criyle/go-sandbox/runner" ) @@ -47,6 +48,14 @@ type confCmd struct { // ContainerConfig set the container config type containerConfig struct { + WorkDir string + + HostName string + DomainName string + + ContainerRoot string + Mounts []mount.Mount + Cred bool } diff --git a/go.mod b/go.mod index 1f4752f..4ddbe31 100644 --- a/go.mod +++ b/go.mod @@ -6,8 +6,8 @@ require ( github.com/davecgh/go-spew v1.1.1 // indirect github.com/elastic/go-seccomp-bpf v1.1.0 github.com/pkg/errors v0.9.1 // indirect - golang.org/x/net v0.0.0-20200904194848-62affa334b73 - golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f + golang.org/x/net v0.0.0-20201110031124-69a78807bb2b + golang.org/x/sys v0.0.0-20201119102817-f84b799fce68 ) replace github.com/elastic/go-seccomp-bpf => ../go-seccomp-bpf diff --git a/go.sum b/go.sum index e8d7ed4..ac1a12b 100644 --- a/go.sum +++ b/go.sum @@ -16,14 +16,16 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3 h1:0GoQqolDA55aaLxZyTzK/Y2ePZzZTUrRacwib7cNsYQ= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20200904194848-62affa334b73 h1:MXfv8rhZWmFeqX3GNZRsd6vOLoaCHjYEX3qkRo3YBUA= -golang.org/x/net v0.0.0-20200904194848-62affa334b73/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b h1:uwuIcX0g4Yl1NC5XAz37xsr2lTtcqevgzYNVt49waME= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190405154228-4b34438f7a67/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f h1:Fqb3ao1hUmOR3GkUOg/Y+BadLwykBIzs5q8Ez2SbHyc= -golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68 h1:nxC68pudNYkKU6jWhgrqdreuFiOQWj1Fs7T3VrH4Pjw= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/pkg/forkexec/bench_linux_test.go b/pkg/forkexec/bench_linux_test.go index 3b56a47..453ce97 100644 --- a/pkg/forkexec/bench_linux_test.go +++ b/pkg/forkexec/bench_linux_test.go @@ -173,7 +173,7 @@ func getMounts(dirs []string) []mount.SyscallParams { Flags: roBind, }) } - m, _ := builder.Build(true) + m, _ := builder.FilterNotExist().Build() return m } diff --git a/pkg/forkexec/fork_linux_test.go b/pkg/forkexec/fork_linux_test.go index deb286e..8403330 100644 --- a/pkg/forkexec/fork_linux_test.go +++ b/pkg/forkexec/fork_linux_test.go @@ -95,7 +95,7 @@ func TestFork_ENOENT(t *testing.T) { WithMount( mount.Mount{ Source: "NOT_EXISTS", - }).Build(false) + }).Build() if err != nil { t.Fatal(err) } diff --git a/pkg/mount/builder_linux.go b/pkg/mount/builder_linux.go index 7a7e6b9..82528a4 100644 --- a/pkg/mount/builder_linux.go +++ b/pkg/mount/builder_linux.go @@ -22,16 +22,12 @@ func NewDefaultBuilder() *Builder { } // Build creates sequence of syscalls for fork_exec -// skipNotExists skips bind mounts that source not exists -func (b *Builder) Build(skipNotExists bool) ([]SyscallParams, error) { +func (b *Builder) Build() ([]SyscallParams, error) { var err error ret := make([]SyscallParams, 0, len(b.Mounts)) for _, m := range b.Mounts { var mknod bool if mknod, err = isBindMountFileOrNotExists(m); err != nil { - if skipNotExists { - continue - } return nil, err } sp, err := m.ToSyscall() @@ -44,8 +40,23 @@ func (b *Builder) Build(skipNotExists bool) ([]SyscallParams, error) { return ret, nil } +// FilterNotExist removes bind mount that does not exists +func (b *Builder) FilterNotExist() *Builder { + rt := b.Mounts[:0] + for _, m := range b.Mounts { + if m.IsBindMount() { + if _, err := os.Stat(m.Source); os.IsNotExist(err) { + continue + } + } + rt = append(rt, m) + } + b.Mounts = rt + return b +} + func isBindMountFileOrNotExists(m Mount) (bool, error) { - if m.Flags&unix.MS_BIND == unix.MS_BIND { + if m.IsBindMount() { if fi, err := os.Stat(m.Source); os.IsNotExist(err) { return false, err } else if !fi.IsDir() { diff --git a/pkg/mount/mount_linux.go b/pkg/mount/mount_linux.go index 3e7388f..f50baa7 100644 --- a/pkg/mount/mount_linux.go +++ b/pkg/mount/mount_linux.go @@ -3,12 +3,13 @@ package mount import ( "fmt" "os" + "path/filepath" "syscall" ) // Mount calls mount syscall func (m *Mount) Mount() error { - if err := os.MkdirAll(m.Target, 0755); err != nil { + if err := ensureMountTargetExists(m.Source, m.Target); err != nil { return err } if err := syscall.Mount(m.Source, m.Target, m.FsType, m.Flags, m.Data); err != nil { @@ -24,6 +25,36 @@ func (m *Mount) Mount() error { return nil } +// IsBindMount returns if it is a bind mount +func (m Mount) IsBindMount() bool { + return m.Flags&syscall.MS_BIND == syscall.MS_BIND +} + +// IsReadOnly returns if it is a readonly mount +func (m Mount) IsReadOnly() bool { + return m.Flags&syscall.MS_RDONLY == syscall.MS_RDONLY +} + +func ensureMountTargetExists(source, target string) error { + isFile := false + if fi, err := os.Stat(source); err == nil { + isFile = !fi.IsDir() + } + dir := target + if isFile { + dir = filepath.Dir(target) + } + if err := os.MkdirAll(dir, 0755); err != nil { + return err + } + if isFile { + if err := syscall.Mknod(target, 0755, 0); err != nil { + return err + } + } + return nil +} + func (m Mount) String() string { switch { case m.Flags&syscall.MS_BIND == syscall.MS_BIND: