Refactor container mounts

This commit is contained in:
criyle 2020-11-21 23:50:06 -08:00
parent 085f7a63c2
commit c832edba58
14 changed files with 232 additions and 105 deletions

View File

@ -5,6 +5,7 @@ import (
"context"
"flag"
"fmt"
"io"
"io/ioutil"
"os"
"os/signal"
@ -171,9 +172,10 @@ func start() (*runner.Result, error) {
// work dir
WithTmpfs("w", "size=8m,nr_inodes=4k").
// tmp dir
WithTmpfs("tmp", "size=8m,nr_inodes=4k")
WithTmpfs("tmp", "size=8m,nr_inodes=4k").
FilterNotExist()
mt, err := mb.Build(true)
mt, err := mb.FilterNotExist().Build()
if err != nil {
return nil, err
}
@ -264,10 +266,15 @@ func start() (*runner.Result, error) {
if cred {
credG = newCredGen()
}
var stderr io.Writer
if showDetails {
stderr = os.Stderr
}
b := container.Builder{
Root: root,
Mounts: mt,
Mounts: mb.Mounts,
Stderr: stderr,
CredGenerator: credG,
CloneFlags: forkexec.UnshareFlags,
}

View File

@ -23,6 +23,7 @@ func BenchmarkContainer(b *testing.B) {
}
builder := &Builder{
Root: tmpDir,
Stderr: os.Stderr,
}
n := runtime.GOMAXPROCS(0)
ch := make(chan Environment, n)
@ -61,7 +62,7 @@ func TestContainerSuccess(t *testing.T) {
})
r := <-rt
if r.Status != runner.StatusNormal {
t.Error(r.Status, r.Error)
t.Fatal(r.Status, r.Error)
}
}
@ -86,7 +87,7 @@ func TestContainerSetCred(t *testing.T) {
})
r := <-rt
if r.Status != runner.StatusNormal {
t.Error(r.Status, r.Error)
t.Fatal(r.Status, r.Error)
}
}
@ -99,7 +100,7 @@ func TestContainerNotExists(t *testing.T) {
})
r := <-rt
if r.Status != runner.StatusRunnerError {
t.Error(r.Status, r.Error)
t.Fatal(r.Status, r.Error)
}
}
@ -116,14 +117,14 @@ func TestContainerSyncFuncFail(t *testing.T) {
})
r := <-rt
if r.Status != runner.StatusRunnerError {
t.Error(r.Status, r.Error)
t.Fatal(r.Status, r.Error)
}
}
func getEnv(t *testing.T, credGen CredGenerator) Environment {
tmpDir, err := ioutil.TempDir("", "")
if err != nil {
t.Error(err)
t.Fatal(err)
}
t.Cleanup(func() {
os.Remove(tmpDir)
@ -131,10 +132,11 @@ func getEnv(t *testing.T, credGen CredGenerator) Environment {
builder := &Builder{
Root: tmpDir,
CredGenerator: credGen,
Stderr: os.Stderr,
}
m, err := builder.Build()
if err != nil {
t.Error(err)
t.Fatal(err)
}
t.Cleanup(func() {
m.Destroy()

View File

@ -11,7 +11,7 @@ const (
cmdKill = "kill"
cmdConf = "conf"
initArg = "init"
initArg = "container_init"
currentExec = "/proc/self/exe"

View File

@ -15,6 +15,9 @@ func (c *containerServer) handlePing() error {
func (c *containerServer) handleConf(conf *confCmd) error {
if conf != nil {
c.containerConfig = conf.Conf
if err := initContainer(conf.Conf); err != nil {
return err
}
}
return c.sendReply(&reply{}, nil)
}

View File

@ -4,6 +4,7 @@ import (
"fmt"
"os"
"runtime"
"syscall"
"github.com/criyle/go-sandbox/pkg/unixsocket"
)
@ -20,7 +21,7 @@ type containerServer struct {
func Init() (err error) {
// noop if self is not container init process
// Notice: docker init is also 1, additional check for args[1] == init
if os.Getpid() != 1 || len(os.Args) != 2 || os.Args[1] != initArg {
if os.Getpid() != 1 || len(os.Args) < 2 || os.Args[1] != initArg {
return nil
}
@ -91,3 +92,54 @@ func (c *containerServer) handleCmd(cmd *cmd, msg *unixsocket.Msg) error {
}
return fmt.Errorf("unknown command: %s", cmd.Cmd)
}
func initContainer(c containerConfig) error {
if err := initFileSystem(c); err != nil {
return err
}
if err := syscall.Setdomainname([]byte(c.DomainName)); err != nil {
return err
}
if err := syscall.Sethostname([]byte(c.HostName)); err != nil {
return err
}
return os.Chdir(c.WorkDir)
}
func initFileSystem(c containerConfig) error {
// mount tmpfs as root
const tmpfs = "tmpfs"
if err := syscall.Mount(tmpfs, c.ContainerRoot, tmpfs, 0, ""); err != nil {
return fmt.Errorf("init_fs: mount / %v", err)
}
// change dir to container root
if err := syscall.Chdir(c.ContainerRoot); err != nil {
return fmt.Errorf("init_fs: chdir %v", err)
}
// performing mounts
for _, m := range c.Mounts {
if err := m.Mount(); err != nil {
return fmt.Errorf("init_fs: mount %v %v", m, err)
}
}
// pivot root
const oldRoot = "old_root"
if err := os.Mkdir(oldRoot, 0755); err != nil {
return fmt.Errorf("init_fs: mkdir(old_root) %v", err)
}
if err := syscall.PivotRoot(c.ContainerRoot, oldRoot); err != nil {
return fmt.Errorf("init_fs: pivot_root(%s, %s) %v", c.ContainerRoot, oldRoot, err)
}
if err := syscall.Unmount(oldRoot, syscall.MNT_DETACH); err != nil {
return fmt.Errorf("init_fs: unmount(old_root) %v", err)
}
if err := os.Remove(oldRoot); err != nil {
return fmt.Errorf("init_fs: unlink(old_root) %v", err)
}
// readonly root
const remountFlag = syscall.MS_BIND | syscall.MS_REMOUNT | syscall.MS_RDONLY | syscall.MS_NOATIME | syscall.MS_NOSUID
if err := syscall.Mount(tmpfs, "/", tmpfs, remountFlag, ""); err != nil {
return fmt.Errorf("init_fs: readonly remount / %v", err)
}
return nil
}

View File

@ -3,7 +3,9 @@ package container
import (
"context"
"fmt"
"io"
"os"
"os/exec"
"sync"
"syscall"
@ -23,10 +25,13 @@ type Builder struct {
Root string
// Mounts defines container mount points, empty uses default mounts
Mounts []mount.SyscallParams
Mounts []mount.Mount
// WorkDir defines container default work directory (default: /w)
WorkDir string
// Stderr defines whether to dup container stderr to stderr for debug
Stderr bool
Stderr io.Writer
// ExecFile defines executable that called Init, otherwise defer current
// executable (/proc/self/exe)
@ -37,6 +42,12 @@ type Builder struct {
// Clone flags defines unshare clone flag to create container
CloneFlags uintptr
// HostName set container hostname (default: go-sandbox)
HostName string
// DomainName set container domainname (default: go-sandbox)
DomainName string
}
// CredGenerator generates uid / gid credential used by container
@ -57,28 +68,31 @@ type Environment interface {
// container manages single pre-forked container environment
type container struct {
pid int // underlying container init pid
process *os.Process // underlying container init pid
socket *socket // host - container communication
mu sync.Mutex // lock to avoid race condition
}
// Build creates new environment with underlying container
func (b *Builder) Build() (Environment, error) {
var (
err error
cred syscall.Credential
uidMap, gidMap []syscall.SysProcIDMap
)
c, err := b.startContainer()
if err != nil {
return nil, err
}
// avoid non cinit enabled executable running as container init process
if err = c.Ping(); err != nil {
c.Destroy()
return nil, fmt.Errorf("container: container init not responding to ping %v", err)
}
// container mount points
mounts := b.Mounts
if len(mounts) == 0 {
if mounts, err = mount.NewDefaultBuilder().
mounts = mount.NewDefaultBuilder().
WithTmpfs("w", ""). // work dir
WithTmpfs("tmp", ""). // tmp
Build(true); err != nil {
return nil, fmt.Errorf("container: failed to build rootfs mount %v", err)
}
FilterNotExist().Mounts
}
// container root directory on the host
@ -88,29 +102,40 @@ func (b *Builder) Build() (Environment, error) {
return nil, fmt.Errorf("container: failed to get work directory %v", err)
}
}
// prepare stdin / stdout / stderr
devNull, err := os.OpenFile(os.DevNull, os.O_RDWR, os.ModePerm)
if err != nil {
return nil, fmt.Errorf("container: failed to open devNull %v", err)
workDir := containerWD
if b.WorkDir != "" {
workDir = b.WorkDir
}
defer devNull.Close()
files := make([]uintptr, 0, 4)
files = append(files, devNull.Fd(), devNull.Fd())
if b.Stderr {
files = append(files, os.Stderr.Fd())
} else {
files = append(files, devNull.Fd())
hostName := containerName
if b.HostName != "" {
hostName = b.HostName
}
domainName := containerName
if b.DomainName != "" {
domainName = b.DomainName
}
// prepare container exec file
execFile, err := b.exec()
if err != nil {
return nil, fmt.Errorf("container: prepare exec %v", err)
// set configuration and check if container creation successful
if err = c.conf(&containerConfig{
WorkDir: workDir,
HostName: hostName,
DomainName: domainName,
ContainerRoot: root,
Mounts: mounts,
Cred: b.CredGenerator != nil,
}); err != nil {
c.Destroy()
return nil, err
}
return c, nil
}
defer execFile.Close()
func (b *Builder) startContainer() (*container, error) {
var (
err error
cred syscall.Credential
uidMap, gidMap []syscall.SysProcIDMap
)
// prepare host <-> container unix socket
ins, outs, err := newPassCredSocketPair()
if err != nil {
@ -125,12 +150,13 @@ func (b *Builder) Build() (Environment, error) {
}
defer outf.Close()
files = append(files, uintptr(outf.Fd()))
// prepare container running credential
if b.CredGenerator != nil {
cred = b.CredGenerator.Get()
uidMap, gidMap = getIDMapping(&cred)
} else {
uidMap = []syscall.SysProcIDMap{{HostID: os.Geteuid(), Size: 1}}
gidMap = []syscall.SysProcIDMap{{HostID: os.Getegid(), Size: 1}}
}
var cloneFlag uintptr
@ -140,45 +166,34 @@ func (b *Builder) Build() (Environment, error) {
cloneFlag = b.CloneFlags & forkexec.UnshareFlags
}
r := &forkexec.Runner{
Args: []string{os.Args[0], initArg},
Env: []string{PathEnv},
ExecFile: execFile.Fd(),
Files: files,
WorkDir: containerWD,
CloneFlags: cloneFlag,
Mounts: mounts,
HostName: containerName,
DomainName: containerName,
PivotRoot: root,
UIDMappings: uidMap,
GIDMappings: gidMap,
args := []string{os.Args[0], initArg}
if b.ExecFile != "" {
args[0] = b.ExecFile
}
pid, err := r.Start()
if err != nil {
r := exec.Cmd{
Path: args[0],
Args: args,
Env: []string{PathEnv},
Stderr: b.Stderr,
ExtraFiles: []*os.File{outf},
SysProcAttr: &syscall.SysProcAttr{
Cloneflags: cloneFlag,
UidMappings: uidMap,
GidMappings: gidMap,
AmbientCaps: []uintptr{
unix.CAP_SYS_ADMIN,
},
},
}
if err = r.Start(); err != nil {
ins.Close()
return nil, fmt.Errorf("container: failed to start container %v", err)
}
c := &container{
pid: pid,
return &container{
process: r.Process,
socket: newSocket(ins),
}
// avoid non cinit enabled executable running as container init process
if err = c.Ping(); err != nil {
c.Destroy()
return nil, fmt.Errorf("container: container init not responding to ping %v", err)
}
// set configuration and check if container creation successful
if err = c.conf(&containerConfig{
Cred: b.CredGenerator != nil,
}); err != nil {
c.Destroy()
return nil, err
}
return c, nil
}, nil
}
// Destroy kill the container process (with its children)
@ -192,13 +207,8 @@ func (c *container) Destroy() error {
defer c.mu.Unlock()
// kill process
var wstatus unix.WaitStatus
unix.Kill(c.pid, unix.SIGKILL)
// wait for container process to exit
_, err := unix.Wait4(c.pid, &wstatus, 0, nil)
for err == unix.EINTR {
_, err = unix.Wait4(c.pid, &wstatus, 0, nil)
}
c.process.Kill()
_, err := c.process.Wait()
return err
}

View File

@ -8,7 +8,7 @@ import (
"github.com/criyle/go-sandbox/pkg/unixsocket"
)
// Ping send ping message to container
// Ping send ping message to container, wait for 3 second before timeout
func (c *container) Ping() error {
c.mu.Lock()
defer c.mu.Unlock()

View File

@ -5,6 +5,7 @@ import (
"syscall"
"time"
"github.com/criyle/go-sandbox/pkg/mount"
"github.com/criyle/go-sandbox/pkg/rlimit"
"github.com/criyle/go-sandbox/runner"
)
@ -47,6 +48,14 @@ type confCmd struct {
// ContainerConfig set the container config
type containerConfig struct {
WorkDir string
HostName string
DomainName string
ContainerRoot string
Mounts []mount.Mount
Cred bool
}

4
go.mod
View File

@ -6,8 +6,8 @@ require (
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/elastic/go-seccomp-bpf v1.1.0
github.com/pkg/errors v0.9.1 // indirect
golang.org/x/net v0.0.0-20200904194848-62affa334b73
golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f
golang.org/x/net v0.0.0-20201110031124-69a78807bb2b
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68
)
replace github.com/elastic/go-seccomp-bpf => ../go-seccomp-bpf

12
go.sum
View File

@ -16,14 +16,16 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3 h1:0GoQqolDA55aaLxZyTzK/Y2ePZzZTUrRacwib7cNsYQ=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20200904194848-62affa334b73 h1:MXfv8rhZWmFeqX3GNZRsd6vOLoaCHjYEX3qkRo3YBUA=
golang.org/x/net v0.0.0-20200904194848-62affa334b73/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
golang.org/x/net v0.0.0-20201110031124-69a78807bb2b h1:uwuIcX0g4Yl1NC5XAz37xsr2lTtcqevgzYNVt49waME=
golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190405154228-4b34438f7a67/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f h1:Fqb3ao1hUmOR3GkUOg/Y+BadLwykBIzs5q8Ez2SbHyc=
golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68 h1:nxC68pudNYkKU6jWhgrqdreuFiOQWj1Fs7T3VrH4Pjw=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

View File

@ -173,7 +173,7 @@ func getMounts(dirs []string) []mount.SyscallParams {
Flags: roBind,
})
}
m, _ := builder.Build(true)
m, _ := builder.FilterNotExist().Build()
return m
}

View File

@ -95,7 +95,7 @@ func TestFork_ENOENT(t *testing.T) {
WithMount(
mount.Mount{
Source: "NOT_EXISTS",
}).Build(false)
}).Build()
if err != nil {
t.Fatal(err)
}

View File

@ -22,16 +22,12 @@ func NewDefaultBuilder() *Builder {
}
// Build creates sequence of syscalls for fork_exec
// skipNotExists skips bind mounts that source not exists
func (b *Builder) Build(skipNotExists bool) ([]SyscallParams, error) {
func (b *Builder) Build() ([]SyscallParams, error) {
var err error
ret := make([]SyscallParams, 0, len(b.Mounts))
for _, m := range b.Mounts {
var mknod bool
if mknod, err = isBindMountFileOrNotExists(m); err != nil {
if skipNotExists {
continue
}
return nil, err
}
sp, err := m.ToSyscall()
@ -44,8 +40,23 @@ func (b *Builder) Build(skipNotExists bool) ([]SyscallParams, error) {
return ret, nil
}
// FilterNotExist removes bind mount that does not exists
func (b *Builder) FilterNotExist() *Builder {
rt := b.Mounts[:0]
for _, m := range b.Mounts {
if m.IsBindMount() {
if _, err := os.Stat(m.Source); os.IsNotExist(err) {
continue
}
}
rt = append(rt, m)
}
b.Mounts = rt
return b
}
func isBindMountFileOrNotExists(m Mount) (bool, error) {
if m.Flags&unix.MS_BIND == unix.MS_BIND {
if m.IsBindMount() {
if fi, err := os.Stat(m.Source); os.IsNotExist(err) {
return false, err
} else if !fi.IsDir() {

View File

@ -3,12 +3,13 @@ package mount
import (
"fmt"
"os"
"path/filepath"
"syscall"
)
// Mount calls mount syscall
func (m *Mount) Mount() error {
if err := os.MkdirAll(m.Target, 0755); err != nil {
if err := ensureMountTargetExists(m.Source, m.Target); err != nil {
return err
}
if err := syscall.Mount(m.Source, m.Target, m.FsType, m.Flags, m.Data); err != nil {
@ -24,6 +25,36 @@ func (m *Mount) Mount() error {
return nil
}
// IsBindMount returns if it is a bind mount
func (m Mount) IsBindMount() bool {
return m.Flags&syscall.MS_BIND == syscall.MS_BIND
}
// IsReadOnly returns if it is a readonly mount
func (m Mount) IsReadOnly() bool {
return m.Flags&syscall.MS_RDONLY == syscall.MS_RDONLY
}
func ensureMountTargetExists(source, target string) error {
isFile := false
if fi, err := os.Stat(source); err == nil {
isFile = !fi.IsDir()
}
dir := target
if isFile {
dir = filepath.Dir(target)
}
if err := os.MkdirAll(dir, 0755); err != nil {
return err
}
if isFile {
if err := syscall.Mknod(target, 0755, 0); err != nil {
return err
}
}
return nil
}
func (m Mount) String() string {
switch {
case m.Flags&syscall.MS_BIND == syscall.MS_BIND: