move fork child to standalone file

This commit is contained in:
criyle 2019-10-29 02:21:43 -07:00
parent 348ed481d3
commit 03ec0a5ccc
6 changed files with 459 additions and 449 deletions

View File

@ -1,5 +1,7 @@
# go-sandbox
[![GoDoc](https://godoc.org/github.com/criyle/go-sandbox?status.svg)](https://godoc.org/github.com/criyle/go-sandbox)
Original goal was to reimplement [uoj-judger/run_program](https://github.com/vfleaking/uoj) in GO language using [libseccomp](https://github.com/pkg/seccomp/libseccomp-golang). As technology grows, it also implements new technologies including Linux namespace and cgroup.
## Install
@ -29,7 +31,7 @@ Default file access syscall check:
- check file read: `readlink`, `readlinkat`
- check file write: `unlink`, `unlinkat`, `chmod`, `rename`
- check file access: `stat`, `lstat`, `access`, `faccessat`
- check file exec: `execveat`
- check file exec: `execve`, `execveat`
### linux namespace + cgroup
@ -91,19 +93,17 @@ $ go test -bench . -benchtime 10s
goos: linux
goarch: amd64
pkg: github.com/criyle/go-sandbox/pkg/forkexec
BenchmarkSimpleFork-4 10000 1106064 ns/op
BenchmarkUnsharePid-4 10000 1367824 ns/op
BenchmarkUnshareUser-4 10000 1311523 ns/op
BenchmarkUnshareUts-4 10000 1140427 ns/op
BenchmarkUnshareCgroup-4 10000 1112713 ns/op
BenchmarkUnshareIpc-4 300 58730786 ns/op
BenchmarkUnshareMount-4 300 55540758 ns/op
BenchmarkUnshareNet-4 100 396957720 ns/op
BenchmarkFastUnshareMountPivot-4 100 114364585 ns/op
BenchmarkUnshareAll-4 100 851014031 ns/op
BenchmarkUnshareMountPivot-4 20 901204445 ns/op
BenchmarkSimpleFork-4 12789 870486 ns/op
BenchmarkUnsharePid-4 13172 917304 ns/op
BenchmarkUnshareUser-4 13148 927952 ns/op
BenchmarkUnshareUts-4 13170 884606 ns/op
BenchmarkUnshareCgroup-4 13650 895186 ns/op
BenchmarkUnshareIpc-4 196 66418708 ns/op
BenchmarkUnshareMount-4 243 46957682 ns/op
BenchmarkUnshareNet-4 100 411869776 ns/op
BenchmarkFastUnshareMountPivot-4 120 107310917 ns/op
BenchmarkUnshareAll-4 100 837352275 ns/op
BenchmarkUnshareMountPivot-4 12 913099234 ns/op
PASS
ok github.com/criyle/go-sandbox/pkg/forkexec 262.112s
ok github.com/criyle/go-sandbox/pkg/forkexec 300.744s
```
## TODO

View File

@ -20,13 +20,13 @@ const (
// used by unshare remount / to private
var (
none = [...]byte{'n', 'o', 'n', 'e', 0}
slash = [...]byte{'/', 0}
empty = [...]byte{0}
tmpfs = [...]byte{'t', 'm', 'p', 'f', 's', 0}
none = []byte("none\000")
slash = []byte("/\000")
empty = []byte("\000")
tmpfs = []byte("tmpfs\000")
// tmp dir made by pivot_root
OldRoot = "old_root"
oldRoot = []byte("old_root\000")
// go does not allow constant uintptr to be negative...
_AT_FDCWD = unix.AT_FDCWD

View File

@ -18,17 +18,9 @@ func afterForkInChild()
// Start will fork, load seccomp and execve and being traced by ptrace
// Return pid and potential error
// Reference to src/syscall/exec_linux.go
// The runtime OS thread must be locked before calling this function
// if ptrace is set to true
//go:norace
func (r *Runner) Start() (int, error) {
var (
err1, err2 syscall.Errno
r1 uintptr
unshareUser = r.UnshareFlags&unix.CLONE_NEWUSER == unix.CLONE_NEWUSER
)
argv0, argv, env, err := prepareExec(r.Args, r.Env)
if err != nil {
return 0, err
@ -53,7 +45,7 @@ func (r *Runner) Start() (int, error) {
}
// prepare pivot_root param
pivotRoot, oldRoot, err := preparePivotRoot(r.PivotRoot)
pivotRoot, err := syscallStringFromString(r.PivotRoot)
if err != nil {
return 0, err
}
@ -66,424 +58,95 @@ func (r *Runner) Start() (int, error) {
return 0, err
}
// similar to exec_linux, avoid side effect by shuffling around
fd, nextfd := prepareFds(r.Files)
pipe := p[1]
// fork in child
pid, err1 := forkAndExecInChild(r, argv0, argv, env, workdir, hostname, domainname, pivotRoot, p)
// Acquire the fork lock so that no other threads
// create new fds that are not yet close-on-exec
// before we fork.
syscall.ForkLock.Lock()
// restore all signals
afterFork()
syscall.ForkLock.Unlock()
// About to call fork.
// No more allocation or calls of non-assembly functions.
beforeFork()
return syncWithChild(r, p, int(pid), err1)
}
// UnshareFlags (new namespaces) is activated by clone syscall
pid, _, err1 := syscall.RawSyscall6(syscall.SYS_CLONE, uintptr(syscall.SIGCHLD)|(r.UnshareFlags&UnshareFlags), 0, 0, 0, 0, 0)
if err1 != 0 || pid != 0 {
// restore all signals
afterFork()
syscall.ForkLock.Unlock()
func syncWithChild(r *Runner, p [2]int, pid int, err1 syscall.Errno) (int, error) {
var (
r1 uintptr
err2 syscall.Errno
err error
unshareUser = r.UnshareFlags&unix.CLONE_NEWUSER == unix.CLONE_NEWUSER
)
// sync with child
unix.Close(p[1])
// sync with child
unix.Close(p[1])
// clone syscall failed
if err1 != 0 {
unix.Close(p[0])
return int(pid), syscall.Errno(err1)
}
// synchronize with child for uid / gid map
if unshareUser {
if err = writeIDMaps(int(pid)); err != nil {
err2 = err.(syscall.Errno)
}
syscall.RawSyscall(syscall.SYS_WRITE, uintptr(p[0]), uintptr(unsafe.Pointer(&err2)), uintptr(unsafe.Sizeof(err2)))
}
r1, _, err1 = syscall.RawSyscall(syscall.SYS_READ, uintptr(p[0]), uintptr(unsafe.Pointer(&err2)), uintptr(unsafe.Sizeof(err2)))
// child returned error code
if r1 != unsafe.Sizeof(err2) || err2 != 0 || err1 != 0 {
unix.Close(p[0])
if r1 == unsafe.Sizeof(err2) {
err = syscall.Errno(err2)
}
if err == nil {
err = syscall.EPIPE
}
handleChildFailed(pid)
return 0, err
}
if r.SyncFunc != nil {
err = r.SyncFunc(int(pid))
}
// if syncfunc return error, then fail child immediately. Otherwise, ack child (err1 == 0)
if err == nil {
syscall.RawSyscall(syscall.SYS_WRITE, uintptr(p[0]), uintptr(unsafe.Pointer(&err1)), uintptr(unsafe.Sizeof((err1))))
} else {
unix.Close(p[0])
handleChildFailed(pid)
return 0, err
}
// if stopped before execve, then do not wait until execve
if r.Ptrace && r.Seccomp != nil || r.StopBeforeSeccomp {
unix.Close(p[0])
return int(pid), nil
}
// if read anything mean child failed after sync (close_on_exec so it should not block)
r1, _, err1 = syscall.RawSyscall(syscall.SYS_READ, uintptr(p[0]), uintptr(unsafe.Pointer(&err2)), uintptr(unsafe.Sizeof(err2)))
// clone syscall failed
if err1 != 0 {
unix.Close(p[0])
if r1 != 0 || err1 != 0 {
if r1 == unsafe.Sizeof(err2) {
err = syscall.Errno(err2)
}
if err == nil {
err = syscall.EPIPE
}
handleChildFailed(pid)
return 0, err
return 0, syscall.Errno(err1)
}
// synchronize with child for uid / gid map
if unshareUser {
if err = writeIDMaps(int(pid)); err != nil {
err2 = err.(syscall.Errno)
}
syscall.RawSyscall(syscall.SYS_WRITE, uintptr(p[0]), uintptr(unsafe.Pointer(&err2)), uintptr(unsafe.Sizeof(err2)))
}
r1, _, err1 = syscall.RawSyscall(syscall.SYS_READ, uintptr(p[0]), uintptr(unsafe.Pointer(&err2)), uintptr(unsafe.Sizeof(err2)))
// child returned error code
if r1 != unsafe.Sizeof(err2) || err2 != 0 || err1 != 0 {
err = handlePipeError(r1, err2)
goto fail
}
// if syncfunc return error, then fail child immediately
if r.SyncFunc != nil {
if err = r.SyncFunc(int(pid)); err != nil {
goto fail
}
}
// otherwise, ack child (err1 == 0)
syscall.RawSyscall(syscall.SYS_WRITE, uintptr(p[0]), uintptr(unsafe.Pointer(&err1)), uintptr(unsafe.Sizeof(err1)))
// if stopped before execve, then do not wait until execve
if r.Ptrace && r.Seccomp != nil || r.StopBeforeSeccomp {
unix.Close(p[0])
return int(pid), nil
}
// In child process
afterForkInChild()
// Notice: cannot call any GO functions beyond this point
// If usernamespace is unshared, uid map and gid map is required to create folders
// and files
// We need parent to setup uid_map / gid_map for us since we do not have capabilities
// in the original namespace
// At the same time, socket pair / pipe synchronization is required as well
if _, _, err1 = syscall.RawSyscall(syscall.SYS_CLOSE, uintptr(p[0]), 0, 0); err1 != 0 {
goto childerror
}
if unshareUser {
r1, _, err1 = syscall.RawSyscall(syscall.SYS_READ, uintptr(pipe), uintptr(unsafe.Pointer(&err2)), unsafe.Sizeof(err2))
if err1 != 0 {
goto childerror
}
if r1 != unsafe.Sizeof(err2) {
err1 = syscall.EINVAL
goto childerror
}
if err2 != 0 {
err1 = err2
goto childerror
}
// if read anything mean child failed after sync (close_on_exec so it should not block)
r1, _, err1 = syscall.RawSyscall(syscall.SYS_READ, uintptr(p[0]), uintptr(unsafe.Pointer(&err2)), uintptr(unsafe.Sizeof(err2)))
unix.Close(p[0])
if r1 != 0 || err1 != 0 {
err = handlePipeError(r1, err2)
goto failAfterClose
}
return int(pid), nil
// Get pid of child
pid, _, err1 = syscall.RawSyscall(syscall.SYS_GETPID, 0, 0, 0)
if err1 != 0 {
goto childerror
}
fail:
unix.Close(p[0])
// Pass 1 & pass 2 assigns fds for child process
// Pass 1: fd[i] < i => nextfd
if pipe < nextfd {
_, _, err1 = syscall.RawSyscall(syscall.SYS_DUP3, uintptr(pipe), uintptr(nextfd), syscall.O_CLOEXEC)
if err1 != 0 {
goto childerror
}
pipe = nextfd
nextfd++
}
if r.ExecFile > 0 && int(r.ExecFile) < nextfd {
_, _, err1 = syscall.RawSyscall(syscall.SYS_DUP3, r.ExecFile, uintptr(nextfd), syscall.O_CLOEXEC)
if err1 != 0 {
goto childerror
}
r.ExecFile = uintptr(nextfd)
nextfd++
}
for i := 0; i < len(fd); i++ {
// Avoid fd rewrite
for nextfd == i || (r.ExecFile > 0 && nextfd == int(r.ExecFile)) {
nextfd++
}
if fd[i] >= 0 && fd[i] < int(i) {
_, _, err1 = syscall.RawSyscall(syscall.SYS_DUP3, uintptr(fd[i]), uintptr(nextfd), syscall.O_CLOEXEC)
if err1 != 0 {
goto childerror
}
// Set up close on exec
fd[i] = nextfd
nextfd++
}
}
// Pass 2: fd[i] => i
for i := 0; i < len(fd); i++ {
if fd[i] == -1 {
syscall.RawSyscall(syscall.SYS_CLOSE, uintptr(i), 0, 0)
continue
}
if fd[i] == int(i) {
// dup2(i, i) will not clear close on exec flag, need to reset the flag
_, _, err1 = syscall.RawSyscall(syscall.SYS_FCNTL, uintptr(fd[i]), syscall.F_SETFD, 0)
if err1 != 0 {
goto childerror
}
continue
}
_, _, err1 = syscall.RawSyscall(syscall.SYS_DUP3, uintptr(fd[i]), uintptr(i), 0)
if err1 != 0 {
goto childerror
}
}
// Set the pgid, so that the wait operation can apply to only certain
// subgroup of processes
_, _, err1 = syscall.RawSyscall(syscall.SYS_SETPGID, 0, 0, 0)
if err1 != 0 {
goto childerror
}
// If mount point is unshared, mark root as private to avoid propagate
// outside to the original mount namespace
if r.UnshareFlags&syscall.CLONE_NEWNS == syscall.CLONE_NEWNS {
_, _, err1 = syscall.RawSyscall6(syscall.SYS_MOUNT, uintptr(unsafe.Pointer(&none[0])),
uintptr(unsafe.Pointer(&slash[0])), 0, syscall.MS_REC|syscall.MS_PRIVATE, 0, 0)
if err1 != 0 {
goto childerror
}
}
// mount tmpfs & chdir to new root before performing mounts
if pivotRoot != nil {
// mount("tmpfs", root, "tmpfs", 0, "")
_, _, err1 = syscall.RawSyscall6(syscall.SYS_MOUNT, uintptr(unsafe.Pointer(&tmpfs[0])),
uintptr(unsafe.Pointer(pivotRoot)), uintptr(unsafe.Pointer(&tmpfs[0])), 0,
uintptr(unsafe.Pointer(&empty[0])), 0)
if err1 != 0 {
goto childerror
}
_, _, err1 = syscall.RawSyscall(syscall.SYS_CHDIR, uintptr(unsafe.Pointer(pivotRoot)), 0, 0)
if err1 != 0 {
goto childerror
}
}
// performing mounts
for _, m := range r.Mounts {
// mkdirs(target)
for i, p := range m.Prefixes {
// if target mount point is a file, mknod(target)
if i == len(m.Prefixes)-1 && m.MakeNod {
_, _, err1 = syscall.RawSyscall(syscall.SYS_MKNODAT, uintptr(_AT_FDCWD), uintptr(unsafe.Pointer(p)), 0755)
if err1 != 0 && err1 != syscall.EEXIST {
goto childerror
}
break
}
_, _, err1 = syscall.RawSyscall(syscall.SYS_MKDIRAT, uintptr(_AT_FDCWD), uintptr(unsafe.Pointer(p)), 0755)
if err1 != 0 && err1 != syscall.EEXIST {
goto childerror
}
}
// mount(source, target, fsType, flags, data)
_, _, err1 = syscall.RawSyscall6(syscall.SYS_MOUNT, uintptr(unsafe.Pointer(m.Source)),
uintptr(unsafe.Pointer(m.Target)), uintptr(unsafe.Pointer(m.FsType)), uintptr(m.Flags),
uintptr(unsafe.Pointer(m.Data)), 0)
if err1 != 0 {
goto childerror
}
// bind mount is not respect ro flag so that read-only bind mount needs remount
if m.Flags&bindRo == bindRo {
_, _, err1 = syscall.RawSyscall6(syscall.SYS_MOUNT, uintptr(unsafe.Pointer(&empty[0])),
uintptr(unsafe.Pointer(m.Target)), uintptr(unsafe.Pointer(m.FsType)),
uintptr(m.Flags|syscall.MS_REMOUNT), uintptr(unsafe.Pointer(m.Data)), 0)
if err1 != 0 {
goto childerror
}
}
}
// pivit_root
if pivotRoot != nil {
// mkdir("old_root")
_, _, err1 = syscall.RawSyscall(syscall.SYS_MKDIRAT, uintptr(_AT_FDCWD), uintptr(unsafe.Pointer(oldRoot)), 0755)
if err1 != 0 {
goto childerror
}
// pivot_root(root, "old_root")
_, _, err1 = syscall.RawSyscall(syscall.SYS_PIVOT_ROOT, uintptr(unsafe.Pointer(pivotRoot)), uintptr(unsafe.Pointer(oldRoot)), 0)
if err1 != 0 {
goto childerror
}
// umount("old_root", MNT_DETACH)
_, _, err1 = syscall.RawSyscall(syscall.SYS_UMOUNT2, uintptr(unsafe.Pointer(oldRoot)), syscall.MNT_DETACH, 0)
if err1 != 0 {
goto childerror
}
// rmdir("old_root")
_, _, err1 = syscall.RawSyscall(syscall.SYS_UNLINKAT, uintptr(_AT_FDCWD), uintptr(unsafe.Pointer(oldRoot)), uintptr(unix.AT_REMOVEDIR))
if err1 != 0 {
goto childerror
}
// mount("tmpfs", "/", "tmpfs", MS_BIND | MS_REMOUNT | MS_RDONLY | MS_NOATIME | MS_NOSUID, nil)
_, _, err1 = syscall.RawSyscall6(syscall.SYS_MOUNT, uintptr(unsafe.Pointer(&tmpfs[0])),
uintptr(unsafe.Pointer(&slash[0])), uintptr(unsafe.Pointer(&tmpfs[0])),
uintptr(syscall.MS_BIND|syscall.MS_REMOUNT|syscall.MS_RDONLY|syscall.MS_NOATIME|syscall.MS_NOSUID),
uintptr(unsafe.Pointer(&empty[0])), 0)
if err1 != 0 {
goto childerror
}
}
// SetHostName
if hostname != nil {
_, _, err1 = syscall.RawSyscall(syscall.SYS_SETHOSTNAME,
uintptr(unsafe.Pointer(hostname)), uintptr(len(r.HostName)), 0)
}
// SetDomainName
if domainname != nil {
_, _, err1 = syscall.RawSyscall(syscall.SYS_SETDOMAINNAME,
uintptr(unsafe.Pointer(domainname)), uintptr(len(r.DomainName)), 0)
}
// chdir for child
if workdir != nil {
_, _, err1 = syscall.RawSyscall(syscall.SYS_CHDIR, uintptr(unsafe.Pointer(workdir)), 0, 0)
if err1 != 0 {
goto childerror
}
}
// Set limit
for _, rlim := range r.RLimits {
// prlimit instead of setrlimit to avoid 32-bit limitation (linux > 3.2)
_, _, err1 = syscall.RawSyscall6(syscall.SYS_PRLIMIT64, 0, uintptr(rlim.Res), uintptr(unsafe.Pointer(&rlim.Rlim)), 0, 0, 0)
if err1 != 0 {
goto childerror
}
}
// No new privs
if r.NoNewPrivs || r.Seccomp != nil {
_, _, err1 = syscall.RawSyscall6(syscall.SYS_PRCTL, unix.PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0, 0)
if err1 != 0 {
goto childerror
}
}
// Drop all capabilities
if r.DropCaps {
_, _, err1 = syscall.RawSyscall(syscall.SYS_CAPSET, uintptr(unsafe.Pointer(&dropCapHeader)), uintptr(unsafe.Pointer(&dropCapData)), 0)
if err1 != 0 {
goto childerror
}
}
// Enable Ptrace & sync with parent (since ptrace_me is a blocking operation)
if r.Ptrace && r.Seccomp != nil {
err2 = 0
r1, _, err1 = syscall.RawSyscall(syscall.SYS_WRITE, uintptr(pipe), uintptr(unsafe.Pointer(&err2)), uintptr(unsafe.Sizeof(err2)))
if r1 == 0 || err1 != 0 {
goto childerror
}
r1, _, err1 = syscall.RawSyscall(syscall.SYS_READ, uintptr(pipe), uintptr(unsafe.Pointer(&err2)), uintptr(unsafe.Sizeof(err2)))
if r1 == 0 || err1 != 0 {
goto childerror
}
_, _, err1 = syscall.RawSyscall(syscall.SYS_PTRACE, uintptr(syscall.PTRACE_TRACEME), 0, 0)
if err1 != 0 {
goto childerror
}
}
// if both seccomp and ptrace is defined, then seccomp filter should have
// traced execve, thus child need parent attached to it first
// actually, this is not effective if pid namespace is unshared
if r.StopBeforeSeccomp || (r.Seccomp != nil && r.Ptrace) {
// Stop to wait for ptrace tracer
_, _, err1 = syscall.RawSyscall(syscall.SYS_KILL, pid, uintptr(syscall.SIGSTOP), 0)
if err1 != 0 {
goto childerror
}
}
// Load seccomp, stop and wait for tracer
if r.Seccomp != nil {
// If execve is seccomp trapped, then tracee stop is necessary
// otherwise execve will fail due to ENOSYS
// Do getpid and kill to send SYS_KILL to self
// need to do before seccomp as these might be traced
// Load seccomp filter
_, _, err1 = syscall.RawSyscall(unix.SYS_SECCOMP, SECCOMP_SET_MODE_FILTER, SECCOMP_FILTER_FLAG_TSYNC, uintptr(unsafe.Pointer(r.Seccomp)))
if err1 != 0 {
goto childerror
}
}
// Before exec, sync with parent through pipe (configured as close_on_exec)
if !r.Ptrace || r.Seccomp == nil {
err2 = 0
r1, _, err1 = syscall.RawSyscall(syscall.SYS_WRITE, uintptr(pipe), uintptr(unsafe.Pointer(&err2)), uintptr(unsafe.Sizeof(err2)))
if r1 == 0 || err1 != 0 {
goto childerror
}
r1, _, err1 = syscall.RawSyscall(syscall.SYS_READ, uintptr(pipe), uintptr(unsafe.Pointer(&err2)), uintptr(unsafe.Sizeof(err2)))
if r1 == 0 || err1 != 0 {
goto childerror
}
}
// Enable ptrace if no seccomp is needed
if r.Ptrace && r.Seccomp == nil {
_, _, err1 = syscall.RawSyscall(syscall.SYS_PTRACE, uintptr(syscall.PTRACE_TRACEME), 0, 0)
if err1 != 0 {
goto childerror
}
}
// at this point, runner is successfully attached for seccomp trap filter
// or execve trapped without seccomp filter
// time to exec
// if execfile fd is specified, call fexecve
if r.ExecFile > 0 {
_, _, err1 = syscall.RawSyscall6(unix.SYS_EXECVEAT, r.ExecFile,
uintptr(unsafe.Pointer(&empty[0])),
uintptr(unsafe.Pointer(&argv[0])),
uintptr(unsafe.Pointer(&env[0])), unix.AT_EMPTY_PATH, 0)
} else {
_, _, err1 = syscall.RawSyscall6(unix.SYS_EXECVEAT, uintptr(_AT_FDCWD),
uintptr(unsafe.Pointer(argv0)),
uintptr(unsafe.Pointer(&argv[0])),
uintptr(unsafe.Pointer(&env[0])), 0, 0)
}
childerror:
// send error code on pipe
syscall.RawSyscall(unix.SYS_WRITE, uintptr(pipe), uintptr(unsafe.Pointer(&err1)), unsafe.Sizeof(err1))
for {
syscall.RawSyscall(syscall.SYS_EXIT, uintptr(err1+err2), 0, 0)
}
// cannot reach this point
failAfterClose:
handleChildFailed(int(pid))
return 0, err
}
func handleChildFailed(pid uintptr) {
// check pipe error
func handlePipeError(r1 uintptr, errno syscall.Errno) error {
if r1 == unsafe.Sizeof(errno) {
return syscall.Errno(errno)
}
return syscall.EPIPE
}
func handleChildFailed(pid int) {
var wstatus syscall.WaitStatus
// make sure not blocked
syscall.Kill(int(pid), syscall.SIGKILL)
syscall.Kill(pid, syscall.SIGKILL)
// child failed; wait for it to exit, to make sure the zombies don't accumulate
_, err := syscall.Wait4(int(pid), &wstatus, 0, nil)
_, err := syscall.Wait4(pid, &wstatus, 0, nil)
for err == syscall.EINTR {
_, err = syscall.Wait4(int(pid), &wstatus, 0, nil)
_, err = syscall.Wait4(pid, &wstatus, 0, nil)
}
}

363
pkg/forkexec/fork_child.go Normal file
View File

@ -0,0 +1,363 @@
package forkexec
import (
"syscall"
"unsafe"
"golang.org/x/sys/unix"
)
// Reference to src/syscall/exec_linux.go
//go:norace
func forkAndExecInChild(r *Runner, argv0 *byte, argv, env []*byte, workdir, hostname, domainname, pivotRoot *byte, p [2]int) (r1 uintptr, err1 syscall.Errno) {
var (
pid uintptr
err2 syscall.Errno
unshareUser = r.UnshareFlags&unix.CLONE_NEWUSER == unix.CLONE_NEWUSER
)
// similar to exec_linux, avoid side effect by shuffling around
fd, nextfd := prepareFds(r.Files)
pipe := p[1]
// Acquire the fork lock so that no other threads
// create new fds that are not yet close-on-exec
// before we fork.
syscall.ForkLock.Lock()
// About to call fork.
// No more allocation or calls of non-assembly functions.
beforeFork()
// UnshareFlags (new namespaces) is activated by clone syscall
r1, _, err1 = syscall.RawSyscall6(syscall.SYS_CLONE, uintptr(syscall.SIGCHLD)|(r.UnshareFlags&UnshareFlags), 0, 0, 0, 0, 0)
if err1 != 0 || r1 != 0 {
// in parent process, immediate return
return
}
// In child process
afterForkInChild()
// Notice: cannot call any GO functions beyond this point
// If usernamespace is unshared, uid map and gid map is required to create folders
// and files
// We need parent to setup uid_map / gid_map for us since we do not have capabilities
// in the original namespace
// At the same time, socket pair / pipe synchronization is required as well
if _, _, err1 = syscall.RawSyscall(syscall.SYS_CLOSE, uintptr(p[0]), 0, 0); err1 != 0 {
goto childerror
}
if unshareUser {
r1, _, err1 = syscall.RawSyscall(syscall.SYS_READ, uintptr(pipe), uintptr(unsafe.Pointer(&err2)), unsafe.Sizeof(err2))
if err1 != 0 {
goto childerror
}
if r1 != unsafe.Sizeof(err2) {
err1 = syscall.EINVAL
goto childerror
}
if err2 != 0 {
err1 = err2
goto childerror
}
}
// Get pid of child
pid, _, err1 = syscall.RawSyscall(syscall.SYS_GETPID, 0, 0, 0)
if err1 != 0 {
goto childerror
}
// Pass 1 & pass 2 assigns fds for child process
// Pass 1: fd[i] < i => nextfd
if pipe < nextfd {
_, _, err1 = syscall.RawSyscall(syscall.SYS_DUP3, uintptr(pipe), uintptr(nextfd), syscall.O_CLOEXEC)
if err1 != 0 {
goto childerror
}
pipe = nextfd
nextfd++
}
if r.ExecFile > 0 && int(r.ExecFile) < nextfd {
_, _, err1 = syscall.RawSyscall(syscall.SYS_DUP3, r.ExecFile, uintptr(nextfd), syscall.O_CLOEXEC)
if err1 != 0 {
goto childerror
}
r.ExecFile = uintptr(nextfd)
nextfd++
}
for i := 0; i < len(fd); i++ {
// Avoid fd rewrite
for nextfd == i || (r.ExecFile > 0 && nextfd == int(r.ExecFile)) {
nextfd++
}
if fd[i] >= 0 && fd[i] < int(i) {
_, _, err1 = syscall.RawSyscall(syscall.SYS_DUP3, uintptr(fd[i]), uintptr(nextfd), syscall.O_CLOEXEC)
if err1 != 0 {
goto childerror
}
// Set up close on exec
fd[i] = nextfd
nextfd++
}
}
// Pass 2: fd[i] => i
for i := 0; i < len(fd); i++ {
if fd[i] == -1 {
syscall.RawSyscall(syscall.SYS_CLOSE, uintptr(i), 0, 0)
continue
}
if fd[i] == int(i) {
// dup2(i, i) will not clear close on exec flag, need to reset the flag
_, _, err1 = syscall.RawSyscall(syscall.SYS_FCNTL, uintptr(fd[i]), syscall.F_SETFD, 0)
if err1 != 0 {
goto childerror
}
continue
}
_, _, err1 = syscall.RawSyscall(syscall.SYS_DUP3, uintptr(fd[i]), uintptr(i), 0)
if err1 != 0 {
goto childerror
}
}
// Set the pgid, so that the wait operation can apply to only certain
// subgroup of processes
_, _, err1 = syscall.RawSyscall(syscall.SYS_SETPGID, 0, 0, 0)
if err1 != 0 {
goto childerror
}
// If mount point is unshared, mark root as private to avoid propagate
// outside to the original mount namespace
if r.UnshareFlags&syscall.CLONE_NEWNS == syscall.CLONE_NEWNS {
_, _, err1 = syscall.RawSyscall6(syscall.SYS_MOUNT, uintptr(unsafe.Pointer(&none[0])),
uintptr(unsafe.Pointer(&slash[0])), 0, syscall.MS_REC|syscall.MS_PRIVATE, 0, 0)
if err1 != 0 {
goto childerror
}
}
// mount tmpfs & chdir to new root before performing mounts
if pivotRoot != nil {
// mount("tmpfs", root, "tmpfs", 0, "")
_, _, err1 = syscall.RawSyscall6(syscall.SYS_MOUNT, uintptr(unsafe.Pointer(&tmpfs[0])),
uintptr(unsafe.Pointer(pivotRoot)), uintptr(unsafe.Pointer(&tmpfs[0])), 0,
uintptr(unsafe.Pointer(&empty[0])), 0)
if err1 != 0 {
goto childerror
}
_, _, err1 = syscall.RawSyscall(syscall.SYS_CHDIR, uintptr(unsafe.Pointer(pivotRoot)), 0, 0)
if err1 != 0 {
goto childerror
}
}
// performing mounts
for _, m := range r.Mounts {
// mkdirs(target)
for i, p := range m.Prefixes {
// if target mount point is a file, mknod(target)
if i == len(m.Prefixes)-1 && m.MakeNod {
_, _, err1 = syscall.RawSyscall(syscall.SYS_MKNODAT, uintptr(_AT_FDCWD), uintptr(unsafe.Pointer(p)), 0755)
if err1 != 0 && err1 != syscall.EEXIST {
goto childerror
}
break
}
_, _, err1 = syscall.RawSyscall(syscall.SYS_MKDIRAT, uintptr(_AT_FDCWD), uintptr(unsafe.Pointer(p)), 0755)
if err1 != 0 && err1 != syscall.EEXIST {
goto childerror
}
}
// mount(source, target, fsType, flags, data)
_, _, err1 = syscall.RawSyscall6(syscall.SYS_MOUNT, uintptr(unsafe.Pointer(m.Source)),
uintptr(unsafe.Pointer(m.Target)), uintptr(unsafe.Pointer(m.FsType)), uintptr(m.Flags),
uintptr(unsafe.Pointer(m.Data)), 0)
if err1 != 0 {
goto childerror
}
// bind mount is not respect ro flag so that read-only bind mount needs remount
if m.Flags&bindRo == bindRo {
_, _, err1 = syscall.RawSyscall6(syscall.SYS_MOUNT, uintptr(unsafe.Pointer(&empty[0])),
uintptr(unsafe.Pointer(m.Target)), uintptr(unsafe.Pointer(m.FsType)),
uintptr(m.Flags|syscall.MS_REMOUNT), uintptr(unsafe.Pointer(m.Data)), 0)
if err1 != 0 {
goto childerror
}
}
}
// pivit_root
if pivotRoot != nil {
// mkdir("old_root")
_, _, err1 = syscall.RawSyscall(syscall.SYS_MKDIRAT, uintptr(_AT_FDCWD), uintptr(unsafe.Pointer(&oldRoot[0])), 0755)
if err1 != 0 {
goto childerror
}
// pivot_root(root, "old_root")
_, _, err1 = syscall.RawSyscall(syscall.SYS_PIVOT_ROOT, uintptr(unsafe.Pointer(pivotRoot)), uintptr(unsafe.Pointer(&oldRoot[0])), 0)
if err1 != 0 {
goto childerror
}
// umount("old_root", MNT_DETACH)
_, _, err1 = syscall.RawSyscall(syscall.SYS_UMOUNT2, uintptr(unsafe.Pointer(&oldRoot[0])), syscall.MNT_DETACH, 0)
if err1 != 0 {
goto childerror
}
// rmdir("old_root")
_, _, err1 = syscall.RawSyscall(syscall.SYS_UNLINKAT, uintptr(_AT_FDCWD), uintptr(unsafe.Pointer(&oldRoot[0])), uintptr(unix.AT_REMOVEDIR))
if err1 != 0 {
goto childerror
}
// mount("tmpfs", "/", "tmpfs", MS_BIND | MS_REMOUNT | MS_RDONLY | MS_NOATIME | MS_NOSUID, nil)
_, _, err1 = syscall.RawSyscall6(syscall.SYS_MOUNT, uintptr(unsafe.Pointer(&tmpfs[0])),
uintptr(unsafe.Pointer(&slash[0])), uintptr(unsafe.Pointer(&tmpfs[0])),
uintptr(syscall.MS_BIND|syscall.MS_REMOUNT|syscall.MS_RDONLY|syscall.MS_NOATIME|syscall.MS_NOSUID),
uintptr(unsafe.Pointer(&empty[0])), 0)
if err1 != 0 {
goto childerror
}
}
// SetHostName
if hostname != nil {
_, _, err1 = syscall.RawSyscall(syscall.SYS_SETHOSTNAME,
uintptr(unsafe.Pointer(hostname)), uintptr(len(r.HostName)), 0)
}
// SetDomainName
if domainname != nil {
_, _, err1 = syscall.RawSyscall(syscall.SYS_SETDOMAINNAME,
uintptr(unsafe.Pointer(domainname)), uintptr(len(r.DomainName)), 0)
}
// chdir for child
if workdir != nil {
_, _, err1 = syscall.RawSyscall(syscall.SYS_CHDIR, uintptr(unsafe.Pointer(workdir)), 0, 0)
if err1 != 0 {
goto childerror
}
}
// Set limit
for _, rlim := range r.RLimits {
// prlimit instead of setrlimit to avoid 32-bit limitation (linux > 3.2)
_, _, err1 = syscall.RawSyscall6(syscall.SYS_PRLIMIT64, 0, uintptr(rlim.Res), uintptr(unsafe.Pointer(&rlim.Rlim)), 0, 0, 0)
if err1 != 0 {
goto childerror
}
}
// No new privs
if r.NoNewPrivs || r.Seccomp != nil {
_, _, err1 = syscall.RawSyscall6(syscall.SYS_PRCTL, unix.PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0, 0)
if err1 != 0 {
goto childerror
}
}
// Drop all capabilities
if r.DropCaps {
_, _, err1 = syscall.RawSyscall(syscall.SYS_CAPSET, uintptr(unsafe.Pointer(&dropCapHeader)), uintptr(unsafe.Pointer(&dropCapData)), 0)
if err1 != 0 {
goto childerror
}
}
// Enable Ptrace & sync with parent (since ptrace_me is a blocking operation)
if r.Ptrace && r.Seccomp != nil {
err2 = 0
r1, _, err1 = syscall.RawSyscall(syscall.SYS_WRITE, uintptr(pipe), uintptr(unsafe.Pointer(&err2)), uintptr(unsafe.Sizeof(err2)))
if r1 == 0 || err1 != 0 {
goto childerror
}
r1, _, err1 = syscall.RawSyscall(syscall.SYS_READ, uintptr(pipe), uintptr(unsafe.Pointer(&err2)), uintptr(unsafe.Sizeof(err2)))
if r1 == 0 || err1 != 0 {
goto childerror
}
_, _, err1 = syscall.RawSyscall(syscall.SYS_PTRACE, uintptr(syscall.PTRACE_TRACEME), 0, 0)
if err1 != 0 {
goto childerror
}
}
// if both seccomp and ptrace is defined, then seccomp filter should have
// traced execve, thus child need parent attached to it first
// actually, this is not effective if pid namespace is unshared
if r.StopBeforeSeccomp || (r.Seccomp != nil && r.Ptrace) {
// Stop to wait for ptrace tracer
_, _, err1 = syscall.RawSyscall(syscall.SYS_KILL, pid, uintptr(syscall.SIGSTOP), 0)
if err1 != 0 {
goto childerror
}
}
// Load seccomp, stop and wait for tracer
if r.Seccomp != nil {
// If execve is seccomp trapped, then tracee stop is necessary
// otherwise execve will fail due to ENOSYS
// Do getpid and kill to send SYS_KILL to self
// need to do before seccomp as these might be traced
// Load seccomp filter
_, _, err1 = syscall.RawSyscall(unix.SYS_SECCOMP, SECCOMP_SET_MODE_FILTER, SECCOMP_FILTER_FLAG_TSYNC, uintptr(unsafe.Pointer(r.Seccomp)))
if err1 != 0 {
goto childerror
}
}
// Before exec, sync with parent through pipe (configured as close_on_exec)
if !r.Ptrace || r.Seccomp == nil {
err2 = 0
r1, _, err1 = syscall.RawSyscall(syscall.SYS_WRITE, uintptr(pipe), uintptr(unsafe.Pointer(&err2)), uintptr(unsafe.Sizeof(err2)))
if r1 == 0 || err1 != 0 {
goto childerror
}
r1, _, err1 = syscall.RawSyscall(syscall.SYS_READ, uintptr(pipe), uintptr(unsafe.Pointer(&err2)), uintptr(unsafe.Sizeof(err2)))
if r1 == 0 || err1 != 0 {
goto childerror
}
}
// Enable ptrace if no seccomp is needed
if r.Ptrace && r.Seccomp == nil {
_, _, err1 = syscall.RawSyscall(syscall.SYS_PTRACE, uintptr(syscall.PTRACE_TRACEME), 0, 0)
if err1 != 0 {
goto childerror
}
}
// at this point, runner is successfully attached for seccomp trap filter
// or execve trapped without seccomp filter
// time to exec
// if execfile fd is specified, call fexecve
if r.ExecFile > 0 {
_, _, err1 = syscall.RawSyscall6(unix.SYS_EXECVEAT, r.ExecFile,
uintptr(unsafe.Pointer(&empty[0])),
uintptr(unsafe.Pointer(&argv[0])),
uintptr(unsafe.Pointer(&env[0])), unix.AT_EMPTY_PATH, 0)
} else {
_, _, err1 = syscall.RawSyscall6(unix.SYS_EXECVEAT, uintptr(_AT_FDCWD),
uintptr(unsafe.Pointer(argv0)),
uintptr(unsafe.Pointer(&argv[0])),
uintptr(unsafe.Pointer(&env[0])), 0, 0)
}
childerror:
// send error code on pipe
syscall.RawSyscall(unix.SYS_WRITE, uintptr(pipe), uintptr(unsafe.Pointer(&err1)), unsafe.Sizeof(err1))
for {
syscall.RawSyscall(syscall.SYS_EXIT, uintptr(err1+err2), 0, 0)
}
// cannot reach this point
}

View File

@ -45,19 +45,3 @@ func syscallStringFromString(str string) (*byte, error) {
}
return nil, nil
}
// preparePivotRoot prepares pivot root parameters
func preparePivotRoot(r string) (*byte, *byte, error) {
if r == "" {
return nil, nil, nil
}
root, err := syscall.BytePtrFromString(r)
if err != nil {
return nil, nil, err
}
oldRoot, err := syscall.BytePtrFromString(OldRoot)
if err != nil {
return nil, nil, err
}
return root, oldRoot, nil
}

View File

@ -1,5 +1,5 @@
// Package forkexec provides interface to run a seccomp filtered, rlimited
// executable and ptraced
// Package forkexec provides interface to run a subprocess with seccomp filter, rlimit and
// containerized or ptraced
package forkexec
import (
@ -9,8 +9,8 @@ import (
"github.com/criyle/go-sandbox/pkg/rlimit"
)
// Runner is the runptraceConfig including the exec path, argv
// and resource limits. It creates tracee for ptrace-based tracer.
// Runner is the configuration including the exec path, argv
// and resource limits. It can creates tracee for ptrace-based tracer.
// It can also create unshared process in another namespace
type Runner struct {
// argv and env for execve syscall for the child process