From 4452ebd8035ffd4a779f5c3b1afa5cba162546c0 Mon Sep 17 00:00:00 2001 From: criyle Date: Mon, 12 Aug 2019 01:36:15 -0700 Subject: [PATCH] updated readme & enforced sync --- README.md | 135 +++++++++++++++++++++++---------------- deamon/container_init.go | 11 +--- deamon/master_cmd.go | 6 +- forkexec/fork.go | 70 +++++++++++++++----- go.mod | 2 +- go.sum | 6 +- 6 files changed, 143 insertions(+), 87 deletions(-) diff --git a/README.md b/README.md index 436bcbf..7987d35 100644 --- a/README.md +++ b/README.md @@ -1,76 +1,103 @@ -## go-judger +# go-judger -Under developing. +Original goal is to reimplement [uoj-judger/run_program](https://github.com/vfleaking/uoj) in GO language using [libseccomp](https://github.com/seccomp/libseccomp-golang). As technology grows, implements new technologies such as Linux namespace & cgroup. -Goal is to reimplement [uoj-judger/run_program](https://github.com/vfleaking/uoj) in GO language using [libseccomp](https://github.com/seccomp/libseccomp-golang). +## Install -Install: +- install go compiler: `apt install golang-go` +- install libseccomp-dev: `apt install libseccomp-dev` +- install: `go install github.com/criyle/go-judger/...` -- install go compiler: `apt install golang-go` -- install libseccomp-dev: `apt install libseccomp-dev` -- install: `go install github.com/criyle/go-judger/...` +## Technologies -Features (same as uoj-judger/run_program): +### libseccomp + ptrace (improved UOJ sandbox) -1. Restricted computing resource: Time & Memory (Stack) & Output -2. Restricted syscall access (by libseccomp & ptrace) -3. Restricted file access (read & write & access & exec) +1. Restricted computing resource by POSIX rlimit: Time & Memory (Stack) & Output +2. Restricted syscall access (by libseccomp & ptrace) +3. Restricted file access (read & write & access & exec). Evaluated by UOJ FileSet -New Features: +Improvements: -1. Percise resource limits (s -> ms, mb -> kb) -2. More architectures (arm32, arm64, x86) -3. Allow multiple traced programs in different threads -4. Allow pipes as input / output files -5. Use Linux Namespace to isolate file access (elimilate ptrace) -6. Use Linux Control Groups to limit & acct CPU & memory (elimilate wait4.rusage) -7. Container tech with execveat memfd, sethostname, setdomainname -8. Pre-fork container deamons to run programs inside +1. Percise resource limits (s -> ms, mb -> kb) +2. More architectures (arm32, arm64, x86) +3. Allow multiple traced programs in different threads +4. Allow pipes as input / output files -Default file access action: +Default file access syscall check: -- check file read / write: `open`, `openat` -- check file read: `readlink`, `readlinkat` -- check file write: `unlink`, `unlinkat`, `chmod`, `rename` -- check file access: `stat`, `lstat`, `access`, `faccessat` -- check file exec: `execveat` +- check file read / write: `open`, `openat` +- check file read: `readlink`, `readlinkat` +- check file write: `unlink`, `unlinkat`, `chmod`, `rename` +- check file access: `stat`, `lstat`, `access`, `faccessat` +- check file exec: `execveat` -Packages: +### linux namespace + cgroup -- seccomp: provides utility function that wrappers libseccomp -- forkexec: fork-exec provides mount, unshare, ptrace, seccomp, capset before exec -- memfd: read regular file and creates a seaed memfd for its contents -- unixsocket: send / recv oob msg from a unix socket -- cgroup: creates cgroup directories and collects resource usage / limits -- deamon: creates pre-forked container to run programs inside -- tracer: ptrace tracer and provides syscall trap filter context -- runprogram: wrapper to call forkexec and trecer -- rununshared: wrapper to call forkexec and unshared namespaces -- runconfig: defines arch & language specified trace condition for seccomp and ptrace -- types: general runtime specs - - mount: provides utility function that wrappers mount syscall - - rlimit: provides utility function that defines rlimit syscall - - specs: provides general res / result data structures +1. Unshare & bind mount rootfs based on hostfs (elimilated ptrace) +2. Use Linux Control Groups to limit & acct CPU & memory (elimilate wait4.rusage) +3. Container tech with execveat memfd, sethostname, setdomainname -Executable: +### pre-forked container -- run_program: safely run program by unshare / ptrace / pre-forked containers +1. Pre-fork container deamons to run programs inside +2. Unix socket to pass fd inside / outside -Configuations: +## Packages -- run_program/config.go: all configs toward running specs +- seccomp: provides utility function that wrappers libseccomp +- forkexec: fork-exec provides mount, unshare, ptrace, seccomp, capset before exec +- memfd: read regular file and creates a seaed memfd for its contents +- unixsocket: send / recv oob msg from a unix socket +- cgroup: creates cgroup directories and collects resource usage / limits +- deamon: creates pre-forked container to run programs inside +- tracer: ptrace tracer and provides syscall trap filter context +- runprogram: wrapper to call forkexec and trecer +- rununshared: wrapper to call forkexec and unshared namespaces +- runconfig: defines arch & language specified trace condition for seccomp and ptrace +- types: general runtime specs + - mount: provides utility function that wrappers mount syscall + - rlimit: provides utility function that defines rlimit syscall + - specs: provides general res / result data structures -Benchmarks (docker desktop amd64 / native arm64): +## Executable -- 1ms / 2ms: fork, unshare pid / user / cgroup -- 4ms / 8ms: run inside pre-forked container -- 50ms / 25ms: unshare ipc / mount -- 100ms / 44ms: unshare pid & user & cgroup & mount & pivot root -- 400ms / 63ms: unshare net -- 800ms / 170ms: unshare all -- 880ms / 170ms: unshare all & pivot root +- run_program: safely run program by unshare / ptrace / pre-forked containers + +## Configuations + +- run_program/config.go: all configs toward running specs + +## Benchmarks (docker desktop amd64 / native arm64) + +- 1ms / 2ms: fork, unshare pid / user / cgroup +- 4ms / 8ms: run inside pre-forked container +- 50ms / 25ms: unshare ipc / mount +- 100ms / 44ms: unshare pid & user & cgroup & mount & pivot root +- 400ms / 63ms: unshare net +- 800ms / 170ms: unshare all +- 880ms / 170ms: unshare all & pivot root It seems unshare net or ipc takes time, maybe limits action by seccomp instead. Pre-forked container also saves time for container creation / cleanup. -TODO: +```bash +$ go test -bench . -benchtime 10s +goos: linux +goarch: amd64 +pkg: github.com/criyle/go-judger/forkexec +BenchmarkSimpleFork-4 10000 1106064 ns/op +BenchmarkUnsharePid-4 10000 1367824 ns/op +BenchmarkUnshareUser-4 10000 1311523 ns/op +BenchmarkUnshareUts-4 10000 1140427 ns/op +BenchmarkUnshareCgroup-4 10000 1112713 ns/op +BenchmarkUnshareIpc-4 300 58730786 ns/op +BenchmarkUnshareMount-4 300 55540758 ns/op +BenchmarkUnshareNet-4 100 396957720 ns/op +BenchmarkFastUnshareMountPivot-4 100 114364585 ns/op +BenchmarkUnshareAll-4 100 851014031 ns/op +BenchmarkUnshareMountPivot-4 20 901204445 ns/op +PASS +ok github.com/criyle/go-judger/forkexec 262.112s +``` + +## TODO diff --git a/deamon/container_init.go b/deamon/container_init.go index 1c007b3..6b3e988 100644 --- a/deamon/container_init.go +++ b/deamon/container_init.go @@ -20,10 +20,7 @@ import ( func ContainerInit() (err error) { // noop if self is not container init process // Notice: docker init is also 1, additional check for args[1] == init - if os.Getpid() != 1 { - return nil - } - if len(os.Args) != 2 || os.Args[1] != initArg { + if os.Getpid() != 1 || len(os.Args) != 2 || os.Args[1] != initArg { return nil } @@ -255,8 +252,7 @@ func recvCmd(s *unixsocket.Socket) (*Cmd, *unixsocket.Msg, error) { if err != nil { return nil, nil, fmt.Errorf("failed RecvMsg(%v)", err) } - dec := gob.NewDecoder(bytes.NewReader(buffer[:n])) - if err := dec.Decode(&cmd); err != nil { + if err := gob.NewDecoder(bytes.NewReader(buffer[:n])).Decode(&cmd); err != nil { return nil, nil, fmt.Errorf("failed to decode(%v)", err) } return &cmd, msg, nil @@ -264,8 +260,7 @@ func recvCmd(s *unixsocket.Socket) (*Cmd, *unixsocket.Msg, error) { func sendReply(s *unixsocket.Socket, reply *Reply, msg *unixsocket.Msg) error { var buffer bytes.Buffer - enc := gob.NewEncoder(&buffer) - if err := enc.Encode(reply); err != nil { + if err := gob.NewEncoder(&buffer).Encode(reply); err != nil { return err } if err := s.SendMsg(buffer.Bytes(), msg); err != nil { diff --git a/deamon/master_cmd.go b/deamon/master_cmd.go index 30d8e58..ea88628 100644 --- a/deamon/master_cmd.go +++ b/deamon/master_cmd.go @@ -107,8 +107,7 @@ func (m *Master) recvReply() (*Reply, *unixsocket.Msg, error) { if err != nil { return nil, nil, fmt.Errorf("failed to RecvMsg(%v)", err) } - dec := gob.NewDecoder(bytes.NewReader(buff[:n])) - if err := dec.Decode(&reply); err != nil { + if err := gob.NewDecoder(bytes.NewReader(buff[:n])).Decode(&reply); err != nil { return nil, nil, fmt.Errorf("failed to decode(%v)", err) } return &reply, msg, nil @@ -116,8 +115,7 @@ func (m *Master) recvReply() (*Reply, *unixsocket.Msg, error) { func (m *Master) sendCmd(cmd *Cmd, msg *unixsocket.Msg) error { var buff bytes.Buffer - enc := gob.NewEncoder(&buff) - if err := enc.Encode(cmd); err != nil { + if err := gob.NewEncoder(&buff).Encode(cmd); err != nil { return fmt.Errorf("failed to encode(%v)", err) } if err := m.socket.SendMsg(buff.Bytes(), msg); err != nil { diff --git a/forkexec/fork.go b/forkexec/fork.go index bdb0519..7709678 100644 --- a/forkexec/fork.go +++ b/forkexec/fork.go @@ -86,12 +86,6 @@ func (r *Runner) Start() (int, error) { // similar to exec_linux, avoid side effect by shuffling around fd, nextfd := prepareFds(r.Files) pipe := p2[1] - if nextfd <= pipe { - nextfd = pipe + 1 - } - if nextfd <= int(r.ExecFile) { - nextfd = int(r.ExecFile) + 1 - } // Acquire the fork lock so that no other threads // create new fds that are not yet close-on-exec @@ -109,11 +103,15 @@ func (r *Runner) Start() (int, error) { afterFork() syscall.ForkLock.Unlock() + // clone syscall failed + if err1 != 0 { + return int(pid), syscall.Errno(err1) + } + // synchronize with child for uid / gid map if unshareUser { unix.Close(p[0]) - err = writeIDMaps(int(pid)) - if err != nil { + if err = writeIDMaps(int(pid)); err != nil { err2 = err.(syscall.Errno) } syscall.RawSyscall(syscall.SYS_WRITE, uintptr(p[1]), uintptr(unsafe.Pointer(&err2)), uintptr(unsafe.Sizeof(err2))) @@ -122,24 +120,46 @@ func (r *Runner) Start() (int, error) { // sync with child unix.Close(p2[1]) - _, _, err1 = syscall.RawSyscall(syscall.SYS_READ, uintptr(p2[0]), uintptr(unsafe.Pointer(&err2)), uintptr(unsafe.Sizeof(err2))) - if err1 != 0 { + + r1, _, err1 = syscall.RawSyscall(syscall.SYS_READ, uintptr(p2[0]), uintptr(unsafe.Pointer(&err2)), uintptr(unsafe.Sizeof(err2))) + // child returned error code + if r1 != unsafe.Sizeof(err2) || err2 != 0 || err1 != 0 { unix.Close(p2[0]) - return int(pid), syscall.Errno(err1) + if r1 == unsafe.Sizeof(err2) { + err = syscall.Errno(err2) + } + if err == nil { + err = syscall.EPIPE + } + handleChildFailed(pid) + return 0, err } if r.SyncFunc != nil { err = r.SyncFunc(int(pid)) } + // if syncfunc return error, then fail child immediately if err == nil { err1 = 0 syscall.RawSyscall(syscall.SYS_WRITE, uintptr(p2[0]), uintptr(unsafe.Pointer(&err1)), uintptr(unsafe.Sizeof((err1)))) + } else { + unix.Close(p2[0]) + handleChildFailed(pid) + return 0, err } + // if read anything mean child failed after sync (close_on_exec so it should not block) + r1, _, err1 = syscall.RawSyscall(syscall.SYS_READ, uintptr(p2[0]), uintptr(unsafe.Pointer(&err2)), uintptr(unsafe.Sizeof(err2))) unix.Close(p2[0]) - - if err1 != 0 { - return int(pid), syscall.Errno(err1) + if r1 != 0 || err1 != 0 { + if r1 == unsafe.Sizeof(err2) { + err = syscall.Errno(err2) + } + if err == nil { + err = syscall.EPIPE + } + handleChildFailed(pid) + return 0, err } return int(pid), nil } @@ -199,6 +219,10 @@ func (r *Runner) Start() (int, error) { nextfd++ } for i := 0; i < len(fd); i++ { + // Avoid fd rewrite + for nextfd == i || (r.ExecFile > 0 && nextfd == int(r.ExecFile)) { + nextfd++ + } if fd[i] >= 0 && fd[i] < int(i) { _, _, err1 = syscall.RawSyscall(syscall.SYS_DUP3, uintptr(fd[i]), uintptr(nextfd), syscall.O_CLOEXEC) if err1 != 0 { @@ -453,7 +477,21 @@ func (r *Runner) Start() (int, error) { } childerror: - syscall.RawSyscall(syscall.SYS_EXIT, uintptr(err1+err2), 0, 0) + // send error code on pipe + syscall.RawSyscall(unix.SYS_WRITE, uintptr(pipe), uintptr(unsafe.Pointer(&err1)), unsafe.Sizeof(err1)) + for { + syscall.RawSyscall(syscall.SYS_EXIT, uintptr(err1+err2), 0, 0) + } // cannot reach this point - panic("cannot reach") +} + +func handleChildFailed(pid uintptr) { + var wstatus syscall.WaitStatus + // make sure not blocked + unix.Kill(int(pid), syscall.SIGKILL) + // child failed; wait for it to exit, to make sure the zombies don't accumulate + _, err3 := syscall.Wait4(int(pid), &wstatus, 0, nil) + for err3 == syscall.EINTR { + _, err3 = syscall.Wait4(int(pid), &wstatus, 0, nil) + } } diff --git a/go.mod b/go.mod index e4777e5..70189ab 100644 --- a/go.mod +++ b/go.mod @@ -4,5 +4,5 @@ go 1.12 require ( github.com/seccomp/libseccomp-golang v0.9.1 - golang.org/x/sys v0.0.0-20190712062909-fae7ac547cb7 + golang.org/x/sys v0.0.0-20190804053845-51ab0e2deafa ) diff --git a/go.sum b/go.sum index 9d5689a..2702148 100644 --- a/go.sum +++ b/go.sum @@ -1,6 +1,4 @@ github.com/seccomp/libseccomp-golang v0.9.1 h1:NJjM5DNFOs0s3kYE1WUOr6G8V97sdt46rlXTMfXGWBo= github.com/seccomp/libseccomp-golang v0.9.1/go.mod h1:GbW5+tmTXfcxTToHLXlScSlAvWlF4P2Ca7zGrPiEpWo= -golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb h1:fgwFCsaw9buMuxNd6+DQfAuSFqbNiQZpcgJQAgJsK6k= -golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190712062909-fae7ac547cb7 h1:LepdCS8Gf/MVejFIt8lsiexZATdoGVyp5bcyS+rYoUI= -golang.org/x/sys v0.0.0-20190712062909-fae7ac547cb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190804053845-51ab0e2deafa h1:KIDDMLT1O0Nr7TSxp8xM5tJcdn8tgyAONntO829og1M= +golang.org/x/sys v0.0.0-20190804053845-51ab0e2deafa/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=