diff --git a/README.md b/README.md index b532170..436bcbf 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,7 @@ New Features: 5. Use Linux Namespace to isolate file access (elimilate ptrace) 6. Use Linux Control Groups to limit & acct CPU & memory (elimilate wait4.rusage) 7. Container tech with execveat memfd, sethostname, setdomainname +8. Pre-fork container deamons to run programs inside Default file access action: @@ -32,7 +33,7 @@ Default file access action: - check file read: `readlink`, `readlinkat` - check file write: `unlink`, `unlinkat`, `chmod`, `rename` - check file access: `stat`, `lstat`, `access`, `faccessat` -- check file exec: `execve` +- check file exec: `execveat` Packages: @@ -41,6 +42,7 @@ Packages: - memfd: read regular file and creates a seaed memfd for its contents - unixsocket: send / recv oob msg from a unix socket - cgroup: creates cgroup directories and collects resource usage / limits +- deamon: creates pre-forked container to run programs inside - tracer: ptrace tracer and provides syscall trap filter context - runprogram: wrapper to call forkexec and trecer - rununshared: wrapper to call forkexec and unshared namespaces @@ -52,23 +54,23 @@ Packages: Executable: -- run_program: under construction +- run_program: safely run program by unshare / ptrace / pre-forked containers Configuations: - run_program/config.go: all configs toward running specs -Benchmarks for unshare: +Benchmarks (docker desktop amd64 / native arm64): -- 1ms: fork, unshare pid / user / cgroup -- 50ms: unshare ipc / mount -- 100ms: unshare pid & user & cgroup & mount & pivot root -- 400ms: unshare net -- 800ms: unshare all -- 880ms: unshare all & pivot root +- 1ms / 2ms: fork, unshare pid / user / cgroup +- 4ms / 8ms: run inside pre-forked container +- 50ms / 25ms: unshare ipc / mount +- 100ms / 44ms: unshare pid & user & cgroup & mount & pivot root +- 400ms / 63ms: unshare net +- 800ms / 170ms: unshare all +- 880ms / 170ms: unshare all & pivot root It seems unshare net or ipc takes time, maybe limits action by seccomp instead. +Pre-forked container also saves time for container creation / cleanup. TODO: - -1. Add ability to pre-fork container deamons diff --git a/deamon/container_init.go b/deamon/container_init.go index ee6e8fb..defccc5 100644 --- a/deamon/container_init.go +++ b/deamon/container_init.go @@ -4,6 +4,7 @@ import ( "bytes" "encoding/gob" "fmt" + "io" "os" "syscall" @@ -53,6 +54,19 @@ func handleCmd(s *unixsocket.Socket, cmd *Cmd, msg *unixsocket.Msg) error { switch cmd.Cmd { case cmdPing: return handlePing(s) + + case cmdCopyIn: + return handleCopyIn(s, cmd, msg) + + case cmdOpen: + return handleOpen(s, cmd) + + case cmdDelete: + return handleDelete(s, cmd) + + case cmdReset: + return handleReset(s) + case cmdExecve: return handleExecve(s, cmd, msg) } @@ -63,6 +77,62 @@ func handlePing(s *unixsocket.Socket) error { return sendReply(s, &Reply{}, nil) } +func handleCopyIn(s *unixsocket.Socket, cmd *Cmd, msg *unixsocket.Msg) error { + if len(msg.Fds) == 0 { + return sendErrorReply(s, "copyin: did not receive fd") + } + inf := os.NewFile(uintptr(msg.Fds[0]), cmd.Path) + if inf == nil { + return sendErrorReply(s, "copyin: newfile failed %v", msg.Fds[0]) + } + defer inf.Close() + + outf, err := os.OpenFile(cmd.Path, os.O_CREATE|os.O_RDWR|os.O_TRUNC, 666) + if err != nil { + return sendErrorReply(s, "copyin: open write file %v", err) + } + defer outf.Close() + + _, err = io.Copy(outf, inf) + if err != nil { + return sendErrorReply(s, "copyin: io.copy %v", err) + } + return sendReply(s, &Reply{}, nil) +} + +func handleOpen(s *unixsocket.Socket, cmd *Cmd) error { + outf, err := os.Open(cmd.Path) + if err != nil { + return sendErrorReply(s, "open: %v", err) + } + defer outf.Close() + + msg := unixsocket.Msg{ + Fds: []int{int(outf.Fd())}, + } + return sendReply(s, &Reply{}, &msg) +} + +func handleDelete(s *unixsocket.Socket, cmd *Cmd) error { + err := os.Remove(cmd.Path) + if err != nil { + return sendErrorReply(s, "delete: %v", err) + } + return sendReply(s, &Reply{}, nil) +} + +func handleReset(s *unixsocket.Socket) error { + err := removeContents("/tmp") + if err != nil { + return sendErrorReply(s, "reset: /tmp %v", err) + } + err = removeContents("/w") + if err != nil { + return sendErrorReply(s, "reset: /w %v", err) + } + return sendReply(s, &Reply{}, nil) +} + func handleExecve(s *unixsocket.Socket, cmd *Cmd, msg *unixsocket.Msg) error { var ( files []uintptr @@ -112,9 +182,7 @@ func handleExecve(s *unixsocket.Socket, cmd *Cmd, msg *unixsocket.Msg) error { } pid, err := r.Start() if err != nil { - reply := Reply{Error: fmt.Sprintf("execve: %v", err)} - sendReply(s, &reply, nil) - return nil + return sendErrorReply(s, "execve: %v", err) } done := make(chan int) @@ -141,8 +209,7 @@ loop: for { _, err = syscall.Wait4(pid, &wstatus, 0, nil) if err != nil { - reply := Reply{Error: fmt.Sprintf("execve: wait4 %v", err)} - sendReply(s, &reply, nil) + sendErrorReply(s, "execve: wait4 %v", err) break loop } switch { @@ -202,3 +269,9 @@ func sendReply(s *unixsocket.Socket, reply *Reply, msg *unixsocket.Msg) error { } return nil } + +// sendErrorReply sends error reply +func sendErrorReply(s *unixsocket.Socket, ft string, v ...interface{}) error { + reply := Reply{Error: fmt.Sprintf(ft, v...)} + return sendReply(s, &reply, nil) +} diff --git a/deamon/deamon.go b/deamon/deamon.go index a8ee59c..d14364f 100644 --- a/deamon/deamon.go +++ b/deamon/deamon.go @@ -9,10 +9,10 @@ Protocol between client and deamon (not thread safe): - ping (alive check): reply: pong - copyin (copy file into container): - send: path, perm, + send: path, reply: "finished" (after copy finished) / "error" -- open (open file inside container): - send: path, flags, perm +- open (open file in read-only mode inside container): + send: path reply: "success", / "error" - delete (unlik file / rmdir dir inside container): send: path @@ -41,9 +41,7 @@ import ( // Cmd is the control message send into deamon type Cmd struct { Cmd string // type of the cmd - Path string // path - Perm int // write perm / open perm - Flags int // open flags + Path string // path (copyin / open) Argv []string // execve argv Envv []string // execve envv RLmits []rlimit.RLimit // execve posix rlimit diff --git a/deamon/master_cmd.go b/deamon/master_cmd.go index 80e4f35..9528b90 100644 --- a/deamon/master_cmd.go +++ b/deamon/master_cmd.go @@ -4,6 +4,7 @@ import ( "bytes" "encoding/gob" "fmt" + "os" "github.com/criyle/go-judger/unixsocket" ) @@ -18,12 +19,82 @@ func (m *Master) Ping() error { return fmt.Errorf("ping: %v", err) } // receive no error - reply, _, err := m.recvReply() + return m.recvAckReply("ping") +} + +// CopyIn copies file to container +func (m *Master) CopyIn(f *os.File, p string) error { + // send copyin + cmd := Cmd{ + Cmd: cmdCopyIn, + Path: p, + } + msg := unixsocket.Msg{ + Fds: []int{int(f.Fd())}, + } + if err := m.sendCmd(&cmd, &msg); err != nil { + return fmt.Errorf("copyin: %v", err) + } + return m.recvAckReply("copyin") +} + +// Open open file in container +func (m *Master) Open(p string) (*os.File, error) { + // send copyin + cmd := Cmd{ + Cmd: cmdOpen, + Path: p, + } + if err := m.sendCmd(&cmd, nil); err != nil { + return nil, fmt.Errorf("open: %v", err) + } + reply, msg, err := m.recvReply() if err != nil { - return fmt.Errorf("ping: %v", err) + return nil, fmt.Errorf("open: %v", err) } if reply.Error != "" { - return fmt.Errorf("ping: reply error(%v)", reply.Error) + return nil, fmt.Errorf("open: %v", reply.Error) + } + if len(msg.Fds) == 0 { + return nil, fmt.Errorf("open: did not receive fd") + } + f := os.NewFile(uintptr(msg.Fds[0]), p) + if f == nil { + return nil, fmt.Errorf("open: failed %v", msg.Fds[0]) + } + return f, nil +} + +// Delete remove file from container +func (m *Master) Delete(p string) error { + cmd := Cmd{ + Cmd: cmdDelete, + Path: p, + } + if err := m.sendCmd(&cmd, nil); err != nil { + return fmt.Errorf("delete: %v", err) + } + return m.recvAckReply("delete") +} + +// Reset remove all from /tmp and /w +func (m *Master) Reset(p string) error { + cmd := Cmd{ + Cmd: cmdReset, + } + if err := m.sendCmd(&cmd, nil); err != nil { + return fmt.Errorf("reset: %v", err) + } + return m.recvAckReply("reset") +} + +func (m *Master) recvAckReply(name string) error { + reply, _, err := m.recvReply() + if err != nil { + return fmt.Errorf(name+": %v", err) + } + if reply.Error != "" { + return fmt.Errorf(name+": %v", reply.Error) } return nil } diff --git a/deamon/utils.go b/deamon/utils.go index b0fd254..134d30a 100644 --- a/deamon/utils.go +++ b/deamon/utils.go @@ -1,6 +1,10 @@ package deamon -import "syscall" +import ( + "os" + "path" + "syscall" +) func intSliceToUintptr(s []int) []uintptr { var r []uintptr @@ -29,3 +33,25 @@ func closeFds(s []int) { syscall.Close(f) } } + +// removeContents delete content of a directory +func removeContents(dir string) error { + d, err := os.Open(dir) + if err != nil { + return err + } + defer d.Close() + + names, err := d.Readdirnames(-1) + if err != nil { + return err + } + + for _, name := range names { + err = os.RemoveAll(path.Join(dir, name)) + if err != nil { + return err + } + } + return nil +}