finished deamon api

This commit is contained in:
criyle 2019-08-08 22:34:34 -07:00
parent cde21dec31
commit 0d3940cb48
5 changed files with 196 additions and 26 deletions

View File

@ -25,6 +25,7 @@ New Features:
5. Use Linux Namespace to isolate file access (elimilate ptrace)
6. Use Linux Control Groups to limit & acct CPU & memory (elimilate wait4.rusage)
7. Container tech with execveat memfd, sethostname, setdomainname
8. Pre-fork container deamons to run programs inside
Default file access action:
@ -32,7 +33,7 @@ Default file access action:
- check file read: `readlink`, `readlinkat`
- check file write: `unlink`, `unlinkat`, `chmod`, `rename`
- check file access: `stat`, `lstat`, `access`, `faccessat`
- check file exec: `execve`
- check file exec: `execveat`
Packages:
@ -41,6 +42,7 @@ Packages:
- memfd: read regular file and creates a seaed memfd for its contents
- unixsocket: send / recv oob msg from a unix socket
- cgroup: creates cgroup directories and collects resource usage / limits
- deamon: creates pre-forked container to run programs inside
- tracer: ptrace tracer and provides syscall trap filter context
- runprogram: wrapper to call forkexec and trecer
- rununshared: wrapper to call forkexec and unshared namespaces
@ -52,23 +54,23 @@ Packages:
Executable:
- run_program: under construction
- run_program: safely run program by unshare / ptrace / pre-forked containers
Configuations:
- run_program/config.go: all configs toward running specs
Benchmarks for unshare:
Benchmarks (docker desktop amd64 / native arm64):
- 1ms: fork, unshare pid / user / cgroup
- 50ms: unshare ipc / mount
- 100ms: unshare pid & user & cgroup & mount & pivot root
- 400ms: unshare net
- 800ms: unshare all
- 880ms: unshare all & pivot root
- 1ms / 2ms: fork, unshare pid / user / cgroup
- 4ms / 8ms: run inside pre-forked container
- 50ms / 25ms: unshare ipc / mount
- 100ms / 44ms: unshare pid & user & cgroup & mount & pivot root
- 400ms / 63ms: unshare net
- 800ms / 170ms: unshare all
- 880ms / 170ms: unshare all & pivot root
It seems unshare net or ipc takes time, maybe limits action by seccomp instead.
Pre-forked container also saves time for container creation / cleanup.
TODO:
1. Add ability to pre-fork container deamons

View File

@ -4,6 +4,7 @@ import (
"bytes"
"encoding/gob"
"fmt"
"io"
"os"
"syscall"
@ -53,6 +54,19 @@ func handleCmd(s *unixsocket.Socket, cmd *Cmd, msg *unixsocket.Msg) error {
switch cmd.Cmd {
case cmdPing:
return handlePing(s)
case cmdCopyIn:
return handleCopyIn(s, cmd, msg)
case cmdOpen:
return handleOpen(s, cmd)
case cmdDelete:
return handleDelete(s, cmd)
case cmdReset:
return handleReset(s)
case cmdExecve:
return handleExecve(s, cmd, msg)
}
@ -63,6 +77,62 @@ func handlePing(s *unixsocket.Socket) error {
return sendReply(s, &Reply{}, nil)
}
func handleCopyIn(s *unixsocket.Socket, cmd *Cmd, msg *unixsocket.Msg) error {
if len(msg.Fds) == 0 {
return sendErrorReply(s, "copyin: did not receive fd")
}
inf := os.NewFile(uintptr(msg.Fds[0]), cmd.Path)
if inf == nil {
return sendErrorReply(s, "copyin: newfile failed %v", msg.Fds[0])
}
defer inf.Close()
outf, err := os.OpenFile(cmd.Path, os.O_CREATE|os.O_RDWR|os.O_TRUNC, 666)
if err != nil {
return sendErrorReply(s, "copyin: open write file %v", err)
}
defer outf.Close()
_, err = io.Copy(outf, inf)
if err != nil {
return sendErrorReply(s, "copyin: io.copy %v", err)
}
return sendReply(s, &Reply{}, nil)
}
func handleOpen(s *unixsocket.Socket, cmd *Cmd) error {
outf, err := os.Open(cmd.Path)
if err != nil {
return sendErrorReply(s, "open: %v", err)
}
defer outf.Close()
msg := unixsocket.Msg{
Fds: []int{int(outf.Fd())},
}
return sendReply(s, &Reply{}, &msg)
}
func handleDelete(s *unixsocket.Socket, cmd *Cmd) error {
err := os.Remove(cmd.Path)
if err != nil {
return sendErrorReply(s, "delete: %v", err)
}
return sendReply(s, &Reply{}, nil)
}
func handleReset(s *unixsocket.Socket) error {
err := removeContents("/tmp")
if err != nil {
return sendErrorReply(s, "reset: /tmp %v", err)
}
err = removeContents("/w")
if err != nil {
return sendErrorReply(s, "reset: /w %v", err)
}
return sendReply(s, &Reply{}, nil)
}
func handleExecve(s *unixsocket.Socket, cmd *Cmd, msg *unixsocket.Msg) error {
var (
files []uintptr
@ -112,9 +182,7 @@ func handleExecve(s *unixsocket.Socket, cmd *Cmd, msg *unixsocket.Msg) error {
}
pid, err := r.Start()
if err != nil {
reply := Reply{Error: fmt.Sprintf("execve: %v", err)}
sendReply(s, &reply, nil)
return nil
return sendErrorReply(s, "execve: %v", err)
}
done := make(chan int)
@ -141,8 +209,7 @@ loop:
for {
_, err = syscall.Wait4(pid, &wstatus, 0, nil)
if err != nil {
reply := Reply{Error: fmt.Sprintf("execve: wait4 %v", err)}
sendReply(s, &reply, nil)
sendErrorReply(s, "execve: wait4 %v", err)
break loop
}
switch {
@ -202,3 +269,9 @@ func sendReply(s *unixsocket.Socket, reply *Reply, msg *unixsocket.Msg) error {
}
return nil
}
// sendErrorReply sends error reply
func sendErrorReply(s *unixsocket.Socket, ft string, v ...interface{}) error {
reply := Reply{Error: fmt.Sprintf(ft, v...)}
return sendReply(s, &reply, nil)
}

View File

@ -9,10 +9,10 @@ Protocol between client and deamon (not thread safe):
- ping (alive check):
reply: pong
- copyin (copy file into container):
send: path, perm, <input fd>
send: path, <input fd>
reply: "finished" (after copy finished) / "error"
- open (open file inside container):
send: path, flags, perm
- open (open file in read-only mode inside container):
send: path
reply: "success", <file fd> / "error"
- delete (unlik file / rmdir dir inside container):
send: path
@ -41,9 +41,7 @@ import (
// Cmd is the control message send into deamon
type Cmd struct {
Cmd string // type of the cmd
Path string // path
Perm int // write perm / open perm
Flags int // open flags
Path string // path (copyin / open)
Argv []string // execve argv
Envv []string // execve envv
RLmits []rlimit.RLimit // execve posix rlimit

View File

@ -4,6 +4,7 @@ import (
"bytes"
"encoding/gob"
"fmt"
"os"
"github.com/criyle/go-judger/unixsocket"
)
@ -18,12 +19,82 @@ func (m *Master) Ping() error {
return fmt.Errorf("ping: %v", err)
}
// receive no error
reply, _, err := m.recvReply()
return m.recvAckReply("ping")
}
// CopyIn copies file to container
func (m *Master) CopyIn(f *os.File, p string) error {
// send copyin
cmd := Cmd{
Cmd: cmdCopyIn,
Path: p,
}
msg := unixsocket.Msg{
Fds: []int{int(f.Fd())},
}
if err := m.sendCmd(&cmd, &msg); err != nil {
return fmt.Errorf("copyin: %v", err)
}
return m.recvAckReply("copyin")
}
// Open open file in container
func (m *Master) Open(p string) (*os.File, error) {
// send copyin
cmd := Cmd{
Cmd: cmdOpen,
Path: p,
}
if err := m.sendCmd(&cmd, nil); err != nil {
return nil, fmt.Errorf("open: %v", err)
}
reply, msg, err := m.recvReply()
if err != nil {
return fmt.Errorf("ping: %v", err)
return nil, fmt.Errorf("open: %v", err)
}
if reply.Error != "" {
return fmt.Errorf("ping: reply error(%v)", reply.Error)
return nil, fmt.Errorf("open: %v", reply.Error)
}
if len(msg.Fds) == 0 {
return nil, fmt.Errorf("open: did not receive fd")
}
f := os.NewFile(uintptr(msg.Fds[0]), p)
if f == nil {
return nil, fmt.Errorf("open: failed %v", msg.Fds[0])
}
return f, nil
}
// Delete remove file from container
func (m *Master) Delete(p string) error {
cmd := Cmd{
Cmd: cmdDelete,
Path: p,
}
if err := m.sendCmd(&cmd, nil); err != nil {
return fmt.Errorf("delete: %v", err)
}
return m.recvAckReply("delete")
}
// Reset remove all from /tmp and /w
func (m *Master) Reset(p string) error {
cmd := Cmd{
Cmd: cmdReset,
}
if err := m.sendCmd(&cmd, nil); err != nil {
return fmt.Errorf("reset: %v", err)
}
return m.recvAckReply("reset")
}
func (m *Master) recvAckReply(name string) error {
reply, _, err := m.recvReply()
if err != nil {
return fmt.Errorf(name+": %v", err)
}
if reply.Error != "" {
return fmt.Errorf(name+": %v", reply.Error)
}
return nil
}

View File

@ -1,6 +1,10 @@
package deamon
import "syscall"
import (
"os"
"path"
"syscall"
)
func intSliceToUintptr(s []int) []uintptr {
var r []uintptr
@ -29,3 +33,25 @@ func closeFds(s []int) {
syscall.Close(f)
}
}
// removeContents delete content of a directory
func removeContents(dir string) error {
d, err := os.Open(dir)
if err != nil {
return err
}
defer d.Close()
names, err := d.Readdirnames(-1)
if err != nil {
return err
}
for _, name := range names {
err = os.RemoveAll(path.Join(dir, name))
if err != nil {
return err
}
}
return nil
}