finished deamon api

This commit is contained in:
criyle 2019-08-08 22:34:34 -07:00
parent cde21dec31
commit 0d3940cb48
5 changed files with 196 additions and 26 deletions

View File

@ -25,6 +25,7 @@ New Features:
5. Use Linux Namespace to isolate file access (elimilate ptrace) 5. Use Linux Namespace to isolate file access (elimilate ptrace)
6. Use Linux Control Groups to limit & acct CPU & memory (elimilate wait4.rusage) 6. Use Linux Control Groups to limit & acct CPU & memory (elimilate wait4.rusage)
7. Container tech with execveat memfd, sethostname, setdomainname 7. Container tech with execveat memfd, sethostname, setdomainname
8. Pre-fork container deamons to run programs inside
Default file access action: Default file access action:
@ -32,7 +33,7 @@ Default file access action:
- check file read: `readlink`, `readlinkat` - check file read: `readlink`, `readlinkat`
- check file write: `unlink`, `unlinkat`, `chmod`, `rename` - check file write: `unlink`, `unlinkat`, `chmod`, `rename`
- check file access: `stat`, `lstat`, `access`, `faccessat` - check file access: `stat`, `lstat`, `access`, `faccessat`
- check file exec: `execve` - check file exec: `execveat`
Packages: Packages:
@ -41,6 +42,7 @@ Packages:
- memfd: read regular file and creates a seaed memfd for its contents - memfd: read regular file and creates a seaed memfd for its contents
- unixsocket: send / recv oob msg from a unix socket - unixsocket: send / recv oob msg from a unix socket
- cgroup: creates cgroup directories and collects resource usage / limits - cgroup: creates cgroup directories and collects resource usage / limits
- deamon: creates pre-forked container to run programs inside
- tracer: ptrace tracer and provides syscall trap filter context - tracer: ptrace tracer and provides syscall trap filter context
- runprogram: wrapper to call forkexec and trecer - runprogram: wrapper to call forkexec and trecer
- rununshared: wrapper to call forkexec and unshared namespaces - rununshared: wrapper to call forkexec and unshared namespaces
@ -52,23 +54,23 @@ Packages:
Executable: Executable:
- run_program: under construction - run_program: safely run program by unshare / ptrace / pre-forked containers
Configuations: Configuations:
- run_program/config.go: all configs toward running specs - run_program/config.go: all configs toward running specs
Benchmarks for unshare: Benchmarks (docker desktop amd64 / native arm64):
- 1ms: fork, unshare pid / user / cgroup - 1ms / 2ms: fork, unshare pid / user / cgroup
- 50ms: unshare ipc / mount - 4ms / 8ms: run inside pre-forked container
- 100ms: unshare pid & user & cgroup & mount & pivot root - 50ms / 25ms: unshare ipc / mount
- 400ms: unshare net - 100ms / 44ms: unshare pid & user & cgroup & mount & pivot root
- 800ms: unshare all - 400ms / 63ms: unshare net
- 880ms: unshare all & pivot root - 800ms / 170ms: unshare all
- 880ms / 170ms: unshare all & pivot root
It seems unshare net or ipc takes time, maybe limits action by seccomp instead. It seems unshare net or ipc takes time, maybe limits action by seccomp instead.
Pre-forked container also saves time for container creation / cleanup.
TODO: TODO:
1. Add ability to pre-fork container deamons

View File

@ -4,6 +4,7 @@ import (
"bytes" "bytes"
"encoding/gob" "encoding/gob"
"fmt" "fmt"
"io"
"os" "os"
"syscall" "syscall"
@ -53,6 +54,19 @@ func handleCmd(s *unixsocket.Socket, cmd *Cmd, msg *unixsocket.Msg) error {
switch cmd.Cmd { switch cmd.Cmd {
case cmdPing: case cmdPing:
return handlePing(s) return handlePing(s)
case cmdCopyIn:
return handleCopyIn(s, cmd, msg)
case cmdOpen:
return handleOpen(s, cmd)
case cmdDelete:
return handleDelete(s, cmd)
case cmdReset:
return handleReset(s)
case cmdExecve: case cmdExecve:
return handleExecve(s, cmd, msg) return handleExecve(s, cmd, msg)
} }
@ -63,6 +77,62 @@ func handlePing(s *unixsocket.Socket) error {
return sendReply(s, &Reply{}, nil) return sendReply(s, &Reply{}, nil)
} }
func handleCopyIn(s *unixsocket.Socket, cmd *Cmd, msg *unixsocket.Msg) error {
if len(msg.Fds) == 0 {
return sendErrorReply(s, "copyin: did not receive fd")
}
inf := os.NewFile(uintptr(msg.Fds[0]), cmd.Path)
if inf == nil {
return sendErrorReply(s, "copyin: newfile failed %v", msg.Fds[0])
}
defer inf.Close()
outf, err := os.OpenFile(cmd.Path, os.O_CREATE|os.O_RDWR|os.O_TRUNC, 666)
if err != nil {
return sendErrorReply(s, "copyin: open write file %v", err)
}
defer outf.Close()
_, err = io.Copy(outf, inf)
if err != nil {
return sendErrorReply(s, "copyin: io.copy %v", err)
}
return sendReply(s, &Reply{}, nil)
}
func handleOpen(s *unixsocket.Socket, cmd *Cmd) error {
outf, err := os.Open(cmd.Path)
if err != nil {
return sendErrorReply(s, "open: %v", err)
}
defer outf.Close()
msg := unixsocket.Msg{
Fds: []int{int(outf.Fd())},
}
return sendReply(s, &Reply{}, &msg)
}
func handleDelete(s *unixsocket.Socket, cmd *Cmd) error {
err := os.Remove(cmd.Path)
if err != nil {
return sendErrorReply(s, "delete: %v", err)
}
return sendReply(s, &Reply{}, nil)
}
func handleReset(s *unixsocket.Socket) error {
err := removeContents("/tmp")
if err != nil {
return sendErrorReply(s, "reset: /tmp %v", err)
}
err = removeContents("/w")
if err != nil {
return sendErrorReply(s, "reset: /w %v", err)
}
return sendReply(s, &Reply{}, nil)
}
func handleExecve(s *unixsocket.Socket, cmd *Cmd, msg *unixsocket.Msg) error { func handleExecve(s *unixsocket.Socket, cmd *Cmd, msg *unixsocket.Msg) error {
var ( var (
files []uintptr files []uintptr
@ -112,9 +182,7 @@ func handleExecve(s *unixsocket.Socket, cmd *Cmd, msg *unixsocket.Msg) error {
} }
pid, err := r.Start() pid, err := r.Start()
if err != nil { if err != nil {
reply := Reply{Error: fmt.Sprintf("execve: %v", err)} return sendErrorReply(s, "execve: %v", err)
sendReply(s, &reply, nil)
return nil
} }
done := make(chan int) done := make(chan int)
@ -141,8 +209,7 @@ loop:
for { for {
_, err = syscall.Wait4(pid, &wstatus, 0, nil) _, err = syscall.Wait4(pid, &wstatus, 0, nil)
if err != nil { if err != nil {
reply := Reply{Error: fmt.Sprintf("execve: wait4 %v", err)} sendErrorReply(s, "execve: wait4 %v", err)
sendReply(s, &reply, nil)
break loop break loop
} }
switch { switch {
@ -202,3 +269,9 @@ func sendReply(s *unixsocket.Socket, reply *Reply, msg *unixsocket.Msg) error {
} }
return nil return nil
} }
// sendErrorReply sends error reply
func sendErrorReply(s *unixsocket.Socket, ft string, v ...interface{}) error {
reply := Reply{Error: fmt.Sprintf(ft, v...)}
return sendReply(s, &reply, nil)
}

View File

@ -9,10 +9,10 @@ Protocol between client and deamon (not thread safe):
- ping (alive check): - ping (alive check):
reply: pong reply: pong
- copyin (copy file into container): - copyin (copy file into container):
send: path, perm, <input fd> send: path, <input fd>
reply: "finished" (after copy finished) / "error" reply: "finished" (after copy finished) / "error"
- open (open file inside container): - open (open file in read-only mode inside container):
send: path, flags, perm send: path
reply: "success", <file fd> / "error" reply: "success", <file fd> / "error"
- delete (unlik file / rmdir dir inside container): - delete (unlik file / rmdir dir inside container):
send: path send: path
@ -41,9 +41,7 @@ import (
// Cmd is the control message send into deamon // Cmd is the control message send into deamon
type Cmd struct { type Cmd struct {
Cmd string // type of the cmd Cmd string // type of the cmd
Path string // path Path string // path (copyin / open)
Perm int // write perm / open perm
Flags int // open flags
Argv []string // execve argv Argv []string // execve argv
Envv []string // execve envv Envv []string // execve envv
RLmits []rlimit.RLimit // execve posix rlimit RLmits []rlimit.RLimit // execve posix rlimit

View File

@ -4,6 +4,7 @@ import (
"bytes" "bytes"
"encoding/gob" "encoding/gob"
"fmt" "fmt"
"os"
"github.com/criyle/go-judger/unixsocket" "github.com/criyle/go-judger/unixsocket"
) )
@ -18,12 +19,82 @@ func (m *Master) Ping() error {
return fmt.Errorf("ping: %v", err) return fmt.Errorf("ping: %v", err)
} }
// receive no error // receive no error
reply, _, err := m.recvReply() return m.recvAckReply("ping")
}
// CopyIn copies file to container
func (m *Master) CopyIn(f *os.File, p string) error {
// send copyin
cmd := Cmd{
Cmd: cmdCopyIn,
Path: p,
}
msg := unixsocket.Msg{
Fds: []int{int(f.Fd())},
}
if err := m.sendCmd(&cmd, &msg); err != nil {
return fmt.Errorf("copyin: %v", err)
}
return m.recvAckReply("copyin")
}
// Open open file in container
func (m *Master) Open(p string) (*os.File, error) {
// send copyin
cmd := Cmd{
Cmd: cmdOpen,
Path: p,
}
if err := m.sendCmd(&cmd, nil); err != nil {
return nil, fmt.Errorf("open: %v", err)
}
reply, msg, err := m.recvReply()
if err != nil { if err != nil {
return fmt.Errorf("ping: %v", err) return nil, fmt.Errorf("open: %v", err)
} }
if reply.Error != "" { if reply.Error != "" {
return fmt.Errorf("ping: reply error(%v)", reply.Error) return nil, fmt.Errorf("open: %v", reply.Error)
}
if len(msg.Fds) == 0 {
return nil, fmt.Errorf("open: did not receive fd")
}
f := os.NewFile(uintptr(msg.Fds[0]), p)
if f == nil {
return nil, fmt.Errorf("open: failed %v", msg.Fds[0])
}
return f, nil
}
// Delete remove file from container
func (m *Master) Delete(p string) error {
cmd := Cmd{
Cmd: cmdDelete,
Path: p,
}
if err := m.sendCmd(&cmd, nil); err != nil {
return fmt.Errorf("delete: %v", err)
}
return m.recvAckReply("delete")
}
// Reset remove all from /tmp and /w
func (m *Master) Reset(p string) error {
cmd := Cmd{
Cmd: cmdReset,
}
if err := m.sendCmd(&cmd, nil); err != nil {
return fmt.Errorf("reset: %v", err)
}
return m.recvAckReply("reset")
}
func (m *Master) recvAckReply(name string) error {
reply, _, err := m.recvReply()
if err != nil {
return fmt.Errorf(name+": %v", err)
}
if reply.Error != "" {
return fmt.Errorf(name+": %v", reply.Error)
} }
return nil return nil
} }

View File

@ -1,6 +1,10 @@
package deamon package deamon
import "syscall" import (
"os"
"path"
"syscall"
)
func intSliceToUintptr(s []int) []uintptr { func intSliceToUintptr(s []int) []uintptr {
var r []uintptr var r []uintptr
@ -29,3 +33,25 @@ func closeFds(s []int) {
syscall.Close(f) syscall.Close(f)
} }
} }
// removeContents delete content of a directory
func removeContents(dir string) error {
d, err := os.Open(dir)
if err != nil {
return err
}
defer d.Close()
names, err := d.Readdirnames(-1)
if err != nil {
return err
}
for _, name := range names {
err = os.RemoveAll(path.Join(dir, name))
if err != nil {
return err
}
}
return nil
}