finished container deamon POC & fixed fd dup

This commit is contained in:
criyle 2019-08-05 19:55:03 -07:00
parent 3247555505
commit 25732d9455
10 changed files with 369 additions and 5 deletions

View File

@ -8,6 +8,7 @@ import (
"time"
"github.com/criyle/go-judger/cgroup"
"github.com/criyle/go-judger/deamon"
"github.com/criyle/go-judger/memfd"
"github.com/criyle/go-judger/runconfig"
"github.com/criyle/go-judger/runprogram"
@ -35,6 +36,7 @@ func main() {
var (
addReadable, addWritable, addRawReadable, addRawWritable arrayFlags
allowProc, unsafe, showDetails, namespace, useCGroup, memfile bool
useDeamon bool
pType, result string
timeLimit, realTimeLimit, memoryLimit, outputLimit, stackLimit uint64
inputFileName, outputFileName, errorFileName, workPath string
@ -45,6 +47,8 @@ func main() {
execFile uintptr
)
deamon.ContainerInit()
flag.Usage = printUsage
flag.Uint64Var(&timeLimit, "tl", 1, "Set time limit (in second)")
flag.Uint64Var(&realTimeLimit, "rtl", 0, "Set real time limit (in second)")
@ -67,6 +71,7 @@ func main() {
flag.BoolVar(&namespace, "ns", false, "Use namespace to restrict file accesses")
flag.BoolVar(&useCGroup, "cgroup", false, "Use cgroup to colloct resource usage")
flag.BoolVar(&memfile, "memfd", false, "Use memfd as exec file")
flag.BoolVar(&useDeamon, "deamon", false, "Use deamon container to execute file")
flag.Parse()
args := flag.Args()
@ -114,6 +119,22 @@ func main() {
return nil
}
if useDeamon {
root, err := ioutil.TempDir("", "dm")
if err != nil {
panic("cannot make temp root for deamon namespace")
}
m, err := deamon.New(root)
if err != nil {
panic(fmt.Sprintln("failed to new master", err))
}
err = m.Ping()
if err != nil {
panic(fmt.Sprintln("failed to ping deamon", err))
}
m.Destroy()
}
if memfile {
fin, err := os.Open(args[0])
if err != nil {

5
deamon/consts.go Normal file
View File

@ -0,0 +1,5 @@
package deamon
const (
cmdPing = "ping"
)

79
deamon/container_init.go Normal file
View File

@ -0,0 +1,79 @@
package deamon
import (
"bytes"
"encoding/gob"
"fmt"
"os"
"github.com/criyle/go-judger/unixsocket"
)
// ContainerInit is called for container init process
// it will check if pid == 1, otherwise it is noop
// ContainerInit will do infinite loop on socket commands,
// and exits when at socket close
func ContainerInit() (err error) {
// noop if self is not container init process
if os.Getpid() != 1 {
return nil
}
// exit process (with whole container) upon exit this function
defer func() {
if err != nil {
fmt.Fprintf(os.Stderr, "container_exit: %v", err)
os.Exit(1)
} else {
fmt.Fprintf(os.Stderr, "container_exit")
os.Exit(0)
}
}()
// new_master shared the socket at fd 3 (marked close_exec)
soc, err := unixsocket.NewSocket(3)
if err != nil {
return fmt.Errorf("container_init: faile to new socket(%v)", err)
}
var (
buffer = make([]byte, bufferSize)
cmd Cmd
)
for {
n, msg, err := soc.RecvMsg(buffer)
if err != nil {
return fmt.Errorf("loop: failed RecvMsg(%v)", err)
}
dec := gob.NewDecoder(bytes.NewReader(buffer[:n]))
if err := dec.Decode(&cmd); err != nil {
return fmt.Errorf("loop: failed to decode(%v)", err)
}
if err := handleCmd(soc, &cmd, msg); err != nil {
return fmt.Errorf("loop: failed to execute cmd(%v)", err)
}
}
}
func handleCmd(s *unixsocket.Socket, cmd *Cmd, msg *unixsocket.Msg) error {
switch cmd.Cmd {
case cmdPing:
return handlePing(s)
}
return nil
}
func handlePing(s *unixsocket.Socket) error {
return sendReply(s, &Reply{})
}
func sendReply(s *unixsocket.Socket, reply *Reply) error {
var buffer bytes.Buffer
enc := gob.NewEncoder(&buffer)
if err := enc.Encode(reply); err != nil {
return err
}
if err := s.SendMsg(buffer.Bytes(), nil); err != nil {
return err
}
return nil
}

View File

@ -6,6 +6,8 @@ package deamon
/*
Protocol between client and deamon (not thread safe):
- ping (alive check):
reply: pong
- copyin (copy file into container):
send: path, perm, <input fd>
reply: "finished" (after copy finished) / "error"

73
deamon/default.go Normal file
View File

@ -0,0 +1,73 @@
package deamon
import (
"os"
"github.com/criyle/go-judger/types/mount"
"golang.org/x/sys/unix"
)
const (
bind = unix.MS_BIND | unix.MS_NOSUID | unix.MS_PRIVATE
roBind = bind | unix.MS_RDONLY
mFlag = unix.MS_NOSUID | unix.MS_NOATIME | unix.MS_NODEV
)
// default parameters. I was tend to reuse the configs but it is hard since there are some
// cross device symblics
var (
DefaultPath = "PATH=/usr/local/bin:/usr/bin:/bin"
// rootfs created by bind mounting
DefaultMounts = []*mount.Mount{
{
Source: "/usr",
Target: "usr",
Flags: roBind,
},
{
Source: "/lib",
Target: "lib",
Flags: roBind,
},
{
Source: "/lib64",
Target: "lib64",
Flags: roBind,
},
{
Source: "/bin",
Target: "bin",
Flags: roBind,
},
// work dir at /w
{
Source: "tmpfs",
Target: "w",
FsType: "tmpfs",
Flags: mFlag,
},
// tmpfs at /tmp
{
Source: "tmpfs",
Target: "tmp",
FsType: "tmpfs",
Flags: mFlag,
},
}
)
func init() {
// check if bind mount source exists, e.g. /lib64 does not exists on arm
mounts := make([]*mount.Mount, 0, len(DefaultMounts))
for _, m := range DefaultMounts {
if m.Source != "tmpfs" {
if _, err := os.Stat(m.Source); !os.IsNotExist(err) {
mounts = append(mounts, m)
}
} else {
mounts = append(mounts, m)
}
}
DefaultMounts = mounts
}

93
deamon/master.go Normal file
View File

@ -0,0 +1,93 @@
package deamon
import (
"fmt"
"os"
"github.com/criyle/go-judger/forkexec"
"github.com/criyle/go-judger/memfd"
"github.com/criyle/go-judger/unixsocket"
"golang.org/x/sys/unix"
)
// Master manages single pre-forked container
type Master struct {
pid int // underlying container init pid
socket *unixsocket.Socket // master - container communication
}
// New creates new master with underlying container
func New(root string) (*Master, error) {
// dummy stdin / stdout / stderr
fnull, err := os.OpenFile(os.DevNull, os.O_RDWR, 0666)
if err != nil {
return nil, fmt.Errorf("deamon: failed to open devNull(%v)", err)
}
defer fnull.Close()
// prepare self memfd
self, err := os.Open("/proc/self/exe")
if err != nil {
return nil, fmt.Errorf("deamon: failed to open /proc/self/exe(%v)", err)
}
defer self.Close()
execFile, err := memfd.DupToMemfd("deamon", self)
if err != nil {
return nil, fmt.Errorf("deamon: failed to create memfd(%v)", err)
}
defer execFile.Close()
// prepare socket
ins, outs, err := unixsocket.NewSocketPair()
if err != nil {
return nil, fmt.Errorf("deamon: failed to create socket(%v)", err)
}
outf, err := outs.Conn.File()
if err != nil {
ins.Conn.Close()
outs.Conn.Close()
return nil, fmt.Errorf("deamon: failed to dup file outs(%v)", err)
}
defer outf.Close()
if err = ins.SetPassCred(1); err != nil {
ins.Conn.Close()
outs.Conn.Close()
return nil, fmt.Errorf("deamon: failed to set pass_cred ins(%v)", err)
}
if err = outs.SetPassCred(1); err != nil {
ins.Conn.Close()
outs.Conn.Close()
return nil, fmt.Errorf("deamon: failed to set pass_cred outs(%v)", err)
}
r := &forkexec.Runner{
Args: []string{os.Args[0]},
Env: []string{DefaultPath},
ExecFile: execFile.Fd(),
Files: []uintptr{fnull.Fd(), fnull.Fd(), fnull.Fd(), uintptr(outf.Fd())},
WorkDir: "/w",
UnshareFlags: forkexec.UnshareFlags,
Mounts: DefaultMounts,
HostName: "deamon",
DomainName: "deamon",
PivotRoot: root,
}
pid, err := r.Start()
if err != nil {
ins.Conn.Close()
outs.Conn.Close()
return nil, fmt.Errorf("deamon: failed to execve(%v)", err)
}
outs.Conn.Close()
return &Master{pid, ins}, nil
}
// Destroy kill the deamon process (with container)
func (m *Master) Destroy() error {
var wstatus unix.WaitStatus
unix.Kill(m.pid, unix.SIGKILL)
_, err := unix.Wait4(m.pid, &wstatus, 0, nil)
return err
}

41
deamon/master_cmd.go Normal file
View File

@ -0,0 +1,41 @@
package deamon
import (
"bytes"
"encoding/gob"
"fmt"
)
// Ping send ping message to container
func (m *Master) Ping() error {
var (
wbuff bytes.Buffer
reply Reply
)
rbuff := GetBuffer()
defer PutBuffer(rbuff)
// send ping
enc := gob.NewEncoder(&wbuff)
cmd := Cmd{
Cmd: cmdPing,
}
if err := enc.Encode(cmd); err != nil {
return fmt.Errorf("ping: failed to encode(%v)", err)
}
if err := m.socket.SendMsg(wbuff.Bytes(), nil); err != nil {
return fmt.Errorf("ping: failed to sendMsg(%v)", err)
}
// receive no error
n, _, err := m.socket.RecvMsg(rbuff)
if err != nil {
return fmt.Errorf("ping: failed to recvMsg(%v)", err)
}
dec := gob.NewDecoder(bytes.NewReader(rbuff[:n]))
if err := dec.Decode(&reply); err != nil {
return fmt.Errorf("ping: failed to decode(%v)", err)
}
if reply.Error != "" {
return fmt.Errorf("ping: reply error(%v)", reply.Error)
}
return nil
}

24
deamon/pool.go Normal file
View File

@ -0,0 +1,24 @@
package deamon
import (
"sync"
)
// 16k buffsize
const bufferSize = 16384
var bufferPool = sync.Pool{
New: func() interface{} {
return make([]byte, bufferSize)
},
}
// GetBuffer get buffer from pool
func GetBuffer() []byte {
return bufferPool.Get().([]byte)
}
// PutBuffer return buffer to the pool
func PutBuffer(x []byte) {
bufferPool.Put(x)
}

View File

@ -86,6 +86,12 @@ func (r *Runner) Start() (int, error) {
// similar to exec_linux, avoid side effect by shuffling around
fd, nextfd := prepareFds(r.Files)
pipe := p2[1]
if nextfd <= pipe {
nextfd = pipe + 1
}
if nextfd <= int(r.ExecFile) {
nextfd = int(r.ExecFile) + 1
}
// Acquire the fork lock so that no other threads
// create new fds that are not yet close-on-exec
@ -177,7 +183,7 @@ func (r *Runner) Start() (int, error) {
// Pass 1 & pass 2 assigns fds for child process
// Pass 1: fd[i] < i => nextfd
if pipe < nextfd {
_, _, err1 = syscall.RawSyscall(syscall.SYS_DUP3, uintptr(pipe), uintptr(nextfd), syscall.FD_CLOEXEC)
_, _, err1 = syscall.RawSyscall(syscall.SYS_DUP3, uintptr(pipe), uintptr(nextfd), syscall.O_CLOEXEC)
if err1 != 0 {
goto childerror
}
@ -185,7 +191,7 @@ func (r *Runner) Start() (int, error) {
nextfd++
}
if r.ExecFile > 0 && int(r.ExecFile) < nextfd {
_, _, err1 = syscall.RawSyscall(syscall.SYS_DUP3, r.ExecFile, uintptr(nextfd), syscall.FD_CLOEXEC)
_, _, err1 = syscall.RawSyscall(syscall.SYS_DUP3, r.ExecFile, uintptr(nextfd), syscall.O_CLOEXEC)
if err1 != 0 {
goto childerror
}
@ -194,7 +200,7 @@ func (r *Runner) Start() (int, error) {
}
for i := 0; i < len(fd); i++ {
if fd[i] >= 0 && fd[i] < int(i) {
_, _, err1 = syscall.RawSyscall(syscall.SYS_DUP3, uintptr(fd[i]), uintptr(nextfd), syscall.FD_CLOEXEC)
_, _, err1 = syscall.RawSyscall(syscall.SYS_DUP3, uintptr(fd[i]), uintptr(nextfd), syscall.O_CLOEXEC)
if err1 != 0 {
goto childerror
}
@ -203,7 +209,6 @@ func (r *Runner) Start() (int, error) {
nextfd++
}
}
// Pass 2: fd[i] => i
for i := 0; i < len(fd); i++ {
if fd[i] == -1 {
@ -448,7 +453,7 @@ func (r *Runner) Start() (int, error) {
}
childerror:
syscall.RawSyscall(syscall.SYS_EXIT, uintptr(err1), 0, 0)
syscall.RawSyscall(syscall.SYS_EXIT, uintptr(err1+err2), 0, 0)
// cannot reach this point
panic("cannot reach")
}

View File

@ -53,6 +53,27 @@ func NewSocket(fd int) (*Socket, error) {
return &Socket{unixConn}, nil
}
// NewSocketPair creates conneted unix socketpair using SOCK_SEQPACKET
func NewSocketPair() (*Socket, *Socket, error) {
fd, err := syscall.Socketpair(syscall.AF_LOCAL, syscall.SOCK_SEQPACKET|syscall.SOCK_CLOEXEC, 0)
if err != nil {
return nil, nil, fmt.Errorf("NewSocketPair: failed to call socketpair(%v)", err)
}
ins, err := NewSocket(fd[0])
if err != nil {
syscall.Close(fd[0])
syscall.Close(fd[1])
return nil, nil, fmt.Errorf("NewSocketPair: failed to call NewSocket ins(%v)", err)
}
outs, err := NewSocket(fd[1])
if err != nil {
ins.Conn.Close()
syscall.Close(fd[1])
return nil, nil, fmt.Errorf("NewSocketPair: failed to call NewSocket outs(%v)", err)
}
return ins, outs, nil
}
// SetPassCred set sockopt for pass cred for unix socket
func (s *Socket) SetPassCred(option int) error {
sysconn, err := s.Conn.SyscallConn()