finished container deamon POC & fixed fd dup

This commit is contained in:
criyle 2019-08-05 19:55:03 -07:00
parent 3247555505
commit 25732d9455
10 changed files with 369 additions and 5 deletions

View File

@ -8,6 +8,7 @@ import (
"time" "time"
"github.com/criyle/go-judger/cgroup" "github.com/criyle/go-judger/cgroup"
"github.com/criyle/go-judger/deamon"
"github.com/criyle/go-judger/memfd" "github.com/criyle/go-judger/memfd"
"github.com/criyle/go-judger/runconfig" "github.com/criyle/go-judger/runconfig"
"github.com/criyle/go-judger/runprogram" "github.com/criyle/go-judger/runprogram"
@ -35,6 +36,7 @@ func main() {
var ( var (
addReadable, addWritable, addRawReadable, addRawWritable arrayFlags addReadable, addWritable, addRawReadable, addRawWritable arrayFlags
allowProc, unsafe, showDetails, namespace, useCGroup, memfile bool allowProc, unsafe, showDetails, namespace, useCGroup, memfile bool
useDeamon bool
pType, result string pType, result string
timeLimit, realTimeLimit, memoryLimit, outputLimit, stackLimit uint64 timeLimit, realTimeLimit, memoryLimit, outputLimit, stackLimit uint64
inputFileName, outputFileName, errorFileName, workPath string inputFileName, outputFileName, errorFileName, workPath string
@ -45,6 +47,8 @@ func main() {
execFile uintptr execFile uintptr
) )
deamon.ContainerInit()
flag.Usage = printUsage flag.Usage = printUsage
flag.Uint64Var(&timeLimit, "tl", 1, "Set time limit (in second)") flag.Uint64Var(&timeLimit, "tl", 1, "Set time limit (in second)")
flag.Uint64Var(&realTimeLimit, "rtl", 0, "Set real time limit (in second)") flag.Uint64Var(&realTimeLimit, "rtl", 0, "Set real time limit (in second)")
@ -67,6 +71,7 @@ func main() {
flag.BoolVar(&namespace, "ns", false, "Use namespace to restrict file accesses") flag.BoolVar(&namespace, "ns", false, "Use namespace to restrict file accesses")
flag.BoolVar(&useCGroup, "cgroup", false, "Use cgroup to colloct resource usage") flag.BoolVar(&useCGroup, "cgroup", false, "Use cgroup to colloct resource usage")
flag.BoolVar(&memfile, "memfd", false, "Use memfd as exec file") flag.BoolVar(&memfile, "memfd", false, "Use memfd as exec file")
flag.BoolVar(&useDeamon, "deamon", false, "Use deamon container to execute file")
flag.Parse() flag.Parse()
args := flag.Args() args := flag.Args()
@ -114,6 +119,22 @@ func main() {
return nil return nil
} }
if useDeamon {
root, err := ioutil.TempDir("", "dm")
if err != nil {
panic("cannot make temp root for deamon namespace")
}
m, err := deamon.New(root)
if err != nil {
panic(fmt.Sprintln("failed to new master", err))
}
err = m.Ping()
if err != nil {
panic(fmt.Sprintln("failed to ping deamon", err))
}
m.Destroy()
}
if memfile { if memfile {
fin, err := os.Open(args[0]) fin, err := os.Open(args[0])
if err != nil { if err != nil {

5
deamon/consts.go Normal file
View File

@ -0,0 +1,5 @@
package deamon
const (
cmdPing = "ping"
)

79
deamon/container_init.go Normal file
View File

@ -0,0 +1,79 @@
package deamon
import (
"bytes"
"encoding/gob"
"fmt"
"os"
"github.com/criyle/go-judger/unixsocket"
)
// ContainerInit is called for container init process
// it will check if pid == 1, otherwise it is noop
// ContainerInit will do infinite loop on socket commands,
// and exits when at socket close
func ContainerInit() (err error) {
// noop if self is not container init process
if os.Getpid() != 1 {
return nil
}
// exit process (with whole container) upon exit this function
defer func() {
if err != nil {
fmt.Fprintf(os.Stderr, "container_exit: %v", err)
os.Exit(1)
} else {
fmt.Fprintf(os.Stderr, "container_exit")
os.Exit(0)
}
}()
// new_master shared the socket at fd 3 (marked close_exec)
soc, err := unixsocket.NewSocket(3)
if err != nil {
return fmt.Errorf("container_init: faile to new socket(%v)", err)
}
var (
buffer = make([]byte, bufferSize)
cmd Cmd
)
for {
n, msg, err := soc.RecvMsg(buffer)
if err != nil {
return fmt.Errorf("loop: failed RecvMsg(%v)", err)
}
dec := gob.NewDecoder(bytes.NewReader(buffer[:n]))
if err := dec.Decode(&cmd); err != nil {
return fmt.Errorf("loop: failed to decode(%v)", err)
}
if err := handleCmd(soc, &cmd, msg); err != nil {
return fmt.Errorf("loop: failed to execute cmd(%v)", err)
}
}
}
func handleCmd(s *unixsocket.Socket, cmd *Cmd, msg *unixsocket.Msg) error {
switch cmd.Cmd {
case cmdPing:
return handlePing(s)
}
return nil
}
func handlePing(s *unixsocket.Socket) error {
return sendReply(s, &Reply{})
}
func sendReply(s *unixsocket.Socket, reply *Reply) error {
var buffer bytes.Buffer
enc := gob.NewEncoder(&buffer)
if err := enc.Encode(reply); err != nil {
return err
}
if err := s.SendMsg(buffer.Bytes(), nil); err != nil {
return err
}
return nil
}

View File

@ -6,6 +6,8 @@ package deamon
/* /*
Protocol between client and deamon (not thread safe): Protocol between client and deamon (not thread safe):
- ping (alive check):
reply: pong
- copyin (copy file into container): - copyin (copy file into container):
send: path, perm, <input fd> send: path, perm, <input fd>
reply: "finished" (after copy finished) / "error" reply: "finished" (after copy finished) / "error"

73
deamon/default.go Normal file
View File

@ -0,0 +1,73 @@
package deamon
import (
"os"
"github.com/criyle/go-judger/types/mount"
"golang.org/x/sys/unix"
)
const (
bind = unix.MS_BIND | unix.MS_NOSUID | unix.MS_PRIVATE
roBind = bind | unix.MS_RDONLY
mFlag = unix.MS_NOSUID | unix.MS_NOATIME | unix.MS_NODEV
)
// default parameters. I was tend to reuse the configs but it is hard since there are some
// cross device symblics
var (
DefaultPath = "PATH=/usr/local/bin:/usr/bin:/bin"
// rootfs created by bind mounting
DefaultMounts = []*mount.Mount{
{
Source: "/usr",
Target: "usr",
Flags: roBind,
},
{
Source: "/lib",
Target: "lib",
Flags: roBind,
},
{
Source: "/lib64",
Target: "lib64",
Flags: roBind,
},
{
Source: "/bin",
Target: "bin",
Flags: roBind,
},
// work dir at /w
{
Source: "tmpfs",
Target: "w",
FsType: "tmpfs",
Flags: mFlag,
},
// tmpfs at /tmp
{
Source: "tmpfs",
Target: "tmp",
FsType: "tmpfs",
Flags: mFlag,
},
}
)
func init() {
// check if bind mount source exists, e.g. /lib64 does not exists on arm
mounts := make([]*mount.Mount, 0, len(DefaultMounts))
for _, m := range DefaultMounts {
if m.Source != "tmpfs" {
if _, err := os.Stat(m.Source); !os.IsNotExist(err) {
mounts = append(mounts, m)
}
} else {
mounts = append(mounts, m)
}
}
DefaultMounts = mounts
}

93
deamon/master.go Normal file
View File

@ -0,0 +1,93 @@
package deamon
import (
"fmt"
"os"
"github.com/criyle/go-judger/forkexec"
"github.com/criyle/go-judger/memfd"
"github.com/criyle/go-judger/unixsocket"
"golang.org/x/sys/unix"
)
// Master manages single pre-forked container
type Master struct {
pid int // underlying container init pid
socket *unixsocket.Socket // master - container communication
}
// New creates new master with underlying container
func New(root string) (*Master, error) {
// dummy stdin / stdout / stderr
fnull, err := os.OpenFile(os.DevNull, os.O_RDWR, 0666)
if err != nil {
return nil, fmt.Errorf("deamon: failed to open devNull(%v)", err)
}
defer fnull.Close()
// prepare self memfd
self, err := os.Open("/proc/self/exe")
if err != nil {
return nil, fmt.Errorf("deamon: failed to open /proc/self/exe(%v)", err)
}
defer self.Close()
execFile, err := memfd.DupToMemfd("deamon", self)
if err != nil {
return nil, fmt.Errorf("deamon: failed to create memfd(%v)", err)
}
defer execFile.Close()
// prepare socket
ins, outs, err := unixsocket.NewSocketPair()
if err != nil {
return nil, fmt.Errorf("deamon: failed to create socket(%v)", err)
}
outf, err := outs.Conn.File()
if err != nil {
ins.Conn.Close()
outs.Conn.Close()
return nil, fmt.Errorf("deamon: failed to dup file outs(%v)", err)
}
defer outf.Close()
if err = ins.SetPassCred(1); err != nil {
ins.Conn.Close()
outs.Conn.Close()
return nil, fmt.Errorf("deamon: failed to set pass_cred ins(%v)", err)
}
if err = outs.SetPassCred(1); err != nil {
ins.Conn.Close()
outs.Conn.Close()
return nil, fmt.Errorf("deamon: failed to set pass_cred outs(%v)", err)
}
r := &forkexec.Runner{
Args: []string{os.Args[0]},
Env: []string{DefaultPath},
ExecFile: execFile.Fd(),
Files: []uintptr{fnull.Fd(), fnull.Fd(), fnull.Fd(), uintptr(outf.Fd())},
WorkDir: "/w",
UnshareFlags: forkexec.UnshareFlags,
Mounts: DefaultMounts,
HostName: "deamon",
DomainName: "deamon",
PivotRoot: root,
}
pid, err := r.Start()
if err != nil {
ins.Conn.Close()
outs.Conn.Close()
return nil, fmt.Errorf("deamon: failed to execve(%v)", err)
}
outs.Conn.Close()
return &Master{pid, ins}, nil
}
// Destroy kill the deamon process (with container)
func (m *Master) Destroy() error {
var wstatus unix.WaitStatus
unix.Kill(m.pid, unix.SIGKILL)
_, err := unix.Wait4(m.pid, &wstatus, 0, nil)
return err
}

41
deamon/master_cmd.go Normal file
View File

@ -0,0 +1,41 @@
package deamon
import (
"bytes"
"encoding/gob"
"fmt"
)
// Ping send ping message to container
func (m *Master) Ping() error {
var (
wbuff bytes.Buffer
reply Reply
)
rbuff := GetBuffer()
defer PutBuffer(rbuff)
// send ping
enc := gob.NewEncoder(&wbuff)
cmd := Cmd{
Cmd: cmdPing,
}
if err := enc.Encode(cmd); err != nil {
return fmt.Errorf("ping: failed to encode(%v)", err)
}
if err := m.socket.SendMsg(wbuff.Bytes(), nil); err != nil {
return fmt.Errorf("ping: failed to sendMsg(%v)", err)
}
// receive no error
n, _, err := m.socket.RecvMsg(rbuff)
if err != nil {
return fmt.Errorf("ping: failed to recvMsg(%v)", err)
}
dec := gob.NewDecoder(bytes.NewReader(rbuff[:n]))
if err := dec.Decode(&reply); err != nil {
return fmt.Errorf("ping: failed to decode(%v)", err)
}
if reply.Error != "" {
return fmt.Errorf("ping: reply error(%v)", reply.Error)
}
return nil
}

24
deamon/pool.go Normal file
View File

@ -0,0 +1,24 @@
package deamon
import (
"sync"
)
// 16k buffsize
const bufferSize = 16384
var bufferPool = sync.Pool{
New: func() interface{} {
return make([]byte, bufferSize)
},
}
// GetBuffer get buffer from pool
func GetBuffer() []byte {
return bufferPool.Get().([]byte)
}
// PutBuffer return buffer to the pool
func PutBuffer(x []byte) {
bufferPool.Put(x)
}

View File

@ -86,6 +86,12 @@ func (r *Runner) Start() (int, error) {
// similar to exec_linux, avoid side effect by shuffling around // similar to exec_linux, avoid side effect by shuffling around
fd, nextfd := prepareFds(r.Files) fd, nextfd := prepareFds(r.Files)
pipe := p2[1] pipe := p2[1]
if nextfd <= pipe {
nextfd = pipe + 1
}
if nextfd <= int(r.ExecFile) {
nextfd = int(r.ExecFile) + 1
}
// Acquire the fork lock so that no other threads // Acquire the fork lock so that no other threads
// create new fds that are not yet close-on-exec // create new fds that are not yet close-on-exec
@ -177,7 +183,7 @@ func (r *Runner) Start() (int, error) {
// Pass 1 & pass 2 assigns fds for child process // Pass 1 & pass 2 assigns fds for child process
// Pass 1: fd[i] < i => nextfd // Pass 1: fd[i] < i => nextfd
if pipe < nextfd { if pipe < nextfd {
_, _, err1 = syscall.RawSyscall(syscall.SYS_DUP3, uintptr(pipe), uintptr(nextfd), syscall.FD_CLOEXEC) _, _, err1 = syscall.RawSyscall(syscall.SYS_DUP3, uintptr(pipe), uintptr(nextfd), syscall.O_CLOEXEC)
if err1 != 0 { if err1 != 0 {
goto childerror goto childerror
} }
@ -185,7 +191,7 @@ func (r *Runner) Start() (int, error) {
nextfd++ nextfd++
} }
if r.ExecFile > 0 && int(r.ExecFile) < nextfd { if r.ExecFile > 0 && int(r.ExecFile) < nextfd {
_, _, err1 = syscall.RawSyscall(syscall.SYS_DUP3, r.ExecFile, uintptr(nextfd), syscall.FD_CLOEXEC) _, _, err1 = syscall.RawSyscall(syscall.SYS_DUP3, r.ExecFile, uintptr(nextfd), syscall.O_CLOEXEC)
if err1 != 0 { if err1 != 0 {
goto childerror goto childerror
} }
@ -194,7 +200,7 @@ func (r *Runner) Start() (int, error) {
} }
for i := 0; i < len(fd); i++ { for i := 0; i < len(fd); i++ {
if fd[i] >= 0 && fd[i] < int(i) { if fd[i] >= 0 && fd[i] < int(i) {
_, _, err1 = syscall.RawSyscall(syscall.SYS_DUP3, uintptr(fd[i]), uintptr(nextfd), syscall.FD_CLOEXEC) _, _, err1 = syscall.RawSyscall(syscall.SYS_DUP3, uintptr(fd[i]), uintptr(nextfd), syscall.O_CLOEXEC)
if err1 != 0 { if err1 != 0 {
goto childerror goto childerror
} }
@ -203,7 +209,6 @@ func (r *Runner) Start() (int, error) {
nextfd++ nextfd++
} }
} }
// Pass 2: fd[i] => i // Pass 2: fd[i] => i
for i := 0; i < len(fd); i++ { for i := 0; i < len(fd); i++ {
if fd[i] == -1 { if fd[i] == -1 {
@ -448,7 +453,7 @@ func (r *Runner) Start() (int, error) {
} }
childerror: childerror:
syscall.RawSyscall(syscall.SYS_EXIT, uintptr(err1), 0, 0) syscall.RawSyscall(syscall.SYS_EXIT, uintptr(err1+err2), 0, 0)
// cannot reach this point // cannot reach this point
panic("cannot reach") panic("cannot reach")
} }

View File

@ -53,6 +53,27 @@ func NewSocket(fd int) (*Socket, error) {
return &Socket{unixConn}, nil return &Socket{unixConn}, nil
} }
// NewSocketPair creates conneted unix socketpair using SOCK_SEQPACKET
func NewSocketPair() (*Socket, *Socket, error) {
fd, err := syscall.Socketpair(syscall.AF_LOCAL, syscall.SOCK_SEQPACKET|syscall.SOCK_CLOEXEC, 0)
if err != nil {
return nil, nil, fmt.Errorf("NewSocketPair: failed to call socketpair(%v)", err)
}
ins, err := NewSocket(fd[0])
if err != nil {
syscall.Close(fd[0])
syscall.Close(fd[1])
return nil, nil, fmt.Errorf("NewSocketPair: failed to call NewSocket ins(%v)", err)
}
outs, err := NewSocket(fd[1])
if err != nil {
ins.Conn.Close()
syscall.Close(fd[1])
return nil, nil, fmt.Errorf("NewSocketPair: failed to call NewSocket outs(%v)", err)
}
return ins, outs, nil
}
// SetPassCred set sockopt for pass cred for unix socket // SetPassCred set sockopt for pass cred for unix socket
func (s *Socket) SetPassCred(option int) error { func (s *Socket) SetPassCred(option int) error {
sysconn, err := s.Conn.SyscallConn() sysconn, err := s.Conn.SyscallConn()