mirror of
https://github.com/criyle/go-sandbox.git
synced 2025-11-04 14:49:53 +08:00
finished container deamon POC & fixed fd dup
This commit is contained in:
parent
3247555505
commit
25732d9455
@ -8,6 +8,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/criyle/go-judger/cgroup"
|
||||
"github.com/criyle/go-judger/deamon"
|
||||
"github.com/criyle/go-judger/memfd"
|
||||
"github.com/criyle/go-judger/runconfig"
|
||||
"github.com/criyle/go-judger/runprogram"
|
||||
@ -35,6 +36,7 @@ func main() {
|
||||
var (
|
||||
addReadable, addWritable, addRawReadable, addRawWritable arrayFlags
|
||||
allowProc, unsafe, showDetails, namespace, useCGroup, memfile bool
|
||||
useDeamon bool
|
||||
pType, result string
|
||||
timeLimit, realTimeLimit, memoryLimit, outputLimit, stackLimit uint64
|
||||
inputFileName, outputFileName, errorFileName, workPath string
|
||||
@ -45,6 +47,8 @@ func main() {
|
||||
execFile uintptr
|
||||
)
|
||||
|
||||
deamon.ContainerInit()
|
||||
|
||||
flag.Usage = printUsage
|
||||
flag.Uint64Var(&timeLimit, "tl", 1, "Set time limit (in second)")
|
||||
flag.Uint64Var(&realTimeLimit, "rtl", 0, "Set real time limit (in second)")
|
||||
@ -67,6 +71,7 @@ func main() {
|
||||
flag.BoolVar(&namespace, "ns", false, "Use namespace to restrict file accesses")
|
||||
flag.BoolVar(&useCGroup, "cgroup", false, "Use cgroup to colloct resource usage")
|
||||
flag.BoolVar(&memfile, "memfd", false, "Use memfd as exec file")
|
||||
flag.BoolVar(&useDeamon, "deamon", false, "Use deamon container to execute file")
|
||||
flag.Parse()
|
||||
|
||||
args := flag.Args()
|
||||
@ -114,6 +119,22 @@ func main() {
|
||||
return nil
|
||||
}
|
||||
|
||||
if useDeamon {
|
||||
root, err := ioutil.TempDir("", "dm")
|
||||
if err != nil {
|
||||
panic("cannot make temp root for deamon namespace")
|
||||
}
|
||||
m, err := deamon.New(root)
|
||||
if err != nil {
|
||||
panic(fmt.Sprintln("failed to new master", err))
|
||||
}
|
||||
err = m.Ping()
|
||||
if err != nil {
|
||||
panic(fmt.Sprintln("failed to ping deamon", err))
|
||||
}
|
||||
m.Destroy()
|
||||
}
|
||||
|
||||
if memfile {
|
||||
fin, err := os.Open(args[0])
|
||||
if err != nil {
|
||||
|
||||
5
deamon/consts.go
Normal file
5
deamon/consts.go
Normal file
@ -0,0 +1,5 @@
|
||||
package deamon
|
||||
|
||||
const (
|
||||
cmdPing = "ping"
|
||||
)
|
||||
79
deamon/container_init.go
Normal file
79
deamon/container_init.go
Normal file
@ -0,0 +1,79 @@
|
||||
package deamon
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/gob"
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
"github.com/criyle/go-judger/unixsocket"
|
||||
)
|
||||
|
||||
// ContainerInit is called for container init process
|
||||
// it will check if pid == 1, otherwise it is noop
|
||||
// ContainerInit will do infinite loop on socket commands,
|
||||
// and exits when at socket close
|
||||
func ContainerInit() (err error) {
|
||||
// noop if self is not container init process
|
||||
if os.Getpid() != 1 {
|
||||
return nil
|
||||
}
|
||||
|
||||
// exit process (with whole container) upon exit this function
|
||||
defer func() {
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "container_exit: %v", err)
|
||||
os.Exit(1)
|
||||
} else {
|
||||
fmt.Fprintf(os.Stderr, "container_exit")
|
||||
os.Exit(0)
|
||||
}
|
||||
}()
|
||||
|
||||
// new_master shared the socket at fd 3 (marked close_exec)
|
||||
soc, err := unixsocket.NewSocket(3)
|
||||
if err != nil {
|
||||
return fmt.Errorf("container_init: faile to new socket(%v)", err)
|
||||
}
|
||||
var (
|
||||
buffer = make([]byte, bufferSize)
|
||||
cmd Cmd
|
||||
)
|
||||
for {
|
||||
n, msg, err := soc.RecvMsg(buffer)
|
||||
if err != nil {
|
||||
return fmt.Errorf("loop: failed RecvMsg(%v)", err)
|
||||
}
|
||||
dec := gob.NewDecoder(bytes.NewReader(buffer[:n]))
|
||||
if err := dec.Decode(&cmd); err != nil {
|
||||
return fmt.Errorf("loop: failed to decode(%v)", err)
|
||||
}
|
||||
if err := handleCmd(soc, &cmd, msg); err != nil {
|
||||
return fmt.Errorf("loop: failed to execute cmd(%v)", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func handleCmd(s *unixsocket.Socket, cmd *Cmd, msg *unixsocket.Msg) error {
|
||||
switch cmd.Cmd {
|
||||
case cmdPing:
|
||||
return handlePing(s)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func handlePing(s *unixsocket.Socket) error {
|
||||
return sendReply(s, &Reply{})
|
||||
}
|
||||
|
||||
func sendReply(s *unixsocket.Socket, reply *Reply) error {
|
||||
var buffer bytes.Buffer
|
||||
enc := gob.NewEncoder(&buffer)
|
||||
if err := enc.Encode(reply); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := s.SendMsg(buffer.Bytes(), nil); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@ -6,6 +6,8 @@ package deamon
|
||||
|
||||
/*
|
||||
Protocol between client and deamon (not thread safe):
|
||||
- ping (alive check):
|
||||
reply: pong
|
||||
- copyin (copy file into container):
|
||||
send: path, perm, <input fd>
|
||||
reply: "finished" (after copy finished) / "error"
|
||||
|
||||
73
deamon/default.go
Normal file
73
deamon/default.go
Normal file
@ -0,0 +1,73 @@
|
||||
package deamon
|
||||
|
||||
import (
|
||||
"os"
|
||||
|
||||
"github.com/criyle/go-judger/types/mount"
|
||||
"golang.org/x/sys/unix"
|
||||
)
|
||||
|
||||
const (
|
||||
bind = unix.MS_BIND | unix.MS_NOSUID | unix.MS_PRIVATE
|
||||
roBind = bind | unix.MS_RDONLY
|
||||
mFlag = unix.MS_NOSUID | unix.MS_NOATIME | unix.MS_NODEV
|
||||
)
|
||||
|
||||
// default parameters. I was tend to reuse the configs but it is hard since there are some
|
||||
// cross device symblics
|
||||
var (
|
||||
DefaultPath = "PATH=/usr/local/bin:/usr/bin:/bin"
|
||||
|
||||
// rootfs created by bind mounting
|
||||
DefaultMounts = []*mount.Mount{
|
||||
{
|
||||
Source: "/usr",
|
||||
Target: "usr",
|
||||
Flags: roBind,
|
||||
},
|
||||
{
|
||||
Source: "/lib",
|
||||
Target: "lib",
|
||||
Flags: roBind,
|
||||
},
|
||||
{
|
||||
Source: "/lib64",
|
||||
Target: "lib64",
|
||||
Flags: roBind,
|
||||
},
|
||||
{
|
||||
Source: "/bin",
|
||||
Target: "bin",
|
||||
Flags: roBind,
|
||||
},
|
||||
// work dir at /w
|
||||
{
|
||||
Source: "tmpfs",
|
||||
Target: "w",
|
||||
FsType: "tmpfs",
|
||||
Flags: mFlag,
|
||||
},
|
||||
// tmpfs at /tmp
|
||||
{
|
||||
Source: "tmpfs",
|
||||
Target: "tmp",
|
||||
FsType: "tmpfs",
|
||||
Flags: mFlag,
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
func init() {
|
||||
// check if bind mount source exists, e.g. /lib64 does not exists on arm
|
||||
mounts := make([]*mount.Mount, 0, len(DefaultMounts))
|
||||
for _, m := range DefaultMounts {
|
||||
if m.Source != "tmpfs" {
|
||||
if _, err := os.Stat(m.Source); !os.IsNotExist(err) {
|
||||
mounts = append(mounts, m)
|
||||
}
|
||||
} else {
|
||||
mounts = append(mounts, m)
|
||||
}
|
||||
}
|
||||
DefaultMounts = mounts
|
||||
}
|
||||
93
deamon/master.go
Normal file
93
deamon/master.go
Normal file
@ -0,0 +1,93 @@
|
||||
package deamon
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
"github.com/criyle/go-judger/forkexec"
|
||||
"github.com/criyle/go-judger/memfd"
|
||||
"github.com/criyle/go-judger/unixsocket"
|
||||
"golang.org/x/sys/unix"
|
||||
)
|
||||
|
||||
// Master manages single pre-forked container
|
||||
type Master struct {
|
||||
pid int // underlying container init pid
|
||||
socket *unixsocket.Socket // master - container communication
|
||||
}
|
||||
|
||||
// New creates new master with underlying container
|
||||
func New(root string) (*Master, error) {
|
||||
// dummy stdin / stdout / stderr
|
||||
fnull, err := os.OpenFile(os.DevNull, os.O_RDWR, 0666)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("deamon: failed to open devNull(%v)", err)
|
||||
}
|
||||
defer fnull.Close()
|
||||
|
||||
// prepare self memfd
|
||||
self, err := os.Open("/proc/self/exe")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("deamon: failed to open /proc/self/exe(%v)", err)
|
||||
}
|
||||
defer self.Close()
|
||||
|
||||
execFile, err := memfd.DupToMemfd("deamon", self)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("deamon: failed to create memfd(%v)", err)
|
||||
}
|
||||
defer execFile.Close()
|
||||
|
||||
// prepare socket
|
||||
ins, outs, err := unixsocket.NewSocketPair()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("deamon: failed to create socket(%v)", err)
|
||||
}
|
||||
outf, err := outs.Conn.File()
|
||||
if err != nil {
|
||||
ins.Conn.Close()
|
||||
outs.Conn.Close()
|
||||
return nil, fmt.Errorf("deamon: failed to dup file outs(%v)", err)
|
||||
}
|
||||
defer outf.Close()
|
||||
if err = ins.SetPassCred(1); err != nil {
|
||||
ins.Conn.Close()
|
||||
outs.Conn.Close()
|
||||
return nil, fmt.Errorf("deamon: failed to set pass_cred ins(%v)", err)
|
||||
}
|
||||
if err = outs.SetPassCred(1); err != nil {
|
||||
ins.Conn.Close()
|
||||
outs.Conn.Close()
|
||||
return nil, fmt.Errorf("deamon: failed to set pass_cred outs(%v)", err)
|
||||
}
|
||||
|
||||
r := &forkexec.Runner{
|
||||
Args: []string{os.Args[0]},
|
||||
Env: []string{DefaultPath},
|
||||
ExecFile: execFile.Fd(),
|
||||
Files: []uintptr{fnull.Fd(), fnull.Fd(), fnull.Fd(), uintptr(outf.Fd())},
|
||||
WorkDir: "/w",
|
||||
UnshareFlags: forkexec.UnshareFlags,
|
||||
Mounts: DefaultMounts,
|
||||
HostName: "deamon",
|
||||
DomainName: "deamon",
|
||||
PivotRoot: root,
|
||||
}
|
||||
pid, err := r.Start()
|
||||
if err != nil {
|
||||
ins.Conn.Close()
|
||||
outs.Conn.Close()
|
||||
return nil, fmt.Errorf("deamon: failed to execve(%v)", err)
|
||||
}
|
||||
|
||||
outs.Conn.Close()
|
||||
return &Master{pid, ins}, nil
|
||||
}
|
||||
|
||||
// Destroy kill the deamon process (with container)
|
||||
func (m *Master) Destroy() error {
|
||||
var wstatus unix.WaitStatus
|
||||
unix.Kill(m.pid, unix.SIGKILL)
|
||||
_, err := unix.Wait4(m.pid, &wstatus, 0, nil)
|
||||
return err
|
||||
}
|
||||
41
deamon/master_cmd.go
Normal file
41
deamon/master_cmd.go
Normal file
@ -0,0 +1,41 @@
|
||||
package deamon
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/gob"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
// Ping send ping message to container
|
||||
func (m *Master) Ping() error {
|
||||
var (
|
||||
wbuff bytes.Buffer
|
||||
reply Reply
|
||||
)
|
||||
rbuff := GetBuffer()
|
||||
defer PutBuffer(rbuff)
|
||||
// send ping
|
||||
enc := gob.NewEncoder(&wbuff)
|
||||
cmd := Cmd{
|
||||
Cmd: cmdPing,
|
||||
}
|
||||
if err := enc.Encode(cmd); err != nil {
|
||||
return fmt.Errorf("ping: failed to encode(%v)", err)
|
||||
}
|
||||
if err := m.socket.SendMsg(wbuff.Bytes(), nil); err != nil {
|
||||
return fmt.Errorf("ping: failed to sendMsg(%v)", err)
|
||||
}
|
||||
// receive no error
|
||||
n, _, err := m.socket.RecvMsg(rbuff)
|
||||
if err != nil {
|
||||
return fmt.Errorf("ping: failed to recvMsg(%v)", err)
|
||||
}
|
||||
dec := gob.NewDecoder(bytes.NewReader(rbuff[:n]))
|
||||
if err := dec.Decode(&reply); err != nil {
|
||||
return fmt.Errorf("ping: failed to decode(%v)", err)
|
||||
}
|
||||
if reply.Error != "" {
|
||||
return fmt.Errorf("ping: reply error(%v)", reply.Error)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
24
deamon/pool.go
Normal file
24
deamon/pool.go
Normal file
@ -0,0 +1,24 @@
|
||||
package deamon
|
||||
|
||||
import (
|
||||
"sync"
|
||||
)
|
||||
|
||||
// 16k buffsize
|
||||
const bufferSize = 16384
|
||||
|
||||
var bufferPool = sync.Pool{
|
||||
New: func() interface{} {
|
||||
return make([]byte, bufferSize)
|
||||
},
|
||||
}
|
||||
|
||||
// GetBuffer get buffer from pool
|
||||
func GetBuffer() []byte {
|
||||
return bufferPool.Get().([]byte)
|
||||
}
|
||||
|
||||
// PutBuffer return buffer to the pool
|
||||
func PutBuffer(x []byte) {
|
||||
bufferPool.Put(x)
|
||||
}
|
||||
@ -86,6 +86,12 @@ func (r *Runner) Start() (int, error) {
|
||||
// similar to exec_linux, avoid side effect by shuffling around
|
||||
fd, nextfd := prepareFds(r.Files)
|
||||
pipe := p2[1]
|
||||
if nextfd <= pipe {
|
||||
nextfd = pipe + 1
|
||||
}
|
||||
if nextfd <= int(r.ExecFile) {
|
||||
nextfd = int(r.ExecFile) + 1
|
||||
}
|
||||
|
||||
// Acquire the fork lock so that no other threads
|
||||
// create new fds that are not yet close-on-exec
|
||||
@ -177,7 +183,7 @@ func (r *Runner) Start() (int, error) {
|
||||
// Pass 1 & pass 2 assigns fds for child process
|
||||
// Pass 1: fd[i] < i => nextfd
|
||||
if pipe < nextfd {
|
||||
_, _, err1 = syscall.RawSyscall(syscall.SYS_DUP3, uintptr(pipe), uintptr(nextfd), syscall.FD_CLOEXEC)
|
||||
_, _, err1 = syscall.RawSyscall(syscall.SYS_DUP3, uintptr(pipe), uintptr(nextfd), syscall.O_CLOEXEC)
|
||||
if err1 != 0 {
|
||||
goto childerror
|
||||
}
|
||||
@ -185,7 +191,7 @@ func (r *Runner) Start() (int, error) {
|
||||
nextfd++
|
||||
}
|
||||
if r.ExecFile > 0 && int(r.ExecFile) < nextfd {
|
||||
_, _, err1 = syscall.RawSyscall(syscall.SYS_DUP3, r.ExecFile, uintptr(nextfd), syscall.FD_CLOEXEC)
|
||||
_, _, err1 = syscall.RawSyscall(syscall.SYS_DUP3, r.ExecFile, uintptr(nextfd), syscall.O_CLOEXEC)
|
||||
if err1 != 0 {
|
||||
goto childerror
|
||||
}
|
||||
@ -194,7 +200,7 @@ func (r *Runner) Start() (int, error) {
|
||||
}
|
||||
for i := 0; i < len(fd); i++ {
|
||||
if fd[i] >= 0 && fd[i] < int(i) {
|
||||
_, _, err1 = syscall.RawSyscall(syscall.SYS_DUP3, uintptr(fd[i]), uintptr(nextfd), syscall.FD_CLOEXEC)
|
||||
_, _, err1 = syscall.RawSyscall(syscall.SYS_DUP3, uintptr(fd[i]), uintptr(nextfd), syscall.O_CLOEXEC)
|
||||
if err1 != 0 {
|
||||
goto childerror
|
||||
}
|
||||
@ -203,7 +209,6 @@ func (r *Runner) Start() (int, error) {
|
||||
nextfd++
|
||||
}
|
||||
}
|
||||
|
||||
// Pass 2: fd[i] => i
|
||||
for i := 0; i < len(fd); i++ {
|
||||
if fd[i] == -1 {
|
||||
@ -448,7 +453,7 @@ func (r *Runner) Start() (int, error) {
|
||||
}
|
||||
|
||||
childerror:
|
||||
syscall.RawSyscall(syscall.SYS_EXIT, uintptr(err1), 0, 0)
|
||||
syscall.RawSyscall(syscall.SYS_EXIT, uintptr(err1+err2), 0, 0)
|
||||
// cannot reach this point
|
||||
panic("cannot reach")
|
||||
}
|
||||
|
||||
@ -53,6 +53,27 @@ func NewSocket(fd int) (*Socket, error) {
|
||||
return &Socket{unixConn}, nil
|
||||
}
|
||||
|
||||
// NewSocketPair creates conneted unix socketpair using SOCK_SEQPACKET
|
||||
func NewSocketPair() (*Socket, *Socket, error) {
|
||||
fd, err := syscall.Socketpair(syscall.AF_LOCAL, syscall.SOCK_SEQPACKET|syscall.SOCK_CLOEXEC, 0)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("NewSocketPair: failed to call socketpair(%v)", err)
|
||||
}
|
||||
ins, err := NewSocket(fd[0])
|
||||
if err != nil {
|
||||
syscall.Close(fd[0])
|
||||
syscall.Close(fd[1])
|
||||
return nil, nil, fmt.Errorf("NewSocketPair: failed to call NewSocket ins(%v)", err)
|
||||
}
|
||||
outs, err := NewSocket(fd[1])
|
||||
if err != nil {
|
||||
ins.Conn.Close()
|
||||
syscall.Close(fd[1])
|
||||
return nil, nil, fmt.Errorf("NewSocketPair: failed to call NewSocket outs(%v)", err)
|
||||
}
|
||||
return ins, outs, nil
|
||||
}
|
||||
|
||||
// SetPassCred set sockopt for pass cred for unix socket
|
||||
func (s *Socket) SetPassCred(option int) error {
|
||||
sysconn, err := s.Conn.SyscallConn()
|
||||
|
||||
Loading…
Reference in New Issue
Block a user