mirror of
https://github.com/criyle/go-sandbox.git
synced 2025-11-04 14:49:53 +08:00
rename daemon -> container
This commit is contained in:
parent
a24933ac82
commit
f2dacb5752
@ -92,7 +92,7 @@ type Runner interface {
|
|||||||
|
|
||||||
### Pre-forked Container Protocol
|
### Pre-forked Container Protocol
|
||||||
|
|
||||||
1. Pre-fork container daemons to run programs inside
|
1. Pre-fork container to run programs inside
|
||||||
2. Unix socket to pass fd inside / outside
|
2. Unix socket to pass fd inside / outside
|
||||||
|
|
||||||
Container / Master Communication Protocol (single thread):
|
Container / Master Communication Protocol (single thread):
|
||||||
@ -120,7 +120,7 @@ Container / Master Communication Protocol (single thread):
|
|||||||
- send: "kill" (as cmd) / reply: "finished"
|
- send: "kill" (as cmd) / reply: "finished"
|
||||||
- reply:
|
- reply:
|
||||||
|
|
||||||
Any socket related error will cause the daemon exit (with all process inside container)
|
Any socket related error will cause the container exit (with all process inside container)
|
||||||
|
|
||||||
## Packages (/pkg)
|
## Packages (/pkg)
|
||||||
|
|
||||||
@ -137,7 +137,7 @@ Any socket related error will cause the daemon exit (with all process inside con
|
|||||||
## Packages
|
## Packages
|
||||||
|
|
||||||
- config: defines arch & language specified trace condition for ptrace runner from UOJ
|
- config: defines arch & language specified trace condition for ptrace runner from UOJ
|
||||||
- daemon: creates pre-forked container to run programs inside
|
- container: creates pre-forked container to run programs inside
|
||||||
- runner: interface to run program
|
- runner: interface to run program
|
||||||
- ptrace: wrapper to call forkexec and ptracer
|
- ptrace: wrapper to call forkexec and ptracer
|
||||||
- filehandler: an example implementation of UOJ file set
|
- filehandler: an example implementation of UOJ file set
|
||||||
|
|||||||
@ -10,7 +10,7 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/criyle/go-sandbox/config"
|
"github.com/criyle/go-sandbox/config"
|
||||||
"github.com/criyle/go-sandbox/daemon"
|
"github.com/criyle/go-sandbox/container"
|
||||||
"github.com/criyle/go-sandbox/pkg/cgroup"
|
"github.com/criyle/go-sandbox/pkg/cgroup"
|
||||||
"github.com/criyle/go-sandbox/pkg/memfd"
|
"github.com/criyle/go-sandbox/pkg/memfd"
|
||||||
"github.com/criyle/go-sandbox/pkg/mount"
|
"github.com/criyle/go-sandbox/pkg/mount"
|
||||||
@ -44,9 +44,9 @@ func printUsage() {
|
|||||||
os.Exit(2)
|
os.Exit(2)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// container init
|
||||||
func init() {
|
func init() {
|
||||||
// container init
|
container.Init()
|
||||||
daemon.Init()
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func main() {
|
func main() {
|
||||||
@ -71,7 +71,7 @@ func main() {
|
|||||||
flag.Var(&addRawWritable, "add-writable-raw", "Add a writable file (don't transform to its real path)")
|
flag.Var(&addRawWritable, "add-writable-raw", "Add a writable file (don't transform to its real path)")
|
||||||
flag.BoolVar(&useCGroup, "cgroup", false, "Use cgroup to colloct resource usage")
|
flag.BoolVar(&useCGroup, "cgroup", false, "Use cgroup to colloct resource usage")
|
||||||
flag.BoolVar(&memfile, "memfd", false, "Use memfd as exec file")
|
flag.BoolVar(&memfile, "memfd", false, "Use memfd as exec file")
|
||||||
flag.StringVar(&runt, "runner", "ptrace", "Runner for the program (ptrace, ns, daemon)")
|
flag.StringVar(&runt, "runner", "ptrace", "Runner for the program (ptrace, ns, container)")
|
||||||
flag.Parse()
|
flag.Parse()
|
||||||
|
|
||||||
args = flag.Args()
|
args = flag.Args()
|
||||||
@ -133,13 +133,13 @@ func main() {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
type daemonRunner struct {
|
type containerRunner struct {
|
||||||
*daemon.Master
|
container.Environment
|
||||||
*daemon.ExecveParam
|
container.ExecveParam
|
||||||
}
|
}
|
||||||
|
|
||||||
func (r *daemonRunner) Run(c context.Context) <-chan types.Result {
|
func (r *containerRunner) Run(c context.Context) <-chan types.Result {
|
||||||
return r.Master.Execve(c, r.ExecveParam)
|
return r.Environment.Execve(c, r.ExecveParam)
|
||||||
}
|
}
|
||||||
|
|
||||||
func start() (*types.Result, error) {
|
func start() (*types.Result, error) {
|
||||||
@ -231,29 +231,29 @@ func start() (*types.Result, error) {
|
|||||||
MemoryLimit: types.Size(memoryLimit << 20),
|
MemoryLimit: types.Size(memoryLimit << 20),
|
||||||
}
|
}
|
||||||
|
|
||||||
if runt == "daemon" {
|
if runt == "container" {
|
||||||
root, err := ioutil.TempDir("", "dm")
|
root, err := ioutil.TempDir("", "dm")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("cannot make temp root for daemon namespace: %v", err)
|
return nil, fmt.Errorf("cannot make temp root for container namespace: %v", err)
|
||||||
}
|
}
|
||||||
defer os.RemoveAll(root)
|
defer os.RemoveAll(root)
|
||||||
|
|
||||||
b := daemon.Builder{
|
b := container.Builder{
|
||||||
Root: root,
|
Root: root,
|
||||||
}
|
}
|
||||||
|
|
||||||
m, err := b.Build()
|
m, err := b.Build()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("failed to new master: %v", err)
|
return nil, fmt.Errorf("failed to new container: %v", err)
|
||||||
}
|
}
|
||||||
defer m.Destroy()
|
defer m.Destroy()
|
||||||
err = m.Ping()
|
err = m.Ping()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("failed to ping daemon: %v", err)
|
return nil, fmt.Errorf("failed to ping container: %v", err)
|
||||||
}
|
}
|
||||||
runner = &daemonRunner{
|
runner = &containerRunner{
|
||||||
Master: m,
|
Environment: m,
|
||||||
ExecveParam: &daemon.ExecveParam{
|
ExecveParam: container.ExecveParam{
|
||||||
Args: args,
|
Args: args,
|
||||||
Env: []string{pathEnv},
|
Env: []string{pathEnv},
|
||||||
Fds: fds,
|
Fds: fds,
|
||||||
|
|||||||
@ -1,4 +1,4 @@
|
|||||||
package daemon
|
package container
|
||||||
|
|
||||||
const (
|
const (
|
||||||
cmdPing = "ping"
|
cmdPing = "ping"
|
||||||
@ -18,7 +18,7 @@ const (
|
|||||||
containerUID = 1000
|
containerUID = 1000
|
||||||
containerGID = 1000
|
containerGID = 1000
|
||||||
|
|
||||||
containerName = "daemon"
|
containerName = "go-sandbox"
|
||||||
containerWD = "/w"
|
containerWD = "/w"
|
||||||
|
|
||||||
containerMaxProc = 1
|
containerMaxProc = 1
|
||||||
86
container/container_cmd.go
Normal file
86
container/container_cmd.go
Normal file
@ -0,0 +1,86 @@
|
|||||||
|
package container
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"syscall"
|
||||||
|
|
||||||
|
"github.com/criyle/go-sandbox/pkg/unixsocket"
|
||||||
|
)
|
||||||
|
|
||||||
|
func (c *containerServer) handlePing() error {
|
||||||
|
return c.sendReply(&reply{}, nil)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *containerServer) handleConf(conf *confCmd) error {
|
||||||
|
if conf != nil {
|
||||||
|
c.containerConfig = conf.Conf
|
||||||
|
}
|
||||||
|
return c.sendReply(&reply{}, nil)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *containerServer) handleOpen(open []OpenCmd) error {
|
||||||
|
if len(open) == 0 {
|
||||||
|
return c.sendErrorReply("open: no open parameter received")
|
||||||
|
}
|
||||||
|
|
||||||
|
// open files
|
||||||
|
fds := make([]int, 0, len(open))
|
||||||
|
for _, o := range open {
|
||||||
|
outFile, err := os.OpenFile(o.Path, o.Flag, o.Perm)
|
||||||
|
if err != nil {
|
||||||
|
return c.sendErrorReply("open: %v", err)
|
||||||
|
}
|
||||||
|
defer outFile.Close()
|
||||||
|
fds = append(fds, int(outFile.Fd()))
|
||||||
|
}
|
||||||
|
|
||||||
|
return c.sendReply(&reply{}, &unixsocket.Msg{Fds: fds})
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *containerServer) handleDelete(delete *deleteCmd) error {
|
||||||
|
if delete == nil {
|
||||||
|
return c.sendErrorReply("delete: no parameter provided")
|
||||||
|
}
|
||||||
|
if err := os.Remove(delete.Path); err != nil {
|
||||||
|
return c.sendErrorReply("delete: %v", err)
|
||||||
|
}
|
||||||
|
return c.sendReply(&reply{}, nil)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *containerServer) handleReset() error {
|
||||||
|
if err := removeContents("/tmp"); err != nil {
|
||||||
|
return c.sendErrorReply("reset: /tmp %v", err)
|
||||||
|
}
|
||||||
|
if err := removeContents("/w"); err != nil {
|
||||||
|
return c.sendErrorReply("reset: /w %v", err)
|
||||||
|
}
|
||||||
|
return c.sendReply(&reply{}, nil)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *containerServer) recvCmd() (*cmd, *unixsocket.Msg, error) {
|
||||||
|
cm := new(cmd)
|
||||||
|
msg, err := (*socket)(c.socket).RecvMsg(cm)
|
||||||
|
if err != nil {
|
||||||
|
return nil, nil, err
|
||||||
|
}
|
||||||
|
return cm, msg, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *containerServer) sendReply(rep *reply, msg *unixsocket.Msg) error {
|
||||||
|
return (*socket)(c.socket).SendMsg(rep, msg)
|
||||||
|
}
|
||||||
|
|
||||||
|
// sendErrorReply sends error reply
|
||||||
|
func (c *containerServer) sendErrorReply(ft string, v ...interface{}) error {
|
||||||
|
errRep := &errorReply{
|
||||||
|
Msg: fmt.Sprintf(ft, v...),
|
||||||
|
}
|
||||||
|
// store errno
|
||||||
|
if len(v) == 1 {
|
||||||
|
if errno, ok := v[0].(syscall.Errno); ok {
|
||||||
|
errRep.Errno = &errno
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return c.sendReply(&reply{Error: errRep}, nil)
|
||||||
|
}
|
||||||
@ -1,4 +1,4 @@
|
|||||||
package daemon
|
package container
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
@ -10,7 +10,7 @@ import (
|
|||||||
"github.com/criyle/go-sandbox/types"
|
"github.com/criyle/go-sandbox/types"
|
||||||
)
|
)
|
||||||
|
|
||||||
func (c *containerServer) handleExecve(cmd *ExecCmd, msg *unixsocket.Msg) error {
|
func (c *containerServer) handleExecve(cmd *execCmd, msg *unixsocket.Msg) error {
|
||||||
var (
|
var (
|
||||||
files []uintptr
|
files []uintptr
|
||||||
execFile uintptr
|
execFile uintptr
|
||||||
@ -37,21 +37,21 @@ func (c *containerServer) handleExecve(cmd *ExecCmd, msg *unixsocket.Msg) error
|
|||||||
}
|
}
|
||||||
|
|
||||||
syncFunc := func(pid int) error {
|
syncFunc := func(pid int) error {
|
||||||
msg2 := &unixsocket.Msg{
|
msg := &unixsocket.Msg{
|
||||||
Cred: &syscall.Ucred{
|
Cred: &syscall.Ucred{
|
||||||
Pid: int32(pid),
|
Pid: int32(pid),
|
||||||
Uid: uint32(syscall.Getuid()),
|
Uid: uint32(syscall.Getuid()),
|
||||||
Gid: uint32(syscall.Getgid()),
|
Gid: uint32(syscall.Getgid()),
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
if err2 := c.sendReply(&Reply{}, msg2); err2 != nil {
|
if err := c.sendReply(&reply{}, msg); err != nil {
|
||||||
return fmt.Errorf("syncFunc: sendReply(%v)", err2)
|
return fmt.Errorf("syncFunc: sendReply %w", err)
|
||||||
}
|
}
|
||||||
cmd2, _, err2 := c.recvCmd()
|
cmd, _, err := c.recvCmd()
|
||||||
if err2 != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("syncFunc: recvCmd(%v)", err2)
|
return fmt.Errorf("syncFunc: recvCmd %w", err)
|
||||||
}
|
}
|
||||||
if cmd2.Cmd == cmdKill {
|
if cmd.Cmd == cmdKill {
|
||||||
return fmt.Errorf("syncFunc: received kill")
|
return fmt.Errorf("syncFunc: received kill")
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
@ -121,8 +121,11 @@ func (c *containerServer) handleExecve(cmd *ExecCmd, msg *unixsocket.Msg) error
|
|||||||
switch {
|
switch {
|
||||||
case wstatus.Exited():
|
case wstatus.Exited():
|
||||||
exitStatus := wstatus.ExitStatus()
|
exitStatus := wstatus.ExitStatus()
|
||||||
c.sendReply(&Reply{
|
if exitStatus != 0 {
|
||||||
ExecReply: &ExecReply{
|
status = types.StatusNonzeroExitStatus
|
||||||
|
}
|
||||||
|
c.sendReply(&reply{
|
||||||
|
ExecReply: &execReply{
|
||||||
Status: status,
|
Status: status,
|
||||||
ExitStatus: exitStatus,
|
ExitStatus: exitStatus,
|
||||||
Time: userTime,
|
Time: userTime,
|
||||||
@ -142,8 +145,8 @@ func (c *containerServer) handleExecve(cmd *ExecCmd, msg *unixsocket.Msg) error
|
|||||||
default:
|
default:
|
||||||
status = types.StatusSignalled
|
status = types.StatusSignalled
|
||||||
}
|
}
|
||||||
c.sendReply(&Reply{
|
c.sendReply(&reply{
|
||||||
ExecReply: &ExecReply{
|
ExecReply: &execReply{
|
||||||
ExitStatus: int(wstatus.Signal()),
|
ExitStatus: int(wstatus.Signal()),
|
||||||
Status: status,
|
Status: status,
|
||||||
Time: userTime,
|
Time: userTime,
|
||||||
@ -158,5 +161,5 @@ func (c *containerServer) handleExecve(cmd *ExecCmd, msg *unixsocket.Msg) error
|
|||||||
|
|
||||||
// wait for kill msg and reply done for finish
|
// wait for kill msg and reply done for finish
|
||||||
<-killDone
|
<-killDone
|
||||||
return c.sendReply(&Reply{}, nil)
|
return c.sendReply(&reply{}, nil)
|
||||||
}
|
}
|
||||||
93
container/container_init.go
Normal file
93
container/container_init.go
Normal file
@ -0,0 +1,93 @@
|
|||||||
|
package container
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"runtime"
|
||||||
|
|
||||||
|
"github.com/criyle/go-sandbox/pkg/unixsocket"
|
||||||
|
)
|
||||||
|
|
||||||
|
type containerServer struct {
|
||||||
|
socket *unixsocket.Socket
|
||||||
|
containerConfig
|
||||||
|
}
|
||||||
|
|
||||||
|
// Init is called for container init process
|
||||||
|
// it will check if pid == 1, otherwise it is noop
|
||||||
|
// Init will do infinite loop on socket commands,
|
||||||
|
// and exits when at socket close, use it in init function
|
||||||
|
func Init() (err error) {
|
||||||
|
// noop if self is not container init process
|
||||||
|
// Notice: docker init is also 1, additional check for args[1] == init
|
||||||
|
if os.Getpid() != 1 || len(os.Args) != 2 || os.Args[1] != initArg {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// exit process (with whole container) upon exit this function
|
||||||
|
// possible reason:
|
||||||
|
// 1. socket broken (parent exit)
|
||||||
|
// 2. panic
|
||||||
|
// 3. undefined cmd (possible race condition)
|
||||||
|
defer func() {
|
||||||
|
if err := recover(); err != nil {
|
||||||
|
fmt.Fprintf(os.Stderr, "container_exit: panic: %v\n", err)
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
fmt.Fprintf(os.Stderr, "container_exit: %v\n", err)
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
fmt.Fprintf(os.Stderr, "container_exit\n")
|
||||||
|
os.Exit(0)
|
||||||
|
}()
|
||||||
|
|
||||||
|
// limit container resource usage
|
||||||
|
runtime.GOMAXPROCS(containerMaxProc)
|
||||||
|
|
||||||
|
// new_master shared the socket at fd 3 (marked close_exec)
|
||||||
|
const defaultFd = 3
|
||||||
|
soc, err := unixsocket.NewSocket(defaultFd)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("container_init: failed to new socket %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// serve forever
|
||||||
|
cs := &containerServer{socket: soc}
|
||||||
|
return cs.serve()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *containerServer) serve() error {
|
||||||
|
for {
|
||||||
|
cmd, msg, err := c.recvCmd()
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("serve: recvCmd %w", err)
|
||||||
|
}
|
||||||
|
if err := c.handleCmd(cmd, msg); err != nil {
|
||||||
|
return fmt.Errorf("serve: failed to execute cmd %w", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *containerServer) handleCmd(cmd *cmd, msg *unixsocket.Msg) error {
|
||||||
|
switch cmd.Cmd {
|
||||||
|
case cmdPing:
|
||||||
|
return c.handlePing()
|
||||||
|
|
||||||
|
case cmdConf:
|
||||||
|
return c.handleConf(cmd.ConfCmd)
|
||||||
|
|
||||||
|
case cmdOpen:
|
||||||
|
return c.handleOpen(cmd.OpenCmd)
|
||||||
|
|
||||||
|
case cmdDelete:
|
||||||
|
return c.handleDelete(cmd.DeleteCmd)
|
||||||
|
|
||||||
|
case cmdReset:
|
||||||
|
return c.handleReset()
|
||||||
|
|
||||||
|
case cmdExecve:
|
||||||
|
return c.handleExecve(cmd.ExecCmd, msg)
|
||||||
|
}
|
||||||
|
return fmt.Errorf("unknown command: %s", cmd.Cmd)
|
||||||
|
}
|
||||||
44
container/doc.go
Normal file
44
container/doc.go
Normal file
@ -0,0 +1,44 @@
|
|||||||
|
// Package container provides pre-forked container environment to
|
||||||
|
// run programs in isolated Linux namespaces.
|
||||||
|
//
|
||||||
|
// Overview
|
||||||
|
//
|
||||||
|
// It creates container within unshared container and communicate
|
||||||
|
// with host process using unix socket with
|
||||||
|
// oob for fd / pid and commands encoded by gob.
|
||||||
|
//
|
||||||
|
// Protocol
|
||||||
|
//
|
||||||
|
// Host to container communication protocol is single threaded and always initiated by
|
||||||
|
// the host:
|
||||||
|
//
|
||||||
|
// - ping (alive check):
|
||||||
|
// - reply: pong
|
||||||
|
//
|
||||||
|
// - conf (set configuration):
|
||||||
|
// - reply pong
|
||||||
|
//
|
||||||
|
// - open (open files in given mode inside container):
|
||||||
|
// - send: []OpenCmd
|
||||||
|
// - reply: "success", file fds / "error"
|
||||||
|
//
|
||||||
|
// - delete (unlink file / rmdir dir inside container):
|
||||||
|
// - send: path
|
||||||
|
// - reply: "finished" / "error"
|
||||||
|
//
|
||||||
|
// - reset (clean up container for later use (clear workdir / tmp)):
|
||||||
|
// - send:
|
||||||
|
// - reply: "success"
|
||||||
|
//
|
||||||
|
// - execve: (execute file inside container):
|
||||||
|
// - send: argv, env, rLimits, fds
|
||||||
|
// - reply:
|
||||||
|
// - success: "success", pid
|
||||||
|
// - failed: "failed"
|
||||||
|
// - send (success): "init_finished" (as cmd)
|
||||||
|
// - reply: "finished" / send: "kill" (as cmd)
|
||||||
|
// - send: "kill" (as cmd) / reply: "finished"
|
||||||
|
// - reply:
|
||||||
|
//
|
||||||
|
// Any socket related error will cause the container exit with all process inside container
|
||||||
|
package container
|
||||||
@ -1,6 +1,7 @@
|
|||||||
package daemon
|
package container
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"context"
|
||||||
"fmt"
|
"fmt"
|
||||||
"os"
|
"os"
|
||||||
"sync"
|
"sync"
|
||||||
@ -10,13 +11,14 @@ import (
|
|||||||
"github.com/criyle/go-sandbox/pkg/memfd"
|
"github.com/criyle/go-sandbox/pkg/memfd"
|
||||||
"github.com/criyle/go-sandbox/pkg/mount"
|
"github.com/criyle/go-sandbox/pkg/mount"
|
||||||
"github.com/criyle/go-sandbox/pkg/unixsocket"
|
"github.com/criyle/go-sandbox/pkg/unixsocket"
|
||||||
|
"github.com/criyle/go-sandbox/types"
|
||||||
"golang.org/x/sys/unix"
|
"golang.org/x/sys/unix"
|
||||||
)
|
)
|
||||||
|
|
||||||
// DefaultPath defines path evironment variable for container init process
|
// PathEnv defines path environment variable for the container init process
|
||||||
const DefaultPath = "PATH=/usr/local/bin:/usr/bin:/bin"
|
const PathEnv = "PATH=/usr/local/bin:/usr/bin:/bin"
|
||||||
|
|
||||||
// Builder builds instance of container masters
|
// Builder builds instance of container environment
|
||||||
type Builder struct {
|
type Builder struct {
|
||||||
// Root is container root mount path, empty uses current work path
|
// Root is container root mount path, empty uses current work path
|
||||||
Root string
|
Root string
|
||||||
@ -41,15 +43,25 @@ type CredGenerator interface {
|
|||||||
Get() syscall.Credential
|
Get() syscall.Credential
|
||||||
}
|
}
|
||||||
|
|
||||||
// Master manages single pre-forked container
|
// Environment holds single progrem containered environment
|
||||||
type Master struct {
|
type Environment interface {
|
||||||
|
Ping() error
|
||||||
|
Open([]OpenCmd) ([]*os.File, error)
|
||||||
|
Delete(p string) error
|
||||||
|
Reset() error
|
||||||
|
Execve(context.Context, ExecveParam) <-chan types.Result
|
||||||
|
Destroy() error
|
||||||
|
}
|
||||||
|
|
||||||
|
// container manages single pre-forked container environment
|
||||||
|
type container struct {
|
||||||
pid int // underlying container init pid
|
pid int // underlying container init pid
|
||||||
socket *unixsocket.Socket // master - container communication
|
socket *unixsocket.Socket // master - container communication
|
||||||
mu sync.Mutex // lock to avoid race condition
|
mu sync.Mutex // lock to avoid race condition
|
||||||
}
|
}
|
||||||
|
|
||||||
// Build creates new master with underlying container
|
// Build creates new environment with underlying container
|
||||||
func (b *Builder) Build() (*Master, error) {
|
func (b *Builder) Build() (Environment, error) {
|
||||||
var (
|
var (
|
||||||
err error
|
err error
|
||||||
cred syscall.Credential
|
cred syscall.Credential
|
||||||
@ -64,19 +76,22 @@ func (b *Builder) Build() (*Master, error) {
|
|||||||
WithTmpfs("w", ""). // work dir
|
WithTmpfs("w", ""). // work dir
|
||||||
WithTmpfs("tmp", ""). // tmp
|
WithTmpfs("tmp", ""). // tmp
|
||||||
Build(true); err != nil {
|
Build(true); err != nil {
|
||||||
return nil, fmt.Errorf("daemon: failed to build rootfs mount %v", err)
|
return nil, fmt.Errorf("container: failed to build rootfs mount %w", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// container root directory on the host
|
||||||
root := b.Root
|
root := b.Root
|
||||||
if root == "" {
|
if root == "" {
|
||||||
if root, err = os.Getwd(); err != nil {
|
if root, err = os.Getwd(); err != nil {
|
||||||
return nil, fmt.Errorf("daemon: failed to get work directory %v", err)
|
return nil, fmt.Errorf("container: failed to get work directory %w", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// prepare stdin / stdout / stderr
|
// prepare stdin / stdout / stderr
|
||||||
devNull, err := os.OpenFile(os.DevNull, os.O_RDWR, os.ModePerm)
|
devNull, err := os.OpenFile(os.DevNull, os.O_RDWR, os.ModePerm)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("daemon: failed to open devNull(%v)", err)
|
return nil, fmt.Errorf("container: failed to open devNull %w", err)
|
||||||
}
|
}
|
||||||
defer devNull.Close()
|
defer devNull.Close()
|
||||||
|
|
||||||
@ -88,30 +103,30 @@ func (b *Builder) Build() (*Master, error) {
|
|||||||
files = append(files, devNull.Fd())
|
files = append(files, devNull.Fd())
|
||||||
}
|
}
|
||||||
|
|
||||||
// prepare self memfd
|
// prepare container exec file
|
||||||
execFile, err := b.exec()
|
execFile, err := b.exec()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("deamon: %v", err)
|
return nil, fmt.Errorf("container: prepare exec %w", err)
|
||||||
}
|
}
|
||||||
defer execFile.Close()
|
defer execFile.Close()
|
||||||
|
|
||||||
// prepare socket
|
// prepare host <-> container unix socket
|
||||||
ins, outs, err := newPassCredSocketPair()
|
ins, outs, err := newPassCredSocketPair()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("daemon: failed to create socket: %v", err)
|
return nil, fmt.Errorf("container: failed to create socket: %w", err)
|
||||||
}
|
}
|
||||||
defer outs.Close()
|
defer outs.Close()
|
||||||
|
|
||||||
outf, err := outs.File()
|
outf, err := outs.File()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
ins.Close()
|
ins.Close()
|
||||||
return nil, fmt.Errorf("daemon: failed to dup file outs(%v)", err)
|
return nil, fmt.Errorf("container: failed to dup container socket fd %w", err)
|
||||||
}
|
}
|
||||||
defer outf.Close()
|
defer outf.Close()
|
||||||
|
|
||||||
files = append(files, uintptr(outf.Fd()))
|
files = append(files, uintptr(outf.Fd()))
|
||||||
|
|
||||||
// prepare credential
|
// prepare container running credential
|
||||||
if b.CredGenerator != nil {
|
if b.CredGenerator != nil {
|
||||||
cred = b.CredGenerator.Get()
|
cred = b.CredGenerator.Get()
|
||||||
uidMap, gidMap = getIDMapping(&cred)
|
uidMap, gidMap = getIDMapping(&cred)
|
||||||
@ -119,7 +134,7 @@ func (b *Builder) Build() (*Master, error) {
|
|||||||
|
|
||||||
r := &forkexec.Runner{
|
r := &forkexec.Runner{
|
||||||
Args: []string{os.Args[0], initArg},
|
Args: []string{os.Args[0], initArg},
|
||||||
Env: []string{DefaultPath},
|
Env: []string{PathEnv},
|
||||||
ExecFile: execFile.Fd(),
|
ExecFile: execFile.Fd(),
|
||||||
Files: files,
|
Files: files,
|
||||||
WorkDir: containerWD,
|
WorkDir: containerWD,
|
||||||
@ -134,41 +149,42 @@ func (b *Builder) Build() (*Master, error) {
|
|||||||
pid, err := r.Start()
|
pid, err := r.Start()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
ins.Close()
|
ins.Close()
|
||||||
return nil, fmt.Errorf("daemon: failed to execve(%v)", err)
|
return nil, fmt.Errorf("container: failed to start container %w", err)
|
||||||
}
|
}
|
||||||
m := &Master{
|
|
||||||
|
c := &container{
|
||||||
pid: pid,
|
pid: pid,
|
||||||
socket: ins,
|
socket: ins,
|
||||||
}
|
}
|
||||||
|
|
||||||
// set configuration and check if container creation successful
|
// set configuration and check if container creation successful
|
||||||
if err = m.conf(&containerConfig{
|
if err = c.conf(&containerConfig{
|
||||||
Cred: b.CredGenerator != nil,
|
Cred: b.CredGenerator != nil,
|
||||||
}); err != nil {
|
}); err != nil {
|
||||||
m.Destroy()
|
c.Destroy()
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
return m, nil
|
return c, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// Destroy kill the daemon process (with container)
|
// Destroy kill the container process (with its children)
|
||||||
// if stderr enabled, collect the output as error
|
// if stderr enabled, collect the output as error
|
||||||
func (m *Master) Destroy() error {
|
func (c *container) Destroy() error {
|
||||||
// close socket (abort any ongoing command)
|
// close socket (abort any ongoing command)
|
||||||
m.socket.Close()
|
c.socket.Close()
|
||||||
|
|
||||||
// wait commands terminates
|
// wait commands terminates
|
||||||
m.mu.Lock()
|
c.mu.Lock()
|
||||||
defer m.mu.Unlock()
|
defer c.mu.Unlock()
|
||||||
|
|
||||||
// kill process
|
// kill process
|
||||||
var wstatus unix.WaitStatus
|
var wstatus unix.WaitStatus
|
||||||
unix.Kill(m.pid, unix.SIGKILL)
|
unix.Kill(c.pid, unix.SIGKILL)
|
||||||
// wait for container process to exit
|
// wait for container process to exit
|
||||||
_, err := unix.Wait4(m.pid, &wstatus, 0, nil)
|
_, err := unix.Wait4(c.pid, &wstatus, 0, nil)
|
||||||
for err == unix.EINTR {
|
for err == unix.EINTR {
|
||||||
_, err = unix.Wait4(m.pid, &wstatus, 0, nil)
|
_, err = unix.Wait4(c.pid, &wstatus, 0, nil)
|
||||||
}
|
}
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@ -181,7 +197,7 @@ func (b *Builder) exec() (*os.File, error) {
|
|||||||
return OpenCurrentExec()
|
return OpenCurrentExec()
|
||||||
}
|
}
|
||||||
|
|
||||||
// OpenCurrentExec opens current executable and dup it to memfd
|
// OpenCurrentExec opens current executable and dup and seal it to memfd
|
||||||
func OpenCurrentExec() (*os.File, error) {
|
func OpenCurrentExec() (*os.File, error) {
|
||||||
self, err := os.Open(currentExec)
|
self, err := os.Open(currentExec)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@ -189,7 +205,7 @@ func OpenCurrentExec() (*os.File, error) {
|
|||||||
}
|
}
|
||||||
defer self.Close()
|
defer self.Close()
|
||||||
|
|
||||||
execFile, err := memfd.DupToMemfd("daemon", self)
|
execFile, err := memfd.DupToMemfd("init", self)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("failed to create memfd: %v", err)
|
return nil, fmt.Errorf("failed to create memfd: %v", err)
|
||||||
}
|
}
|
||||||
@ -1,4 +1,4 @@
|
|||||||
package daemon
|
package container
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
@ -9,55 +9,55 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
// Ping send ping message to container
|
// Ping send ping message to container
|
||||||
func (m *Master) Ping() error {
|
func (c *container) Ping() error {
|
||||||
m.mu.Lock()
|
c.mu.Lock()
|
||||||
defer m.mu.Unlock()
|
defer c.mu.Unlock()
|
||||||
|
|
||||||
// avoid infinite wait (max 3s)
|
// avoid infinite wait (max 3s)
|
||||||
const pingWait = 3 * time.Second
|
const pingWait = 3 * time.Second
|
||||||
m.socket.SetDeadline(time.Now().Add(pingWait))
|
c.socket.SetDeadline(time.Now().Add(pingWait))
|
||||||
defer m.socket.SetDeadline(time.Time{})
|
defer c.socket.SetDeadline(time.Time{})
|
||||||
|
|
||||||
// send ping
|
// send ping
|
||||||
cmd := Cmd{
|
cmd := cmd{
|
||||||
Cmd: cmdPing,
|
Cmd: cmdPing,
|
||||||
}
|
}
|
||||||
if err := m.sendCmd(&cmd, nil); err != nil {
|
if err := c.sendCmd(&cmd, nil); err != nil {
|
||||||
return fmt.Errorf("ping: %v", err)
|
return fmt.Errorf("ping: %v", err)
|
||||||
}
|
}
|
||||||
// receive no error
|
// receive no error
|
||||||
return m.recvAckReply("ping")
|
return c.recvAckReply("ping")
|
||||||
}
|
}
|
||||||
|
|
||||||
// conf send configuration to container (used by builder only)
|
// conf send configuration to container (used by builder only)
|
||||||
func (m *Master) conf(conf *containerConfig) error {
|
func (c *container) conf(conf *containerConfig) error {
|
||||||
m.mu.Lock()
|
c.mu.Lock()
|
||||||
defer m.mu.Unlock()
|
defer c.mu.Unlock()
|
||||||
|
|
||||||
cmd := Cmd{
|
cmd := cmd{
|
||||||
Cmd: cmdConf,
|
Cmd: cmdConf,
|
||||||
ConfCmd: &ConfCmd{Conf: *conf},
|
ConfCmd: &confCmd{Conf: *conf},
|
||||||
}
|
}
|
||||||
if err := m.sendCmd(&cmd, nil); err != nil {
|
if err := c.sendCmd(&cmd, nil); err != nil {
|
||||||
return fmt.Errorf("conf: %v", err)
|
return fmt.Errorf("conf: %v", err)
|
||||||
}
|
}
|
||||||
return m.recvAckReply("conf")
|
return c.recvAckReply("conf")
|
||||||
}
|
}
|
||||||
|
|
||||||
// Open open files in container
|
// Open open files in container
|
||||||
func (m *Master) Open(p []OpenCmd) ([]*os.File, error) {
|
func (c *container) Open(p []OpenCmd) ([]*os.File, error) {
|
||||||
m.mu.Lock()
|
c.mu.Lock()
|
||||||
defer m.mu.Unlock()
|
defer c.mu.Unlock()
|
||||||
|
|
||||||
// send copyin
|
// send copyin
|
||||||
cmd := Cmd{
|
cmd := cmd{
|
||||||
Cmd: cmdOpen,
|
Cmd: cmdOpen,
|
||||||
OpenCmd: p,
|
OpenCmd: p,
|
||||||
}
|
}
|
||||||
if err := m.sendCmd(&cmd, nil); err != nil {
|
if err := c.sendCmd(&cmd, nil); err != nil {
|
||||||
return nil, fmt.Errorf("open: %v", err)
|
return nil, fmt.Errorf("open: %v", err)
|
||||||
}
|
}
|
||||||
reply, msg, err := m.recvReply()
|
reply, msg, err := c.recvReply()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("open: %v", err)
|
return nil, fmt.Errorf("open: %v", err)
|
||||||
}
|
}
|
||||||
@ -82,36 +82,36 @@ func (m *Master) Open(p []OpenCmd) ([]*os.File, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Delete remove file from container
|
// Delete remove file from container
|
||||||
func (m *Master) Delete(p string) error {
|
func (c *container) Delete(p string) error {
|
||||||
m.mu.Lock()
|
c.mu.Lock()
|
||||||
defer m.mu.Unlock()
|
defer c.mu.Unlock()
|
||||||
|
|
||||||
cmd := Cmd{
|
cmd := cmd{
|
||||||
Cmd: cmdDelete,
|
Cmd: cmdDelete,
|
||||||
DeleteCmd: &DeleteCmd{Path: p},
|
DeleteCmd: &deleteCmd{Path: p},
|
||||||
}
|
}
|
||||||
if err := m.sendCmd(&cmd, nil); err != nil {
|
if err := c.sendCmd(&cmd, nil); err != nil {
|
||||||
return fmt.Errorf("delete: %v", err)
|
return fmt.Errorf("delete: %v", err)
|
||||||
}
|
}
|
||||||
return m.recvAckReply("delete")
|
return c.recvAckReply("delete")
|
||||||
}
|
}
|
||||||
|
|
||||||
// Reset remove all from /tmp and /w
|
// Reset remove all from /tmp and /w
|
||||||
func (m *Master) Reset() error {
|
func (c *container) Reset() error {
|
||||||
m.mu.Lock()
|
c.mu.Lock()
|
||||||
defer m.mu.Unlock()
|
defer c.mu.Unlock()
|
||||||
|
|
||||||
cmd := Cmd{
|
cmd := cmd{
|
||||||
Cmd: cmdReset,
|
Cmd: cmdReset,
|
||||||
}
|
}
|
||||||
if err := m.sendCmd(&cmd, nil); err != nil {
|
if err := c.sendCmd(&cmd, nil); err != nil {
|
||||||
return fmt.Errorf("reset: %v", err)
|
return fmt.Errorf("reset: %v", err)
|
||||||
}
|
}
|
||||||
return m.recvAckReply("reset")
|
return c.recvAckReply("reset")
|
||||||
}
|
}
|
||||||
|
|
||||||
func (m *Master) recvAckReply(name string) error {
|
func (c *container) recvAckReply(name string) error {
|
||||||
reply, _, err := m.recvReply()
|
reply, _, err := c.recvReply()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("%v: recvAck %v", name, err)
|
return fmt.Errorf("%v: recvAck %v", name, err)
|
||||||
}
|
}
|
||||||
@ -121,15 +121,15 @@ func (m *Master) recvAckReply(name string) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (m *Master) recvReply() (*Reply, *unixsocket.Msg, error) {
|
func (c *container) recvReply() (*reply, *unixsocket.Msg, error) {
|
||||||
reply := new(Reply)
|
reply := new(reply)
|
||||||
msg, err := (*socket)(m.socket).RecvMsg(reply)
|
msg, err := (*socket)(c.socket).RecvMsg(reply)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, nil, err
|
return nil, nil, err
|
||||||
}
|
}
|
||||||
return reply, msg, nil
|
return reply, msg, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (m *Master) sendCmd(cmd *Cmd, msg *unixsocket.Msg) error {
|
func (c *container) sendCmd(cmd *cmd, msg *unixsocket.Msg) error {
|
||||||
return (*socket)(m.socket).SendMsg(cmd, msg)
|
return (*socket)(c.socket).SendMsg(cmd, msg)
|
||||||
}
|
}
|
||||||
@ -1,4 +1,4 @@
|
|||||||
package daemon
|
package container
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
@ -16,20 +16,19 @@ type ExecveParam struct {
|
|||||||
Env []string
|
Env []string
|
||||||
Fds []uintptr
|
Fds []uintptr
|
||||||
|
|
||||||
// fexecve fd
|
// fd parameter of fexecve
|
||||||
ExecFile uintptr
|
ExecFile uintptr
|
||||||
|
|
||||||
// POSIX Resource limit set by set rlimit
|
// POSIX Resource limit set by set rlimit
|
||||||
RLimits []rlimit.RLimit
|
RLimits []rlimit.RLimit
|
||||||
|
|
||||||
// SyncFunc called with pid before execve (e.g. add to cgroup)
|
// SyncFunc called with pid before execve (for adding the process to cgroups)
|
||||||
SyncFunc func(pid int) error
|
SyncFunc func(pid int) error
|
||||||
}
|
}
|
||||||
|
|
||||||
// Execve runs process inside container
|
// Execve runs process inside container. It accepts context cancelation as time limit exceeded.
|
||||||
// accepts context for cancelation
|
func (c *container) Execve(ctx context.Context, param ExecveParam) <-chan types.Result {
|
||||||
func (m *Master) Execve(c context.Context, param *ExecveParam) <-chan types.Result {
|
c.mu.Lock()
|
||||||
m.mu.Lock()
|
|
||||||
|
|
||||||
sTime := time.Now()
|
sTime := time.Now()
|
||||||
|
|
||||||
@ -53,46 +52,46 @@ func (m *Master) Execve(c context.Context, param *ExecveParam) <-chan types.Resu
|
|||||||
msg := &unixsocket.Msg{
|
msg := &unixsocket.Msg{
|
||||||
Fds: files,
|
Fds: files,
|
||||||
}
|
}
|
||||||
execCmd := &ExecCmd{
|
execCmd := &execCmd{
|
||||||
Argv: param.Args,
|
Argv: param.Args,
|
||||||
Env: param.Env,
|
Env: param.Env,
|
||||||
RLimits: param.RLimits,
|
RLimits: param.RLimits,
|
||||||
FdExec: param.ExecFile > 0,
|
FdExec: param.ExecFile > 0,
|
||||||
}
|
}
|
||||||
cmd := Cmd{
|
cm := cmd{
|
||||||
Cmd: cmdExecve,
|
Cmd: cmdExecve,
|
||||||
ExecCmd: execCmd,
|
ExecCmd: execCmd,
|
||||||
}
|
}
|
||||||
if err := m.sendCmd(&cmd, msg); err != nil {
|
if err := c.sendCmd(&cm, msg); err != nil {
|
||||||
m.mu.Unlock()
|
c.mu.Unlock()
|
||||||
return errResult("execve: sendCmd %v", err)
|
return errResult("execve: sendCmd %v", err)
|
||||||
}
|
}
|
||||||
// sync function
|
// sync function
|
||||||
reply, msg, err := m.recvReply()
|
reply, msg, err := c.recvReply()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
m.mu.Unlock()
|
c.mu.Unlock()
|
||||||
return errResult("execve: recvReply %v", err)
|
return errResult("execve: recvReply %v", err)
|
||||||
}
|
}
|
||||||
// if sync function did not involved
|
// if sync function did not involved
|
||||||
if reply.Error != nil || msg == nil || msg.Cred == nil {
|
if reply.Error != nil || msg == nil || msg.Cred == nil {
|
||||||
// tell kill function to exit and sync
|
// tell kill function to exit and sync
|
||||||
m.execveSyncKill()
|
c.execveSyncKill()
|
||||||
m.mu.Unlock()
|
c.mu.Unlock()
|
||||||
return errResult("execve: no pid received or error %v", reply.Error)
|
return errResult("execve: no pid received or error %v", reply.Error)
|
||||||
}
|
}
|
||||||
if param.SyncFunc != nil {
|
if param.SyncFunc != nil {
|
||||||
if err := param.SyncFunc(int(msg.Cred.Pid)); err != nil {
|
if err := param.SyncFunc(int(msg.Cred.Pid)); err != nil {
|
||||||
// tell sync function to exit and recv error
|
// tell sync function to exit and recv error
|
||||||
m.execveSyncKill()
|
c.execveSyncKill()
|
||||||
// tell kill function to exit and sync
|
// tell kill function to exit and sync
|
||||||
m.execveSyncKill()
|
c.execveSyncKill()
|
||||||
m.mu.Unlock()
|
c.mu.Unlock()
|
||||||
return errResult("execve: syncfunc failed %v", err)
|
return errResult("execve: syncfunc failed %v", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
// send to syncFunc ack ok
|
// send to syncFunc ack ok
|
||||||
if err := m.sendCmd(&Cmd{Cmd: cmdOk}, nil); err != nil {
|
if err := c.sendCmd(&cmd{Cmd: cmdOk}, nil); err != nil {
|
||||||
m.mu.Unlock()
|
c.mu.Unlock()
|
||||||
return errResult("execve: ack failed %v", err)
|
return errResult("execve: ack failed %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -102,12 +101,12 @@ func (m *Master) Execve(c context.Context, param *ExecveParam) <-chan types.Resu
|
|||||||
|
|
||||||
// Wait
|
// Wait
|
||||||
go func() {
|
go func() {
|
||||||
reply2, _, err := m.recvReply()
|
reply2, _, err := c.recvReply()
|
||||||
close(waitDone)
|
close(waitDone)
|
||||||
// done signal (should recv after kill)
|
// done signal (should recv after kill)
|
||||||
m.recvReply()
|
c.recvReply()
|
||||||
// unlock after last read / write
|
// unlock after last read / write
|
||||||
m.mu.Unlock()
|
c.mu.Unlock()
|
||||||
|
|
||||||
// handle potential error
|
// handle potential error
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@ -145,17 +144,17 @@ func (m *Master) Execve(c context.Context, param *ExecveParam) <-chan types.Resu
|
|||||||
// Kill (if wait is done, a kill message need to be send to collect zombies)
|
// Kill (if wait is done, a kill message need to be send to collect zombies)
|
||||||
go func() {
|
go func() {
|
||||||
select {
|
select {
|
||||||
case <-c.Done():
|
case <-ctx.Done():
|
||||||
case <-waitDone:
|
case <-waitDone:
|
||||||
}
|
}
|
||||||
m.sendCmd(&Cmd{Cmd: cmdKill}, nil)
|
c.sendCmd(&cmd{Cmd: cmdKill}, nil)
|
||||||
}()
|
}()
|
||||||
|
|
||||||
return result
|
return result
|
||||||
}
|
}
|
||||||
|
|
||||||
// execveSyncKill will send kill and recv reply
|
// execveSyncKill will send kill and recv reply
|
||||||
func (m *Master) execveSyncKill() {
|
func (c *container) execveSyncKill() {
|
||||||
m.sendCmd(&Cmd{Cmd: cmdKill}, nil)
|
c.sendCmd(&cmd{Cmd: cmdKill}, nil)
|
||||||
m.recvReply()
|
c.recvReply()
|
||||||
}
|
}
|
||||||
74
container/protocol.go
Normal file
74
container/protocol.go
Normal file
@ -0,0 +1,74 @@
|
|||||||
|
package container
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
"syscall"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/criyle/go-sandbox/pkg/rlimit"
|
||||||
|
"github.com/criyle/go-sandbox/types"
|
||||||
|
)
|
||||||
|
|
||||||
|
// cmd is the control message send into container
|
||||||
|
type cmd struct {
|
||||||
|
Cmd string // type of the cmd
|
||||||
|
|
||||||
|
OpenCmd []OpenCmd // open argument
|
||||||
|
DeleteCmd *deleteCmd // delete argument
|
||||||
|
ExecCmd *execCmd // execve argument
|
||||||
|
ConfCmd *confCmd // to set configuration
|
||||||
|
}
|
||||||
|
|
||||||
|
// OpenCmd correspond to a single open syscall
|
||||||
|
type OpenCmd struct {
|
||||||
|
Path string
|
||||||
|
Flag int
|
||||||
|
Perm os.FileMode
|
||||||
|
}
|
||||||
|
|
||||||
|
// deleteCmd stores delete command
|
||||||
|
type deleteCmd struct {
|
||||||
|
Path string
|
||||||
|
}
|
||||||
|
|
||||||
|
// execCmd stores execve parameter
|
||||||
|
type execCmd struct {
|
||||||
|
Argv []string // execve argv
|
||||||
|
Env []string // execve env
|
||||||
|
RLimits []rlimit.RLimit // execve posix rlimit
|
||||||
|
FdExec bool // if use fexecve (fd[0] as exec)
|
||||||
|
}
|
||||||
|
|
||||||
|
// confCmd stores conf parameter
|
||||||
|
type confCmd struct {
|
||||||
|
Conf containerConfig
|
||||||
|
}
|
||||||
|
|
||||||
|
// ContainerConfig set the container config
|
||||||
|
type containerConfig struct {
|
||||||
|
Cred bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// reply is the reply message send back to controller
|
||||||
|
type reply struct {
|
||||||
|
Error *errorReply // nil if no error
|
||||||
|
ExecReply *execReply
|
||||||
|
}
|
||||||
|
|
||||||
|
// errorReply stores error returned back from container
|
||||||
|
type errorReply struct {
|
||||||
|
Msg string
|
||||||
|
Errno *syscall.Errno
|
||||||
|
}
|
||||||
|
|
||||||
|
// execReply stores execve result
|
||||||
|
type execReply struct {
|
||||||
|
ExitStatus int // waitpid exit status
|
||||||
|
Status types.Status // return status
|
||||||
|
Time time.Duration // waitpid user CPU (ns)
|
||||||
|
Memory types.Size // waitpid user memory (byte)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *errorReply) Error() string {
|
||||||
|
return e.Msg
|
||||||
|
}
|
||||||
@ -1,19 +1,30 @@
|
|||||||
package daemon
|
package container
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
"encoding/gob"
|
"encoding/gob"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"sync"
|
||||||
|
|
||||||
"github.com/criyle/go-sandbox/pkg/unixsocket"
|
"github.com/criyle/go-sandbox/pkg/unixsocket"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// 16k buffsize
|
||||||
|
const bufferSize = 16 << 10
|
||||||
|
|
||||||
|
var bufferPool = sync.Pool{
|
||||||
|
New: func() interface{} {
|
||||||
|
return make([]byte, bufferSize)
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
type socket unixsocket.Socket
|
type socket unixsocket.Socket
|
||||||
|
|
||||||
func (s *socket) RecvMsg(e interface{}) (*unixsocket.Msg, error) {
|
func (s *socket) RecvMsg(e interface{}) (*unixsocket.Msg, error) {
|
||||||
soc := (*unixsocket.Socket)(s)
|
soc := (*unixsocket.Socket)(s)
|
||||||
buff := GetBuffer()
|
|
||||||
defer PutBuffer(buff)
|
buff := bufferPool.Get().([]byte)
|
||||||
|
defer bufferPool.Put(buff)
|
||||||
|
|
||||||
n, msg, err := soc.RecvMsg(buff)
|
n, msg, err := soc.RecvMsg(buff)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@ -21,24 +32,25 @@ func (s *socket) RecvMsg(e interface{}) (*unixsocket.Msg, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if err := gob.NewDecoder(bytes.NewReader(buff[:n])).Decode(e); err != nil {
|
if err := gob.NewDecoder(bytes.NewReader(buff[:n])).Decode(e); err != nil {
|
||||||
return nil, fmt.Errorf("RecvMsg: failed to decode %v", err)
|
return nil, fmt.Errorf("RecvMsg: failed to decode %w", err)
|
||||||
}
|
}
|
||||||
return msg, nil
|
return msg, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *socket) SendMsg(e interface{}, msg *unixsocket.Msg) error {
|
func (s *socket) SendMsg(e interface{}, msg *unixsocket.Msg) error {
|
||||||
soc := (*unixsocket.Socket)(s)
|
soc := (*unixsocket.Socket)(s)
|
||||||
buf := GetBuffer()
|
|
||||||
defer PutBuffer(buf)
|
buf := bufferPool.Get().([]byte)
|
||||||
|
defer bufferPool.Put(buf)
|
||||||
|
|
||||||
// use buf pool to reduce allocation
|
// use buf pool to reduce allocation
|
||||||
buff := bytes.NewBuffer(buf[:0])
|
buff := bytes.NewBuffer(buf[:0])
|
||||||
if err := gob.NewEncoder(buff).Encode(e); err != nil {
|
if err := gob.NewEncoder(buff).Encode(e); err != nil {
|
||||||
return fmt.Errorf("SendMsg: failed to encode %v", err)
|
return fmt.Errorf("SendMsg: failed to encode %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := soc.SendMsg(buff.Bytes(), msg); err != nil {
|
if err := soc.SendMsg(buff.Bytes(), msg); err != nil {
|
||||||
return fmt.Errorf("SendMsg: failed to SendMsg %v", err)
|
return fmt.Errorf("SendMsg: failed to SendMsg %w", err)
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
@ -1,4 +1,4 @@
|
|||||||
package daemon
|
package container
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"os"
|
"os"
|
||||||
@ -1,176 +0,0 @@
|
|||||||
package daemon
|
|
||||||
|
|
||||||
import (
|
|
||||||
"fmt"
|
|
||||||
"os"
|
|
||||||
"runtime"
|
|
||||||
"syscall"
|
|
||||||
|
|
||||||
"github.com/criyle/go-sandbox/pkg/unixsocket"
|
|
||||||
)
|
|
||||||
|
|
||||||
type containerServer struct {
|
|
||||||
socket *unixsocket.Socket
|
|
||||||
containerConfig
|
|
||||||
}
|
|
||||||
|
|
||||||
// ContainerConfig set the container config
|
|
||||||
type containerConfig struct {
|
|
||||||
Cred bool
|
|
||||||
}
|
|
||||||
|
|
||||||
// Init is called for container init process
|
|
||||||
// it will check if pid == 1, otherwise it is noop
|
|
||||||
// Init will do infinite loop on socket commands,
|
|
||||||
// and exits when at socket close, use it in init function
|
|
||||||
func Init() (err error) {
|
|
||||||
// noop if self is not container init process
|
|
||||||
// Notice: docker init is also 1, additional check for args[1] == init
|
|
||||||
if os.Getpid() != 1 || len(os.Args) != 2 || os.Args[1] != initArg {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// exit process (with whole container) upon exit this function
|
|
||||||
// possible reason:
|
|
||||||
// 1. socket broken (parent exit)
|
|
||||||
// 2. panic
|
|
||||||
// 3. undefined cmd (possible race condition)
|
|
||||||
defer func() {
|
|
||||||
if err2 := recover(); err2 != nil {
|
|
||||||
fmt.Fprintf(os.Stderr, "container_panic: %v\n", err)
|
|
||||||
os.Exit(1)
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
fmt.Fprintf(os.Stderr, "container_exit: %v\n", err)
|
|
||||||
os.Exit(1)
|
|
||||||
}
|
|
||||||
fmt.Fprintf(os.Stderr, "container_exit\n")
|
|
||||||
os.Exit(0)
|
|
||||||
}()
|
|
||||||
|
|
||||||
// limit container resource usage
|
|
||||||
runtime.GOMAXPROCS(containerMaxProc)
|
|
||||||
|
|
||||||
// new_master shared the socket at fd 3 (marked close_exec)
|
|
||||||
const defaultFd = 3
|
|
||||||
soc, err := unixsocket.NewSocket(defaultFd)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("container_init: failed to new socket(%v)", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// serve forever
|
|
||||||
cs := &containerServer{socket: soc}
|
|
||||||
return cs.serve()
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *containerServer) serve() error {
|
|
||||||
for {
|
|
||||||
cmd, msg, err := c.recvCmd()
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("serve: %v", err)
|
|
||||||
}
|
|
||||||
if err := c.handleCmd(cmd, msg); err != nil {
|
|
||||||
return fmt.Errorf("serve: failed to execute cmd %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *containerServer) handleCmd(cmd *Cmd, msg *unixsocket.Msg) error {
|
|
||||||
switch cmd.Cmd {
|
|
||||||
case cmdPing:
|
|
||||||
return c.handlePing()
|
|
||||||
|
|
||||||
case cmdConf:
|
|
||||||
return c.handleConf(cmd.ConfCmd)
|
|
||||||
|
|
||||||
case cmdOpen:
|
|
||||||
return c.handleOpen(cmd.OpenCmd)
|
|
||||||
|
|
||||||
case cmdDelete:
|
|
||||||
return c.handleDelete(cmd.DeleteCmd)
|
|
||||||
|
|
||||||
case cmdReset:
|
|
||||||
return c.handleReset()
|
|
||||||
|
|
||||||
case cmdExecve:
|
|
||||||
return c.handleExecve(cmd.ExecCmd, msg)
|
|
||||||
}
|
|
||||||
return fmt.Errorf("unknown command: %v", cmd.Cmd)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *containerServer) handlePing() error {
|
|
||||||
return c.sendReply(&Reply{}, nil)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *containerServer) handleConf(conf *ConfCmd) error {
|
|
||||||
if conf != nil {
|
|
||||||
c.containerConfig = conf.Conf
|
|
||||||
}
|
|
||||||
return c.sendReply(&Reply{}, nil)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *containerServer) handleOpen(open []OpenCmd) error {
|
|
||||||
if len(open) == 0 {
|
|
||||||
return c.sendErrorReply("open: no open parameter received")
|
|
||||||
}
|
|
||||||
|
|
||||||
// open files
|
|
||||||
fds := make([]int, 0, len(open))
|
|
||||||
for _, o := range open {
|
|
||||||
outFile, err := os.OpenFile(o.Path, o.Flag, o.Perm)
|
|
||||||
if err != nil {
|
|
||||||
return c.sendErrorReply("open: %v", err)
|
|
||||||
}
|
|
||||||
defer outFile.Close()
|
|
||||||
fds = append(fds, int(outFile.Fd()))
|
|
||||||
}
|
|
||||||
|
|
||||||
return c.sendReply(&Reply{}, &unixsocket.Msg{Fds: fds})
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *containerServer) handleDelete(delete *DeleteCmd) error {
|
|
||||||
if delete == nil {
|
|
||||||
return c.sendErrorReply("delete: no parameter provided")
|
|
||||||
}
|
|
||||||
if err := os.Remove(delete.Path); err != nil {
|
|
||||||
return c.sendErrorReply("delete: %v", err)
|
|
||||||
}
|
|
||||||
return c.sendReply(&Reply{}, nil)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *containerServer) handleReset() error {
|
|
||||||
if err := removeContents("/tmp"); err != nil {
|
|
||||||
return c.sendErrorReply("reset: /tmp %v", err)
|
|
||||||
}
|
|
||||||
if err := removeContents("/w"); err != nil {
|
|
||||||
return c.sendErrorReply("reset: /w %v", err)
|
|
||||||
}
|
|
||||||
return c.sendReply(&Reply{}, nil)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *containerServer) recvCmd() (*Cmd, *unixsocket.Msg, error) {
|
|
||||||
cmd := new(Cmd)
|
|
||||||
msg, err := (*socket)(c.socket).RecvMsg(cmd)
|
|
||||||
if err != nil {
|
|
||||||
return nil, nil, err
|
|
||||||
}
|
|
||||||
return cmd, msg, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *containerServer) sendReply(reply *Reply, msg *unixsocket.Msg) error {
|
|
||||||
return (*socket)(c.socket).SendMsg(reply, msg)
|
|
||||||
}
|
|
||||||
|
|
||||||
// sendErrorReply sends error reply
|
|
||||||
func (c *containerServer) sendErrorReply(ft string, v ...interface{}) error {
|
|
||||||
errorReply := &ErrorReply{
|
|
||||||
Msg: fmt.Sprintf(ft, v...),
|
|
||||||
}
|
|
||||||
// store errno
|
|
||||||
if len(v) == 1 {
|
|
||||||
if errno, ok := v[0].(syscall.Errno); ok {
|
|
||||||
errorReply.Errno = &errno
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return c.sendReply(&Reply{Error: errorReply}, nil)
|
|
||||||
}
|
|
||||||
102
daemon/deamon.go
102
daemon/deamon.go
@ -1,102 +0,0 @@
|
|||||||
// Package daemon provides pre-forked container to reduce the container
|
|
||||||
// create / destroy costs (about 160ms). It creates daemon within unshared
|
|
||||||
// container and communicate with original process using unix socket with
|
|
||||||
// oob for fd / pid and struct are encoded in gob format.
|
|
||||||
package daemon
|
|
||||||
|
|
||||||
/*
|
|
||||||
Container / Master Communication Protocol (single thread):
|
|
||||||
|
|
||||||
- ping (alive check):
|
|
||||||
- reply: pong
|
|
||||||
- conf (set configuration):
|
|
||||||
- reply pong
|
|
||||||
- open (open files in given mode inside container):
|
|
||||||
- send: []OpenCmd
|
|
||||||
- reply: "success", file fds / "error"
|
|
||||||
- delete (unlink file / rmdir dir inside container):
|
|
||||||
- send: path
|
|
||||||
- reply: "finished" / "error"
|
|
||||||
- reset (clean up container for later use (clear workdir / tmp)):
|
|
||||||
- send:
|
|
||||||
- reply: "success"
|
|
||||||
- execve: (execute file inside container):
|
|
||||||
- send: argv, env, rLimits, fds
|
|
||||||
- reply:
|
|
||||||
- success: "success", pid
|
|
||||||
- failed: "failed"
|
|
||||||
- send (success): "init_finished" (as cmd)
|
|
||||||
- reply: "finished" / send: "kill" (as cmd)
|
|
||||||
- send: "kill" (as cmd) / reply: "finished"
|
|
||||||
- reply:
|
|
||||||
|
|
||||||
Any socket related error will cause the daemon exit (with all process inside container)
|
|
||||||
*/
|
|
||||||
|
|
||||||
import (
|
|
||||||
"os"
|
|
||||||
"syscall"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/criyle/go-sandbox/pkg/rlimit"
|
|
||||||
"github.com/criyle/go-sandbox/types"
|
|
||||||
)
|
|
||||||
|
|
||||||
// Cmd is the control message send into daemon
|
|
||||||
type Cmd struct {
|
|
||||||
Cmd string // type of the cmd
|
|
||||||
|
|
||||||
OpenCmd []OpenCmd // open argument
|
|
||||||
DeleteCmd *DeleteCmd // delete argument
|
|
||||||
ExecCmd *ExecCmd // execve argument
|
|
||||||
ConfCmd *ConfCmd // to set configuration
|
|
||||||
}
|
|
||||||
|
|
||||||
// OpenCmd correspond to a single open syscall
|
|
||||||
type OpenCmd struct {
|
|
||||||
Path string
|
|
||||||
Flag int
|
|
||||||
Perm os.FileMode
|
|
||||||
}
|
|
||||||
|
|
||||||
// DeleteCmd stores delete command
|
|
||||||
type DeleteCmd struct {
|
|
||||||
Path string
|
|
||||||
}
|
|
||||||
|
|
||||||
// ExecCmd stores execve parameter
|
|
||||||
type ExecCmd struct {
|
|
||||||
Argv []string // execve argv
|
|
||||||
Env []string // execve env
|
|
||||||
RLimits []rlimit.RLimit // execve posix rlimit
|
|
||||||
FdExec bool // if use fexecve (fd[0] as exec)
|
|
||||||
}
|
|
||||||
|
|
||||||
// ConfCmd stores conf parameter
|
|
||||||
type ConfCmd struct {
|
|
||||||
Conf containerConfig
|
|
||||||
}
|
|
||||||
|
|
||||||
// Reply is the reply message send back to controller
|
|
||||||
type Reply struct {
|
|
||||||
Error *ErrorReply // nil if no error
|
|
||||||
ExecReply *ExecReply
|
|
||||||
}
|
|
||||||
|
|
||||||
// ErrorReply stores error returned back from container
|
|
||||||
type ErrorReply struct {
|
|
||||||
Msg string
|
|
||||||
Errno *syscall.Errno
|
|
||||||
}
|
|
||||||
|
|
||||||
// ExecReply stores execve result
|
|
||||||
type ExecReply struct {
|
|
||||||
ExitStatus int // waitpid exit status
|
|
||||||
Status types.Status // return status
|
|
||||||
Time time.Duration // waitpid user CPU (ns)
|
|
||||||
Memory types.Size // waitpid user memory (byte)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (e *ErrorReply) Error() string {
|
|
||||||
return e.Msg
|
|
||||||
}
|
|
||||||
@ -1,24 +0,0 @@
|
|||||||
package daemon
|
|
||||||
|
|
||||||
import (
|
|
||||||
"sync"
|
|
||||||
)
|
|
||||||
|
|
||||||
// 16k buffsize
|
|
||||||
const bufferSize = 16384
|
|
||||||
|
|
||||||
var bufferPool = sync.Pool{
|
|
||||||
New: func() interface{} {
|
|
||||||
return make([]byte, bufferSize)
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
// GetBuffer get buffer from pool
|
|
||||||
func GetBuffer() []byte {
|
|
||||||
return bufferPool.Get().([]byte)
|
|
||||||
}
|
|
||||||
|
|
||||||
// PutBuffer return buffer to the pool
|
|
||||||
func PutBuffer(x []byte) {
|
|
||||||
bufferPool.Put(x)
|
|
||||||
}
|
|
||||||
Loading…
Reference in New Issue
Block a user