mirror of
https://github.com/criyle/go-sandbox.git
synced 2025-09-26 23:19:11 +08:00
379 lines
8.6 KiB
Go
379 lines
8.6 KiB
Go
package container
|
|
|
|
import (
|
|
"errors"
|
|
"fmt"
|
|
"os"
|
|
"os/exec"
|
|
"os/signal"
|
|
"path/filepath"
|
|
"runtime"
|
|
"strconv"
|
|
"sync"
|
|
"syscall"
|
|
|
|
"github.com/criyle/go-sandbox/pkg/unixsocket"
|
|
)
|
|
|
|
type containerServer struct {
|
|
socket *socket
|
|
containerConfig
|
|
defaultEnv []string
|
|
|
|
done chan struct{}
|
|
err error
|
|
doneOnce sync.Once
|
|
|
|
recvCh chan recvCmd
|
|
sendCh chan sendReply
|
|
|
|
waitPid chan int
|
|
waitPidResult chan waitPidResult
|
|
|
|
waitAll chan struct{}
|
|
waitAllDone chan struct{}
|
|
}
|
|
|
|
type recvCmd struct {
|
|
Cmd cmd
|
|
Msg unixsocket.Msg
|
|
}
|
|
|
|
type sendReply struct {
|
|
Reply reply
|
|
Msg unixsocket.Msg
|
|
FileToClose []*os.File
|
|
}
|
|
|
|
type waitPidResult struct {
|
|
WaitStatus syscall.WaitStatus
|
|
Rusage syscall.Rusage
|
|
Err error
|
|
}
|
|
|
|
// Init is called for container init process
|
|
// it will check if pid == 1, otherwise it is noop
|
|
// Init will do infinite loop on socket commands,
|
|
// and exits when at socket close, use it in init function
|
|
func Init() (err error) {
|
|
// noop if self is not container init process
|
|
// Notice: docker init is also 1, additional check for args[1] == init
|
|
if os.Getpid() != 1 || len(os.Args) < 2 || os.Args[1] != initArg {
|
|
return nil
|
|
}
|
|
|
|
// exit process (with whole container) upon exit this function
|
|
// possible reason:
|
|
// 1. socket broken (parent exit)
|
|
// 2. panic
|
|
// 3. undefined cmd (possible race condition)
|
|
defer func() {
|
|
if err := recover(); err != nil {
|
|
fmt.Fprintf(os.Stderr, "container_exit: panic: %v\n", err)
|
|
os.Exit(1)
|
|
}
|
|
if err != nil {
|
|
fmt.Fprintf(os.Stderr, "container_exit: %v\n", err)
|
|
os.Exit(1)
|
|
}
|
|
fmt.Fprintf(os.Stderr, "container_exit\n")
|
|
os.Exit(0)
|
|
}()
|
|
|
|
// ignore any signal that kills the init process
|
|
ignoreSignals()
|
|
|
|
// limit container resource usage
|
|
runtime.GOMAXPROCS(containerMaxProc)
|
|
|
|
// ensure there's no fd leak to child process (e.g. VSCode leaks ptmx fd)
|
|
if err := closeOnExecAllFds(); err != nil {
|
|
return fmt.Errorf("container_init: failed to close on exec all fds: %w", err)
|
|
}
|
|
|
|
// new_container environment shared the socket at fd 3 (marked close_exec)
|
|
const defaultFd = 3
|
|
soc, err := unixsocket.NewSocket(defaultFd)
|
|
if err != nil {
|
|
return fmt.Errorf("container_init: failed to create new socket: %w", err)
|
|
}
|
|
|
|
// serve forever
|
|
cs := &containerServer{
|
|
socket: newSocket(soc),
|
|
done: make(chan struct{}),
|
|
sendCh: make(chan sendReply, 1),
|
|
recvCh: make(chan recvCmd, 1),
|
|
waitPid: make(chan int),
|
|
waitAll: make(chan struct{}),
|
|
waitPidResult: make(chan waitPidResult, 1),
|
|
waitAllDone: make(chan struct{}, 1),
|
|
}
|
|
go cs.sendLoop()
|
|
go cs.recvLoop()
|
|
go cs.waitLoop()
|
|
|
|
return cs.serve()
|
|
}
|
|
|
|
func (c *containerServer) sendLoop() {
|
|
for {
|
|
select {
|
|
case <-c.done:
|
|
return
|
|
|
|
case rep, ok := <-c.sendCh:
|
|
if !ok {
|
|
return
|
|
}
|
|
err := c.socket.SendMsg(rep.Reply, rep.Msg)
|
|
for _, f := range rep.FileToClose {
|
|
f.Close()
|
|
}
|
|
if err != nil {
|
|
c.socketError(err)
|
|
return
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func (c *containerServer) recvLoop() {
|
|
for {
|
|
var cmd cmd
|
|
msg, err := c.socket.RecvMsg(&cmd)
|
|
if err != nil {
|
|
c.socketError(err)
|
|
return
|
|
}
|
|
c.recvCh <- recvCmd{
|
|
Cmd: cmd,
|
|
Msg: msg,
|
|
}
|
|
}
|
|
}
|
|
|
|
func (c *containerServer) socketError(err error) {
|
|
c.doneOnce.Do(func() {
|
|
c.err = err
|
|
close(c.done)
|
|
})
|
|
}
|
|
|
|
func (c *containerServer) waitLoop() {
|
|
for {
|
|
select {
|
|
case pid := <-c.waitPid:
|
|
var waitStatus syscall.WaitStatus
|
|
var rusage syscall.Rusage
|
|
|
|
_, err := syscall.Wait4(pid, &waitStatus, 0, &rusage)
|
|
for err == syscall.EINTR {
|
|
_, err = syscall.Wait4(pid, &waitStatus, 0, &rusage)
|
|
}
|
|
if err != nil {
|
|
c.waitPidResult <- waitPidResult{
|
|
Err: err,
|
|
}
|
|
continue
|
|
}
|
|
c.waitPidResult <- waitPidResult{
|
|
WaitStatus: waitStatus,
|
|
Rusage: rusage,
|
|
}
|
|
|
|
case <-c.waitAll:
|
|
for {
|
|
if _, err := syscall.Wait4(-1, nil, syscall.WNOHANG, nil); err != nil && err != syscall.EINTR {
|
|
break
|
|
}
|
|
}
|
|
c.waitAllDone <- struct{}{}
|
|
}
|
|
}
|
|
}
|
|
|
|
func (c *containerServer) serve() error {
|
|
for {
|
|
cmd, msg, err := c.recvCmd()
|
|
if err != nil {
|
|
return fmt.Errorf("serve: recvCmd: %w", err)
|
|
}
|
|
if err := c.handleCmd(cmd, msg); err != nil {
|
|
return fmt.Errorf("serve: failed to execute cmd: %w", err)
|
|
}
|
|
}
|
|
}
|
|
|
|
func (c *containerServer) handleCmd(cmd cmd, msg unixsocket.Msg) error {
|
|
switch cmd.Cmd {
|
|
case cmdPing:
|
|
return c.handlePing()
|
|
|
|
case cmdConf:
|
|
return c.handleConf(cmd.ConfCmd)
|
|
|
|
case cmdOpen:
|
|
return c.handleOpen(cmd.OpenCmd)
|
|
|
|
case cmdDelete:
|
|
return c.handleDelete(cmd.DeleteCmd)
|
|
|
|
case cmdReset:
|
|
return c.handleReset()
|
|
|
|
case cmdExecve:
|
|
return c.handleExecve(cmd.ExecCmd, msg)
|
|
}
|
|
return fmt.Errorf("unknown command: %v", cmd.Cmd)
|
|
}
|
|
|
|
func initContainer(c containerConfig) error {
|
|
if err := initFileSystem(c); err != nil {
|
|
return err
|
|
}
|
|
if err := syscall.Setdomainname([]byte(c.DomainName)); err != nil {
|
|
return err
|
|
}
|
|
if err := syscall.Sethostname([]byte(c.HostName)); err != nil {
|
|
return err
|
|
}
|
|
if err := os.Chdir(c.WorkDir); err != nil {
|
|
return err
|
|
}
|
|
if len(c.InitCommand) > 0 {
|
|
cm := exec.Command(c.InitCommand[0], c.InitCommand[1:]...)
|
|
if output, err := cm.CombinedOutput(); err != nil {
|
|
os.Stderr.Write(output)
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func initFileSystem(c containerConfig) error {
|
|
// mount tmpfs as root
|
|
const tmpfs = "tmpfs"
|
|
if err := syscall.Mount(tmpfs, c.ContainerRoot, tmpfs, 0, ""); err != nil {
|
|
return fmt.Errorf("init_fs: mount /: %w", err)
|
|
}
|
|
// change dir to container root
|
|
if err := syscall.Chdir(c.ContainerRoot); err != nil {
|
|
return fmt.Errorf("init_fs: chdir: %w", err)
|
|
}
|
|
// performing mounts
|
|
for _, m := range c.Mounts {
|
|
if err := m.Mount(); err != nil {
|
|
return fmt.Errorf("init_fs: mount %v: %w", m, err)
|
|
}
|
|
}
|
|
// pivot root
|
|
const oldRoot = "old_root"
|
|
if err := os.Mkdir(oldRoot, 0755); err != nil {
|
|
return fmt.Errorf("init_fs: mkdir old_root: %w", err)
|
|
}
|
|
if err := syscall.PivotRoot(c.ContainerRoot, oldRoot); err != nil {
|
|
return fmt.Errorf("init_fs: pivot_root(%s, %s): %w", c.ContainerRoot, oldRoot, err)
|
|
}
|
|
if err := syscall.Unmount(oldRoot, syscall.MNT_DETACH); err != nil {
|
|
return fmt.Errorf("init_fs: unmount old_root: %w", err)
|
|
}
|
|
if err := os.Remove(oldRoot); err != nil {
|
|
return fmt.Errorf("init_fs: unlink old_root: %w", err)
|
|
}
|
|
// create symlinks
|
|
for _, l := range c.SymbolicLinks {
|
|
// ensure dir exists
|
|
dir := filepath.Dir(l.LinkPath)
|
|
if err := os.MkdirAll(dir, 0755); err != nil {
|
|
return fmt.Errorf("init_fs: mkdir_all(%s): %w", dir, err)
|
|
}
|
|
if err := os.Symlink(l.Target, l.LinkPath); err != nil {
|
|
return fmt.Errorf("init_fs: symlink: %w", err)
|
|
}
|
|
}
|
|
// mask paths
|
|
for _, p := range c.MaskPaths {
|
|
if err := maskPath(p); err != nil {
|
|
return fmt.Errorf("init_fs: mask path: %w", err)
|
|
}
|
|
}
|
|
// readonly root
|
|
const remountFlag = syscall.MS_BIND | syscall.MS_REMOUNT | syscall.MS_RDONLY | syscall.MS_NOATIME | syscall.MS_NOSUID
|
|
if err := syscall.Mount(tmpfs, "/", tmpfs, remountFlag, ""); err != nil {
|
|
return fmt.Errorf("init_fs: readonly remount /: %w", err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (c *containerServer) recvCmd() (cmd, unixsocket.Msg, error) {
|
|
select {
|
|
case <-c.done:
|
|
return cmd{}, unixsocket.Msg{}, c.err
|
|
|
|
case recv := <-c.recvCh:
|
|
return recv.Cmd, recv.Msg, nil
|
|
}
|
|
}
|
|
|
|
func (c *containerServer) sendReplyFiles(rep reply, msg unixsocket.Msg, fileToClose []*os.File) error {
|
|
select {
|
|
case <-c.done:
|
|
return c.err
|
|
|
|
case c.sendCh <- sendReply{Reply: rep, Msg: msg, FileToClose: fileToClose}:
|
|
return nil
|
|
}
|
|
}
|
|
|
|
func (c *containerServer) sendReply(rep reply, msg unixsocket.Msg) error {
|
|
return c.sendReplyFiles(rep, msg, nil)
|
|
}
|
|
|
|
// sendErrorReply sends error reply
|
|
func (c *containerServer) sendErrorReply(ft string, v ...interface{}) error {
|
|
errRep := &errorReply{
|
|
Msg: fmt.Sprintf(ft, v...),
|
|
}
|
|
// store errno
|
|
if len(v) == 1 {
|
|
if errno, ok := v[0].(syscall.Errno); ok {
|
|
errRep.Errno = &errno
|
|
}
|
|
}
|
|
return c.sendReply(reply{Error: errRep}, unixsocket.Msg{})
|
|
}
|
|
|
|
func closeOnExecAllFds() error {
|
|
// get all fd from /proc/self/fd
|
|
const fdPath = "/proc/self/fd"
|
|
fds, err := os.ReadDir(fdPath)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
for _, f := range fds {
|
|
fd, err := strconv.Atoi(f.Name())
|
|
if err != nil {
|
|
return err
|
|
}
|
|
syscall.CloseOnExec(fd)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func maskPath(path string) error {
|
|
// bind mount /dev/null if it is file
|
|
if err := syscall.Mount("/dev/null", path, "", syscall.MS_BIND, ""); err != nil && !errors.Is(err, os.ErrNotExist) {
|
|
if errors.Is(err, syscall.ENOTDIR) {
|
|
// otherwise, mount tmpfs to mask it
|
|
return syscall.Mount("tmpfs", path, "tmpfs", syscall.MS_RDONLY, "")
|
|
}
|
|
return fmt.Errorf("mask path: %w", err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func ignoreSignals() {
|
|
signal.Ignore(signalToIgnore...)
|
|
}
|