Add ability unshare cgroup after sync

This commit is contained in:
criyle 2020-03-20 18:50:31 -04:00
parent cfa40fa36a
commit 5f133175fd
9 changed files with 158 additions and 11 deletions

View File

@ -8,11 +8,14 @@ import (
"io/ioutil"
"os"
"os/signal"
"sync/atomic"
"syscall"
"time"
"github.com/criyle/go-sandbox/config"
"github.com/criyle/go-sandbox/container"
"github.com/criyle/go-sandbox/pkg/cgroup"
"github.com/criyle/go-sandbox/pkg/forkexec"
"github.com/criyle/go-sandbox/pkg/memfd"
"github.com/criyle/go-sandbox/pkg/mount"
"github.com/criyle/go-sandbox/pkg/rlimit"
@ -30,7 +33,7 @@ const (
var (
addReadable, addWritable, addRawReadable, addRawWritable arrayFlags
allowProc, unsafe, showDetails, useCGroup, memfile bool
allowProc, unsafe, showDetails, useCGroup, memfile, cred bool
timeLimit, realTimeLimit, memoryLimit, outputLimit, stackLimit uint64
inputFileName, outputFileName, errorFileName, workPath, runt string
@ -72,6 +75,7 @@ func main() {
flag.BoolVar(&useCGroup, "cgroup", false, "Use cgroup to colloct resource usage")
flag.BoolVar(&memfile, "memfd", false, "Use memfd as exec file")
flag.StringVar(&runt, "runner", "ptrace", "Runner for the program (ptrace, ns, container)")
flag.BoolVar(&cred, "cred", false, "Generate credential for containers (uid=10000)")
flag.Parse()
args = flag.Args()
@ -155,6 +159,35 @@ func start() (*runner.Result, error) {
addWrite := filehandler.GetExtraSet(addWritable, addRawWritable)
args, allow, trace, h := config.GetConf(pType, workPath, args, addRead, addWrite, allowProc)
mb := mount.NewBuilder().
// basic exec and lib
WithBind("/bin", "bin", true).
WithBind("/lib", "lib", true).
WithBind("/lib64", "lib64", true).
WithBind("/usr", "usr", true).
// java wants /proc/self/exe as it need relative path for lib
// however, /proc gives interface like /proc/1/fd/3 ..
// it is fine since open that file will be a EPERM
// changing the fs uid and gid would be a good idea
WithProc().
// some compiler have multiple version
WithBind("/etc/alternatives", "etc/alternatives", true).
// fpc wants /etc/fpc.cfg
WithBind("/etc/fpc.cfg", "etc/fpc.cfg", true).
// go wants /dev/null
WithBind("/dev/null", "dev/null", false).
// ghc wants /var/lib/ghc
WithBind("/var/lib/ghc", "var/lib/ghc", true).
// work dir
WithTmpfs("w", "size=8m,nr_inodes=4k").
// tmp dir
WithTmpfs("tmp", "size=8m,nr_inodes=4k")
mt, err := mb.Build(true)
if err != nil {
return nil, err
}
if useCGroup {
b, err := cgroup.NewBuilder("runprog").WithCPUAcct().WithMemory().WithPids().FilterByEnv()
if err != nil {
@ -237,8 +270,16 @@ func start() (*runner.Result, error) {
}
defer os.RemoveAll(root)
var credG container.CredGenerator
if cred {
credG = newCredGen()
}
b := container.Builder{
Root: root,
Root: root,
Mounts: mt,
CredGenerator: credG,
CloneFlags: forkexec.UnshareFlags,
}
m, err := b.Build()
@ -275,10 +316,6 @@ func start() (*runner.Result, error) {
return nil, fmt.Errorf("cannot make temp root for new namespace")
}
defer os.RemoveAll(root)
mounts, err := mount.NewDefaultBuilder().WithBind(root, "w", true).Build(true)
if err != nil {
return nil, fmt.Errorf("cannot make rootfs mounts")
}
r = &unshare.Runner{
Args: args,
Env: []string{pathEnv},
@ -289,7 +326,7 @@ func start() (*runner.Result, error) {
Limit: limit,
Seccomp: filter,
Root: root,
Mounts: mounts,
Mounts: mt,
ShowDetails: showDetails,
SyncFunc: syncFunc,
HostName: "run_program",
@ -414,3 +451,19 @@ func getStatus(s runner.Status) int {
return int(StatusFatal)
}
}
type credGen struct {
cur uint32
}
func newCredGen() *credGen {
return &credGen{cur: 10000}
}
func (c *credGen) Get() syscall.Credential {
n := atomic.AddUint32(&c.cur, 1)
return syscall.Credential{
Uid: n,
Gid: n,
}
}

View File

@ -76,6 +76,8 @@ func (c *containerServer) handleExecve(cmd *execCmd, msg *unixsocket.Msg) error
DropCaps: true,
SyncFunc: syncFunc,
Credential: cred,
UnshareCgroupAfterSync: true,
}
// starts the runner, error is handled same as wait4 to make communication equal
pid, err := r.Start()

View File

@ -34,6 +34,9 @@ type Builder struct {
// CredGenerator defines a credential generator used to create new container
CredGenerator CredGenerator
// Clone flags defines unshare clone flag to create container
CloneFlags uintptr
}
// CredGenerator generates uid / gid credential used by container
@ -130,13 +133,20 @@ func (b *Builder) Build() (Environment, error) {
uidMap, gidMap = getIDMapping(&cred)
}
var cloneFlag uintptr
if b.CloneFlags == 0 {
cloneFlag = forkexec.UnshareFlags
} else {
cloneFlag = b.CloneFlags & forkexec.UnshareFlags
}
r := &forkexec.Runner{
Args: []string{os.Args[0], initArg},
Env: []string{PathEnv},
ExecFile: execFile.Fd(),
Files: files,
WorkDir: containerWD,
CloneFlags: forkexec.UnshareFlags,
CloneFlags: cloneFlag,
Mounts: mounts,
HostName: containerName,
DomainName: containerName,

View File

@ -49,3 +49,17 @@ var (
Inheritable: 0,
}
)
const (
_SECURE_NOROOT = 1 << iota
_SECURE_NOROOT_LOCKED
_SECURE_NO_SETUID_FIXUP
_SECURE_NO_SETUID_FIXUP_LOCKED
_SECURE_KEEP_CAPS
_SECURE_KEEP_CAPS_LOCKED
_SECURE_NO_CAP_AMBIENT_RAISE
_SECURE_NO_CAP_AMBIENT_RAISE_LOCKED
)

View File

@ -73,6 +73,13 @@ func forkAndExecInChild(r *Runner, argv0 *byte, argv, env []*byte, workdir, host
goto childerror
}
// keep capabilities through set_uid / set_gid calls (make sure we can use unshare cgroup), later dropped
_, _, err1 = syscall.RawSyscall(syscall.SYS_PRCTL, syscall.PR_SET_SECUREBITS,
_SECURE_KEEP_CAPS_LOCKED|_SECURE_NO_SETUID_FIXUP|_SECURE_NO_SETUID_FIXUP_LOCKED, 0)
if err1 != 0 {
goto childerror
}
// set the credential for the child process(exec_linux.go)
if cred := r.Credential; cred != nil {
ngroups := uintptr(len(cred.Groups))
@ -291,7 +298,13 @@ func forkAndExecInChild(r *Runner, argv0 *byte, argv, env []*byte, workdir, host
}
// Drop all capabilities
if r.DropCaps {
if (r.Credential != nil || r.DropCaps) && !r.UnshareCgroupAfterSync {
// make sure the children have no privilege at all
_, _, err1 = syscall.RawSyscall(syscall.SYS_PRCTL, syscall.PR_SET_SECUREBITS,
_SECURE_KEEP_CAPS_LOCKED|_SECURE_NO_SETUID_FIXUP|_SECURE_NO_SETUID_FIXUP_LOCKED|_SECURE_NOROOT|_SECURE_NOROOT_LOCKED, 0)
if err1 != 0 {
goto childerror
}
_, _, err1 = syscall.RawSyscall(syscall.SYS_CAPSET, uintptr(unsafe.Pointer(&dropCapHeader)), uintptr(unsafe.Pointer(&dropCapData)), 0)
if err1 != 0 {
goto childerror
@ -311,6 +324,26 @@ func forkAndExecInChild(r *Runner, argv0 *byte, argv, env []*byte, workdir, host
goto childerror
}
// unshare cgroup namespace
if r.UnshareCgroupAfterSync {
r1, _, err1 = syscall.RawSyscall(syscall.SYS_UNSHARE, uintptr(unix.CLONE_NEWCGROUP), 0, 0)
if err1 != 0 {
goto childerror
}
if r.DropCaps || r.Credential != nil {
// make sure the children have no privilege at all
_, _, err1 = syscall.RawSyscall(syscall.SYS_PRCTL, syscall.PR_SET_SECUREBITS,
_SECURE_KEEP_CAPS_LOCKED|_SECURE_NO_SETUID_FIXUP|_SECURE_NO_SETUID_FIXUP_LOCKED|_SECURE_NOROOT|_SECURE_NOROOT_LOCKED, 0)
if err1 != 0 {
goto childerror
}
_, _, err1 = syscall.RawSyscall(syscall.SYS_CAPSET, uintptr(unsafe.Pointer(&dropCapHeader)), uintptr(unsafe.Pointer(&dropCapData)), 0)
if err1 != 0 {
goto childerror
}
}
}
_, _, err1 = syscall.RawSyscall(syscall.SYS_PTRACE, uintptr(syscall.PTRACE_TRACEME), 0, 0)
if err1 != 0 {
goto childerror
@ -329,7 +362,7 @@ func forkAndExecInChild(r *Runner, argv0 *byte, argv, env []*byte, workdir, host
}
// Load seccomp, stop and wait for tracer
if r.Seccomp != nil {
if r.Seccomp != nil && (!r.UnshareCgroupAfterSync || r.Ptrace) {
// If execve is seccomp trapped, then tracee stop is necessary
// otherwise execve will fail due to ENOSYS
// Do getpid and kill to send SYS_KILL to self
@ -354,6 +387,33 @@ func forkAndExecInChild(r *Runner, argv0 *byte, argv, env []*byte, workdir, host
if r1 == 0 || err1 != 0 {
goto childerror
}
// unshare cgroup namespace
if r.UnshareCgroupAfterSync {
r1, _, err1 = syscall.RawSyscall(syscall.SYS_UNSHARE, uintptr(unix.CLONE_NEWCGROUP), 0, 0)
if err1 != 0 {
goto childerror
}
if r.DropCaps || r.Credential != nil {
// make sure the children have no privilege at all
_, _, err1 = syscall.RawSyscall(syscall.SYS_PRCTL, syscall.PR_SET_SECUREBITS,
_SECURE_KEEP_CAPS_LOCKED|_SECURE_NO_SETUID_FIXUP|_SECURE_NO_SETUID_FIXUP_LOCKED|_SECURE_NOROOT|_SECURE_NOROOT_LOCKED, 0)
if err1 != 0 {
goto childerror
}
_, _, err1 = syscall.RawSyscall(syscall.SYS_CAPSET, uintptr(unsafe.Pointer(&dropCapHeader)), uintptr(unsafe.Pointer(&dropCapData)), 0)
if err1 != 0 {
goto childerror
}
}
if r.Seccomp != nil {
// Load seccomp filter
_, _, err1 = syscall.RawSyscall(unix.SYS_SECCOMP, SECCOMP_SET_MODE_FILTER, SECCOMP_FILTER_FLAG_TSYNC, uintptr(unsafe.Pointer(r.Seccomp)))
if err1 != 0 {
goto childerror
}
}
}
}
// Enable ptrace if no seccomp is needed

View File

@ -96,4 +96,8 @@ type Runner struct {
// parent will signal child to stop and report the error
// SyncFunc is called right before execve, thus it could track cpu more accurately
SyncFunc func(int) error
// UnshareCgroupAfterSync specifies whether to unshare cgroup namespace after
// sync (the syncFunc might be add the child to the cgroup)
UnshareCgroupAfterSync bool
}

View File

@ -99,7 +99,7 @@ func (b *Builder) WithProc() *Builder {
Source: "proc",
Target: "proc",
FsType: "proc",
Flags: unix.MS_NOSUID,
Flags: unix.MS_NOSUID | unix.MS_RDONLY,
})
return b
}

View File

@ -20,6 +20,8 @@ func (r *Runner) Run(c context.Context) <-chan runner.Result {
Seccomp: r.Seccomp.SockFprog(),
Ptrace: true,
SyncFunc: r.SyncFunc,
UnshareCgroupAfterSync: true,
}
th := &tracerHandler{

View File

@ -46,6 +46,8 @@ func (r *Runner) trace(c context.Context) (result runner.Result) {
PivotRoot: r.Root,
DropCaps: true,
SyncFunc: r.SyncFunc,
UnshareCgroupAfterSync: true,
}
var (