Add ability unshare cgroup after sync

This commit is contained in:
criyle 2020-03-20 18:50:31 -04:00
parent cfa40fa36a
commit 5f133175fd
9 changed files with 158 additions and 11 deletions

View File

@ -8,11 +8,14 @@ import (
"io/ioutil" "io/ioutil"
"os" "os"
"os/signal" "os/signal"
"sync/atomic"
"syscall"
"time" "time"
"github.com/criyle/go-sandbox/config" "github.com/criyle/go-sandbox/config"
"github.com/criyle/go-sandbox/container" "github.com/criyle/go-sandbox/container"
"github.com/criyle/go-sandbox/pkg/cgroup" "github.com/criyle/go-sandbox/pkg/cgroup"
"github.com/criyle/go-sandbox/pkg/forkexec"
"github.com/criyle/go-sandbox/pkg/memfd" "github.com/criyle/go-sandbox/pkg/memfd"
"github.com/criyle/go-sandbox/pkg/mount" "github.com/criyle/go-sandbox/pkg/mount"
"github.com/criyle/go-sandbox/pkg/rlimit" "github.com/criyle/go-sandbox/pkg/rlimit"
@ -30,7 +33,7 @@ const (
var ( var (
addReadable, addWritable, addRawReadable, addRawWritable arrayFlags addReadable, addWritable, addRawReadable, addRawWritable arrayFlags
allowProc, unsafe, showDetails, useCGroup, memfile bool allowProc, unsafe, showDetails, useCGroup, memfile, cred bool
timeLimit, realTimeLimit, memoryLimit, outputLimit, stackLimit uint64 timeLimit, realTimeLimit, memoryLimit, outputLimit, stackLimit uint64
inputFileName, outputFileName, errorFileName, workPath, runt string inputFileName, outputFileName, errorFileName, workPath, runt string
@ -72,6 +75,7 @@ func main() {
flag.BoolVar(&useCGroup, "cgroup", false, "Use cgroup to colloct resource usage") flag.BoolVar(&useCGroup, "cgroup", false, "Use cgroup to colloct resource usage")
flag.BoolVar(&memfile, "memfd", false, "Use memfd as exec file") flag.BoolVar(&memfile, "memfd", false, "Use memfd as exec file")
flag.StringVar(&runt, "runner", "ptrace", "Runner for the program (ptrace, ns, container)") flag.StringVar(&runt, "runner", "ptrace", "Runner for the program (ptrace, ns, container)")
flag.BoolVar(&cred, "cred", false, "Generate credential for containers (uid=10000)")
flag.Parse() flag.Parse()
args = flag.Args() args = flag.Args()
@ -155,6 +159,35 @@ func start() (*runner.Result, error) {
addWrite := filehandler.GetExtraSet(addWritable, addRawWritable) addWrite := filehandler.GetExtraSet(addWritable, addRawWritable)
args, allow, trace, h := config.GetConf(pType, workPath, args, addRead, addWrite, allowProc) args, allow, trace, h := config.GetConf(pType, workPath, args, addRead, addWrite, allowProc)
mb := mount.NewBuilder().
// basic exec and lib
WithBind("/bin", "bin", true).
WithBind("/lib", "lib", true).
WithBind("/lib64", "lib64", true).
WithBind("/usr", "usr", true).
// java wants /proc/self/exe as it need relative path for lib
// however, /proc gives interface like /proc/1/fd/3 ..
// it is fine since open that file will be a EPERM
// changing the fs uid and gid would be a good idea
WithProc().
// some compiler have multiple version
WithBind("/etc/alternatives", "etc/alternatives", true).
// fpc wants /etc/fpc.cfg
WithBind("/etc/fpc.cfg", "etc/fpc.cfg", true).
// go wants /dev/null
WithBind("/dev/null", "dev/null", false).
// ghc wants /var/lib/ghc
WithBind("/var/lib/ghc", "var/lib/ghc", true).
// work dir
WithTmpfs("w", "size=8m,nr_inodes=4k").
// tmp dir
WithTmpfs("tmp", "size=8m,nr_inodes=4k")
mt, err := mb.Build(true)
if err != nil {
return nil, err
}
if useCGroup { if useCGroup {
b, err := cgroup.NewBuilder("runprog").WithCPUAcct().WithMemory().WithPids().FilterByEnv() b, err := cgroup.NewBuilder("runprog").WithCPUAcct().WithMemory().WithPids().FilterByEnv()
if err != nil { if err != nil {
@ -237,8 +270,16 @@ func start() (*runner.Result, error) {
} }
defer os.RemoveAll(root) defer os.RemoveAll(root)
var credG container.CredGenerator
if cred {
credG = newCredGen()
}
b := container.Builder{ b := container.Builder{
Root: root, Root: root,
Mounts: mt,
CredGenerator: credG,
CloneFlags: forkexec.UnshareFlags,
} }
m, err := b.Build() m, err := b.Build()
@ -275,10 +316,6 @@ func start() (*runner.Result, error) {
return nil, fmt.Errorf("cannot make temp root for new namespace") return nil, fmt.Errorf("cannot make temp root for new namespace")
} }
defer os.RemoveAll(root) defer os.RemoveAll(root)
mounts, err := mount.NewDefaultBuilder().WithBind(root, "w", true).Build(true)
if err != nil {
return nil, fmt.Errorf("cannot make rootfs mounts")
}
r = &unshare.Runner{ r = &unshare.Runner{
Args: args, Args: args,
Env: []string{pathEnv}, Env: []string{pathEnv},
@ -289,7 +326,7 @@ func start() (*runner.Result, error) {
Limit: limit, Limit: limit,
Seccomp: filter, Seccomp: filter,
Root: root, Root: root,
Mounts: mounts, Mounts: mt,
ShowDetails: showDetails, ShowDetails: showDetails,
SyncFunc: syncFunc, SyncFunc: syncFunc,
HostName: "run_program", HostName: "run_program",
@ -414,3 +451,19 @@ func getStatus(s runner.Status) int {
return int(StatusFatal) return int(StatusFatal)
} }
} }
type credGen struct {
cur uint32
}
func newCredGen() *credGen {
return &credGen{cur: 10000}
}
func (c *credGen) Get() syscall.Credential {
n := atomic.AddUint32(&c.cur, 1)
return syscall.Credential{
Uid: n,
Gid: n,
}
}

View File

@ -76,6 +76,8 @@ func (c *containerServer) handleExecve(cmd *execCmd, msg *unixsocket.Msg) error
DropCaps: true, DropCaps: true,
SyncFunc: syncFunc, SyncFunc: syncFunc,
Credential: cred, Credential: cred,
UnshareCgroupAfterSync: true,
} }
// starts the runner, error is handled same as wait4 to make communication equal // starts the runner, error is handled same as wait4 to make communication equal
pid, err := r.Start() pid, err := r.Start()

View File

@ -34,6 +34,9 @@ type Builder struct {
// CredGenerator defines a credential generator used to create new container // CredGenerator defines a credential generator used to create new container
CredGenerator CredGenerator CredGenerator CredGenerator
// Clone flags defines unshare clone flag to create container
CloneFlags uintptr
} }
// CredGenerator generates uid / gid credential used by container // CredGenerator generates uid / gid credential used by container
@ -130,13 +133,20 @@ func (b *Builder) Build() (Environment, error) {
uidMap, gidMap = getIDMapping(&cred) uidMap, gidMap = getIDMapping(&cred)
} }
var cloneFlag uintptr
if b.CloneFlags == 0 {
cloneFlag = forkexec.UnshareFlags
} else {
cloneFlag = b.CloneFlags & forkexec.UnshareFlags
}
r := &forkexec.Runner{ r := &forkexec.Runner{
Args: []string{os.Args[0], initArg}, Args: []string{os.Args[0], initArg},
Env: []string{PathEnv}, Env: []string{PathEnv},
ExecFile: execFile.Fd(), ExecFile: execFile.Fd(),
Files: files, Files: files,
WorkDir: containerWD, WorkDir: containerWD,
CloneFlags: forkexec.UnshareFlags, CloneFlags: cloneFlag,
Mounts: mounts, Mounts: mounts,
HostName: containerName, HostName: containerName,
DomainName: containerName, DomainName: containerName,

View File

@ -49,3 +49,17 @@ var (
Inheritable: 0, Inheritable: 0,
} }
) )
const (
_SECURE_NOROOT = 1 << iota
_SECURE_NOROOT_LOCKED
_SECURE_NO_SETUID_FIXUP
_SECURE_NO_SETUID_FIXUP_LOCKED
_SECURE_KEEP_CAPS
_SECURE_KEEP_CAPS_LOCKED
_SECURE_NO_CAP_AMBIENT_RAISE
_SECURE_NO_CAP_AMBIENT_RAISE_LOCKED
)

View File

@ -73,6 +73,13 @@ func forkAndExecInChild(r *Runner, argv0 *byte, argv, env []*byte, workdir, host
goto childerror goto childerror
} }
// keep capabilities through set_uid / set_gid calls (make sure we can use unshare cgroup), later dropped
_, _, err1 = syscall.RawSyscall(syscall.SYS_PRCTL, syscall.PR_SET_SECUREBITS,
_SECURE_KEEP_CAPS_LOCKED|_SECURE_NO_SETUID_FIXUP|_SECURE_NO_SETUID_FIXUP_LOCKED, 0)
if err1 != 0 {
goto childerror
}
// set the credential for the child process(exec_linux.go) // set the credential for the child process(exec_linux.go)
if cred := r.Credential; cred != nil { if cred := r.Credential; cred != nil {
ngroups := uintptr(len(cred.Groups)) ngroups := uintptr(len(cred.Groups))
@ -291,7 +298,13 @@ func forkAndExecInChild(r *Runner, argv0 *byte, argv, env []*byte, workdir, host
} }
// Drop all capabilities // Drop all capabilities
if r.DropCaps { if (r.Credential != nil || r.DropCaps) && !r.UnshareCgroupAfterSync {
// make sure the children have no privilege at all
_, _, err1 = syscall.RawSyscall(syscall.SYS_PRCTL, syscall.PR_SET_SECUREBITS,
_SECURE_KEEP_CAPS_LOCKED|_SECURE_NO_SETUID_FIXUP|_SECURE_NO_SETUID_FIXUP_LOCKED|_SECURE_NOROOT|_SECURE_NOROOT_LOCKED, 0)
if err1 != 0 {
goto childerror
}
_, _, err1 = syscall.RawSyscall(syscall.SYS_CAPSET, uintptr(unsafe.Pointer(&dropCapHeader)), uintptr(unsafe.Pointer(&dropCapData)), 0) _, _, err1 = syscall.RawSyscall(syscall.SYS_CAPSET, uintptr(unsafe.Pointer(&dropCapHeader)), uintptr(unsafe.Pointer(&dropCapData)), 0)
if err1 != 0 { if err1 != 0 {
goto childerror goto childerror
@ -311,6 +324,26 @@ func forkAndExecInChild(r *Runner, argv0 *byte, argv, env []*byte, workdir, host
goto childerror goto childerror
} }
// unshare cgroup namespace
if r.UnshareCgroupAfterSync {
r1, _, err1 = syscall.RawSyscall(syscall.SYS_UNSHARE, uintptr(unix.CLONE_NEWCGROUP), 0, 0)
if err1 != 0 {
goto childerror
}
if r.DropCaps || r.Credential != nil {
// make sure the children have no privilege at all
_, _, err1 = syscall.RawSyscall(syscall.SYS_PRCTL, syscall.PR_SET_SECUREBITS,
_SECURE_KEEP_CAPS_LOCKED|_SECURE_NO_SETUID_FIXUP|_SECURE_NO_SETUID_FIXUP_LOCKED|_SECURE_NOROOT|_SECURE_NOROOT_LOCKED, 0)
if err1 != 0 {
goto childerror
}
_, _, err1 = syscall.RawSyscall(syscall.SYS_CAPSET, uintptr(unsafe.Pointer(&dropCapHeader)), uintptr(unsafe.Pointer(&dropCapData)), 0)
if err1 != 0 {
goto childerror
}
}
}
_, _, err1 = syscall.RawSyscall(syscall.SYS_PTRACE, uintptr(syscall.PTRACE_TRACEME), 0, 0) _, _, err1 = syscall.RawSyscall(syscall.SYS_PTRACE, uintptr(syscall.PTRACE_TRACEME), 0, 0)
if err1 != 0 { if err1 != 0 {
goto childerror goto childerror
@ -329,7 +362,7 @@ func forkAndExecInChild(r *Runner, argv0 *byte, argv, env []*byte, workdir, host
} }
// Load seccomp, stop and wait for tracer // Load seccomp, stop and wait for tracer
if r.Seccomp != nil { if r.Seccomp != nil && (!r.UnshareCgroupAfterSync || r.Ptrace) {
// If execve is seccomp trapped, then tracee stop is necessary // If execve is seccomp trapped, then tracee stop is necessary
// otherwise execve will fail due to ENOSYS // otherwise execve will fail due to ENOSYS
// Do getpid and kill to send SYS_KILL to self // Do getpid and kill to send SYS_KILL to self
@ -354,6 +387,33 @@ func forkAndExecInChild(r *Runner, argv0 *byte, argv, env []*byte, workdir, host
if r1 == 0 || err1 != 0 { if r1 == 0 || err1 != 0 {
goto childerror goto childerror
} }
// unshare cgroup namespace
if r.UnshareCgroupAfterSync {
r1, _, err1 = syscall.RawSyscall(syscall.SYS_UNSHARE, uintptr(unix.CLONE_NEWCGROUP), 0, 0)
if err1 != 0 {
goto childerror
}
if r.DropCaps || r.Credential != nil {
// make sure the children have no privilege at all
_, _, err1 = syscall.RawSyscall(syscall.SYS_PRCTL, syscall.PR_SET_SECUREBITS,
_SECURE_KEEP_CAPS_LOCKED|_SECURE_NO_SETUID_FIXUP|_SECURE_NO_SETUID_FIXUP_LOCKED|_SECURE_NOROOT|_SECURE_NOROOT_LOCKED, 0)
if err1 != 0 {
goto childerror
}
_, _, err1 = syscall.RawSyscall(syscall.SYS_CAPSET, uintptr(unsafe.Pointer(&dropCapHeader)), uintptr(unsafe.Pointer(&dropCapData)), 0)
if err1 != 0 {
goto childerror
}
}
if r.Seccomp != nil {
// Load seccomp filter
_, _, err1 = syscall.RawSyscall(unix.SYS_SECCOMP, SECCOMP_SET_MODE_FILTER, SECCOMP_FILTER_FLAG_TSYNC, uintptr(unsafe.Pointer(r.Seccomp)))
if err1 != 0 {
goto childerror
}
}
}
} }
// Enable ptrace if no seccomp is needed // Enable ptrace if no seccomp is needed

View File

@ -96,4 +96,8 @@ type Runner struct {
// parent will signal child to stop and report the error // parent will signal child to stop and report the error
// SyncFunc is called right before execve, thus it could track cpu more accurately // SyncFunc is called right before execve, thus it could track cpu more accurately
SyncFunc func(int) error SyncFunc func(int) error
// UnshareCgroupAfterSync specifies whether to unshare cgroup namespace after
// sync (the syncFunc might be add the child to the cgroup)
UnshareCgroupAfterSync bool
} }

View File

@ -99,7 +99,7 @@ func (b *Builder) WithProc() *Builder {
Source: "proc", Source: "proc",
Target: "proc", Target: "proc",
FsType: "proc", FsType: "proc",
Flags: unix.MS_NOSUID, Flags: unix.MS_NOSUID | unix.MS_RDONLY,
}) })
return b return b
} }

View File

@ -20,6 +20,8 @@ func (r *Runner) Run(c context.Context) <-chan runner.Result {
Seccomp: r.Seccomp.SockFprog(), Seccomp: r.Seccomp.SockFprog(),
Ptrace: true, Ptrace: true,
SyncFunc: r.SyncFunc, SyncFunc: r.SyncFunc,
UnshareCgroupAfterSync: true,
} }
th := &tracerHandler{ th := &tracerHandler{

View File

@ -46,6 +46,8 @@ func (r *Runner) trace(c context.Context) (result runner.Result) {
PivotRoot: r.Root, PivotRoot: r.Root,
DropCaps: true, DropCaps: true,
SyncFunc: r.SyncFunc, SyncFunc: r.SyncFunc,
UnshareCgroupAfterSync: true,
} }
var ( var (