From 5f133175fdf6fea46b3d0baf87c0603ae852f9e7 Mon Sep 17 00:00:00 2001 From: criyle Date: Fri, 20 Mar 2020 18:50:31 -0400 Subject: [PATCH] Add ability unshare cgroup after sync --- cmd/runprog/main.go | 67 +++++++++++++++++++++++++++++++++---- container/container_exec.go | 2 ++ container/environment.go | 12 ++++++- pkg/forkexec/consts.go | 14 ++++++++ pkg/forkexec/fork_child.go | 64 +++++++++++++++++++++++++++++++++-- pkg/forkexec/runner.go | 4 +++ pkg/mount/builder_linux.go | 2 +- runner/ptrace/run.go | 2 ++ runner/unshare/run.go | 2 ++ 9 files changed, 158 insertions(+), 11 deletions(-) diff --git a/cmd/runprog/main.go b/cmd/runprog/main.go index 4256c39..3ccff67 100644 --- a/cmd/runprog/main.go +++ b/cmd/runprog/main.go @@ -8,11 +8,14 @@ import ( "io/ioutil" "os" "os/signal" + "sync/atomic" + "syscall" "time" "github.com/criyle/go-sandbox/config" "github.com/criyle/go-sandbox/container" "github.com/criyle/go-sandbox/pkg/cgroup" + "github.com/criyle/go-sandbox/pkg/forkexec" "github.com/criyle/go-sandbox/pkg/memfd" "github.com/criyle/go-sandbox/pkg/mount" "github.com/criyle/go-sandbox/pkg/rlimit" @@ -30,7 +33,7 @@ const ( var ( addReadable, addWritable, addRawReadable, addRawWritable arrayFlags - allowProc, unsafe, showDetails, useCGroup, memfile bool + allowProc, unsafe, showDetails, useCGroup, memfile, cred bool timeLimit, realTimeLimit, memoryLimit, outputLimit, stackLimit uint64 inputFileName, outputFileName, errorFileName, workPath, runt string @@ -72,6 +75,7 @@ func main() { flag.BoolVar(&useCGroup, "cgroup", false, "Use cgroup to colloct resource usage") flag.BoolVar(&memfile, "memfd", false, "Use memfd as exec file") flag.StringVar(&runt, "runner", "ptrace", "Runner for the program (ptrace, ns, container)") + flag.BoolVar(&cred, "cred", false, "Generate credential for containers (uid=10000)") flag.Parse() args = flag.Args() @@ -155,6 +159,35 @@ func start() (*runner.Result, error) { addWrite := filehandler.GetExtraSet(addWritable, addRawWritable) args, allow, trace, h := config.GetConf(pType, workPath, args, addRead, addWrite, allowProc) + mb := mount.NewBuilder(). + // basic exec and lib + WithBind("/bin", "bin", true). + WithBind("/lib", "lib", true). + WithBind("/lib64", "lib64", true). + WithBind("/usr", "usr", true). + // java wants /proc/self/exe as it need relative path for lib + // however, /proc gives interface like /proc/1/fd/3 .. + // it is fine since open that file will be a EPERM + // changing the fs uid and gid would be a good idea + WithProc(). + // some compiler have multiple version + WithBind("/etc/alternatives", "etc/alternatives", true). + // fpc wants /etc/fpc.cfg + WithBind("/etc/fpc.cfg", "etc/fpc.cfg", true). + // go wants /dev/null + WithBind("/dev/null", "dev/null", false). + // ghc wants /var/lib/ghc + WithBind("/var/lib/ghc", "var/lib/ghc", true). + // work dir + WithTmpfs("w", "size=8m,nr_inodes=4k"). + // tmp dir + WithTmpfs("tmp", "size=8m,nr_inodes=4k") + + mt, err := mb.Build(true) + if err != nil { + return nil, err + } + if useCGroup { b, err := cgroup.NewBuilder("runprog").WithCPUAcct().WithMemory().WithPids().FilterByEnv() if err != nil { @@ -237,8 +270,16 @@ func start() (*runner.Result, error) { } defer os.RemoveAll(root) + var credG container.CredGenerator + if cred { + credG = newCredGen() + } + b := container.Builder{ - Root: root, + Root: root, + Mounts: mt, + CredGenerator: credG, + CloneFlags: forkexec.UnshareFlags, } m, err := b.Build() @@ -275,10 +316,6 @@ func start() (*runner.Result, error) { return nil, fmt.Errorf("cannot make temp root for new namespace") } defer os.RemoveAll(root) - mounts, err := mount.NewDefaultBuilder().WithBind(root, "w", true).Build(true) - if err != nil { - return nil, fmt.Errorf("cannot make rootfs mounts") - } r = &unshare.Runner{ Args: args, Env: []string{pathEnv}, @@ -289,7 +326,7 @@ func start() (*runner.Result, error) { Limit: limit, Seccomp: filter, Root: root, - Mounts: mounts, + Mounts: mt, ShowDetails: showDetails, SyncFunc: syncFunc, HostName: "run_program", @@ -414,3 +451,19 @@ func getStatus(s runner.Status) int { return int(StatusFatal) } } + +type credGen struct { + cur uint32 +} + +func newCredGen() *credGen { + return &credGen{cur: 10000} +} + +func (c *credGen) Get() syscall.Credential { + n := atomic.AddUint32(&c.cur, 1) + return syscall.Credential{ + Uid: n, + Gid: n, + } +} diff --git a/container/container_exec.go b/container/container_exec.go index 2aa74cb..aa68ddc 100644 --- a/container/container_exec.go +++ b/container/container_exec.go @@ -76,6 +76,8 @@ func (c *containerServer) handleExecve(cmd *execCmd, msg *unixsocket.Msg) error DropCaps: true, SyncFunc: syncFunc, Credential: cred, + + UnshareCgroupAfterSync: true, } // starts the runner, error is handled same as wait4 to make communication equal pid, err := r.Start() diff --git a/container/environment.go b/container/environment.go index 3c1f711..f1f8f35 100644 --- a/container/environment.go +++ b/container/environment.go @@ -34,6 +34,9 @@ type Builder struct { // CredGenerator defines a credential generator used to create new container CredGenerator CredGenerator + + // Clone flags defines unshare clone flag to create container + CloneFlags uintptr } // CredGenerator generates uid / gid credential used by container @@ -130,13 +133,20 @@ func (b *Builder) Build() (Environment, error) { uidMap, gidMap = getIDMapping(&cred) } + var cloneFlag uintptr + if b.CloneFlags == 0 { + cloneFlag = forkexec.UnshareFlags + } else { + cloneFlag = b.CloneFlags & forkexec.UnshareFlags + } + r := &forkexec.Runner{ Args: []string{os.Args[0], initArg}, Env: []string{PathEnv}, ExecFile: execFile.Fd(), Files: files, WorkDir: containerWD, - CloneFlags: forkexec.UnshareFlags, + CloneFlags: cloneFlag, Mounts: mounts, HostName: containerName, DomainName: containerName, diff --git a/pkg/forkexec/consts.go b/pkg/forkexec/consts.go index 202212b..8f02358 100644 --- a/pkg/forkexec/consts.go +++ b/pkg/forkexec/consts.go @@ -49,3 +49,17 @@ var ( Inheritable: 0, } ) + +const ( + _SECURE_NOROOT = 1 << iota + _SECURE_NOROOT_LOCKED + + _SECURE_NO_SETUID_FIXUP + _SECURE_NO_SETUID_FIXUP_LOCKED + + _SECURE_KEEP_CAPS + _SECURE_KEEP_CAPS_LOCKED + + _SECURE_NO_CAP_AMBIENT_RAISE + _SECURE_NO_CAP_AMBIENT_RAISE_LOCKED +) diff --git a/pkg/forkexec/fork_child.go b/pkg/forkexec/fork_child.go index de4c24d..62c6f80 100644 --- a/pkg/forkexec/fork_child.go +++ b/pkg/forkexec/fork_child.go @@ -73,6 +73,13 @@ func forkAndExecInChild(r *Runner, argv0 *byte, argv, env []*byte, workdir, host goto childerror } + // keep capabilities through set_uid / set_gid calls (make sure we can use unshare cgroup), later dropped + _, _, err1 = syscall.RawSyscall(syscall.SYS_PRCTL, syscall.PR_SET_SECUREBITS, + _SECURE_KEEP_CAPS_LOCKED|_SECURE_NO_SETUID_FIXUP|_SECURE_NO_SETUID_FIXUP_LOCKED, 0) + if err1 != 0 { + goto childerror + } + // set the credential for the child process(exec_linux.go) if cred := r.Credential; cred != nil { ngroups := uintptr(len(cred.Groups)) @@ -291,7 +298,13 @@ func forkAndExecInChild(r *Runner, argv0 *byte, argv, env []*byte, workdir, host } // Drop all capabilities - if r.DropCaps { + if (r.Credential != nil || r.DropCaps) && !r.UnshareCgroupAfterSync { + // make sure the children have no privilege at all + _, _, err1 = syscall.RawSyscall(syscall.SYS_PRCTL, syscall.PR_SET_SECUREBITS, + _SECURE_KEEP_CAPS_LOCKED|_SECURE_NO_SETUID_FIXUP|_SECURE_NO_SETUID_FIXUP_LOCKED|_SECURE_NOROOT|_SECURE_NOROOT_LOCKED, 0) + if err1 != 0 { + goto childerror + } _, _, err1 = syscall.RawSyscall(syscall.SYS_CAPSET, uintptr(unsafe.Pointer(&dropCapHeader)), uintptr(unsafe.Pointer(&dropCapData)), 0) if err1 != 0 { goto childerror @@ -311,6 +324,26 @@ func forkAndExecInChild(r *Runner, argv0 *byte, argv, env []*byte, workdir, host goto childerror } + // unshare cgroup namespace + if r.UnshareCgroupAfterSync { + r1, _, err1 = syscall.RawSyscall(syscall.SYS_UNSHARE, uintptr(unix.CLONE_NEWCGROUP), 0, 0) + if err1 != 0 { + goto childerror + } + if r.DropCaps || r.Credential != nil { + // make sure the children have no privilege at all + _, _, err1 = syscall.RawSyscall(syscall.SYS_PRCTL, syscall.PR_SET_SECUREBITS, + _SECURE_KEEP_CAPS_LOCKED|_SECURE_NO_SETUID_FIXUP|_SECURE_NO_SETUID_FIXUP_LOCKED|_SECURE_NOROOT|_SECURE_NOROOT_LOCKED, 0) + if err1 != 0 { + goto childerror + } + _, _, err1 = syscall.RawSyscall(syscall.SYS_CAPSET, uintptr(unsafe.Pointer(&dropCapHeader)), uintptr(unsafe.Pointer(&dropCapData)), 0) + if err1 != 0 { + goto childerror + } + } + } + _, _, err1 = syscall.RawSyscall(syscall.SYS_PTRACE, uintptr(syscall.PTRACE_TRACEME), 0, 0) if err1 != 0 { goto childerror @@ -329,7 +362,7 @@ func forkAndExecInChild(r *Runner, argv0 *byte, argv, env []*byte, workdir, host } // Load seccomp, stop and wait for tracer - if r.Seccomp != nil { + if r.Seccomp != nil && (!r.UnshareCgroupAfterSync || r.Ptrace) { // If execve is seccomp trapped, then tracee stop is necessary // otherwise execve will fail due to ENOSYS // Do getpid and kill to send SYS_KILL to self @@ -354,6 +387,33 @@ func forkAndExecInChild(r *Runner, argv0 *byte, argv, env []*byte, workdir, host if r1 == 0 || err1 != 0 { goto childerror } + + // unshare cgroup namespace + if r.UnshareCgroupAfterSync { + r1, _, err1 = syscall.RawSyscall(syscall.SYS_UNSHARE, uintptr(unix.CLONE_NEWCGROUP), 0, 0) + if err1 != 0 { + goto childerror + } + if r.DropCaps || r.Credential != nil { + // make sure the children have no privilege at all + _, _, err1 = syscall.RawSyscall(syscall.SYS_PRCTL, syscall.PR_SET_SECUREBITS, + _SECURE_KEEP_CAPS_LOCKED|_SECURE_NO_SETUID_FIXUP|_SECURE_NO_SETUID_FIXUP_LOCKED|_SECURE_NOROOT|_SECURE_NOROOT_LOCKED, 0) + if err1 != 0 { + goto childerror + } + _, _, err1 = syscall.RawSyscall(syscall.SYS_CAPSET, uintptr(unsafe.Pointer(&dropCapHeader)), uintptr(unsafe.Pointer(&dropCapData)), 0) + if err1 != 0 { + goto childerror + } + } + if r.Seccomp != nil { + // Load seccomp filter + _, _, err1 = syscall.RawSyscall(unix.SYS_SECCOMP, SECCOMP_SET_MODE_FILTER, SECCOMP_FILTER_FLAG_TSYNC, uintptr(unsafe.Pointer(r.Seccomp))) + if err1 != 0 { + goto childerror + } + } + } } // Enable ptrace if no seccomp is needed diff --git a/pkg/forkexec/runner.go b/pkg/forkexec/runner.go index 505c225..9fcd54c 100644 --- a/pkg/forkexec/runner.go +++ b/pkg/forkexec/runner.go @@ -96,4 +96,8 @@ type Runner struct { // parent will signal child to stop and report the error // SyncFunc is called right before execve, thus it could track cpu more accurately SyncFunc func(int) error + + // UnshareCgroupAfterSync specifies whether to unshare cgroup namespace after + // sync (the syncFunc might be add the child to the cgroup) + UnshareCgroupAfterSync bool } diff --git a/pkg/mount/builder_linux.go b/pkg/mount/builder_linux.go index a74bc4d..7a7e6b9 100644 --- a/pkg/mount/builder_linux.go +++ b/pkg/mount/builder_linux.go @@ -99,7 +99,7 @@ func (b *Builder) WithProc() *Builder { Source: "proc", Target: "proc", FsType: "proc", - Flags: unix.MS_NOSUID, + Flags: unix.MS_NOSUID | unix.MS_RDONLY, }) return b } diff --git a/runner/ptrace/run.go b/runner/ptrace/run.go index 6cfe4cc..ae3828e 100644 --- a/runner/ptrace/run.go +++ b/runner/ptrace/run.go @@ -20,6 +20,8 @@ func (r *Runner) Run(c context.Context) <-chan runner.Result { Seccomp: r.Seccomp.SockFprog(), Ptrace: true, SyncFunc: r.SyncFunc, + + UnshareCgroupAfterSync: true, } th := &tracerHandler{ diff --git a/runner/unshare/run.go b/runner/unshare/run.go index 5427a7e..5fe311d 100644 --- a/runner/unshare/run.go +++ b/runner/unshare/run.go @@ -46,6 +46,8 @@ func (r *Runner) trace(c context.Context) (result runner.Result) { PivotRoot: r.Root, DropCaps: true, SyncFunc: r.SyncFunc, + + UnshareCgroupAfterSync: true, } var (