mirror of
https://github.com/criyle/go-sandbox.git
synced 2025-11-04 14:49:53 +08:00
Add ability unshare cgroup after sync
This commit is contained in:
parent
cfa40fa36a
commit
5f133175fd
@ -8,11 +8,14 @@ import (
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"os/signal"
|
||||
"sync/atomic"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/criyle/go-sandbox/config"
|
||||
"github.com/criyle/go-sandbox/container"
|
||||
"github.com/criyle/go-sandbox/pkg/cgroup"
|
||||
"github.com/criyle/go-sandbox/pkg/forkexec"
|
||||
"github.com/criyle/go-sandbox/pkg/memfd"
|
||||
"github.com/criyle/go-sandbox/pkg/mount"
|
||||
"github.com/criyle/go-sandbox/pkg/rlimit"
|
||||
@ -30,7 +33,7 @@ const (
|
||||
|
||||
var (
|
||||
addReadable, addWritable, addRawReadable, addRawWritable arrayFlags
|
||||
allowProc, unsafe, showDetails, useCGroup, memfile bool
|
||||
allowProc, unsafe, showDetails, useCGroup, memfile, cred bool
|
||||
timeLimit, realTimeLimit, memoryLimit, outputLimit, stackLimit uint64
|
||||
inputFileName, outputFileName, errorFileName, workPath, runt string
|
||||
|
||||
@ -72,6 +75,7 @@ func main() {
|
||||
flag.BoolVar(&useCGroup, "cgroup", false, "Use cgroup to colloct resource usage")
|
||||
flag.BoolVar(&memfile, "memfd", false, "Use memfd as exec file")
|
||||
flag.StringVar(&runt, "runner", "ptrace", "Runner for the program (ptrace, ns, container)")
|
||||
flag.BoolVar(&cred, "cred", false, "Generate credential for containers (uid=10000)")
|
||||
flag.Parse()
|
||||
|
||||
args = flag.Args()
|
||||
@ -155,6 +159,35 @@ func start() (*runner.Result, error) {
|
||||
addWrite := filehandler.GetExtraSet(addWritable, addRawWritable)
|
||||
args, allow, trace, h := config.GetConf(pType, workPath, args, addRead, addWrite, allowProc)
|
||||
|
||||
mb := mount.NewBuilder().
|
||||
// basic exec and lib
|
||||
WithBind("/bin", "bin", true).
|
||||
WithBind("/lib", "lib", true).
|
||||
WithBind("/lib64", "lib64", true).
|
||||
WithBind("/usr", "usr", true).
|
||||
// java wants /proc/self/exe as it need relative path for lib
|
||||
// however, /proc gives interface like /proc/1/fd/3 ..
|
||||
// it is fine since open that file will be a EPERM
|
||||
// changing the fs uid and gid would be a good idea
|
||||
WithProc().
|
||||
// some compiler have multiple version
|
||||
WithBind("/etc/alternatives", "etc/alternatives", true).
|
||||
// fpc wants /etc/fpc.cfg
|
||||
WithBind("/etc/fpc.cfg", "etc/fpc.cfg", true).
|
||||
// go wants /dev/null
|
||||
WithBind("/dev/null", "dev/null", false).
|
||||
// ghc wants /var/lib/ghc
|
||||
WithBind("/var/lib/ghc", "var/lib/ghc", true).
|
||||
// work dir
|
||||
WithTmpfs("w", "size=8m,nr_inodes=4k").
|
||||
// tmp dir
|
||||
WithTmpfs("tmp", "size=8m,nr_inodes=4k")
|
||||
|
||||
mt, err := mb.Build(true)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if useCGroup {
|
||||
b, err := cgroup.NewBuilder("runprog").WithCPUAcct().WithMemory().WithPids().FilterByEnv()
|
||||
if err != nil {
|
||||
@ -237,8 +270,16 @@ func start() (*runner.Result, error) {
|
||||
}
|
||||
defer os.RemoveAll(root)
|
||||
|
||||
var credG container.CredGenerator
|
||||
if cred {
|
||||
credG = newCredGen()
|
||||
}
|
||||
|
||||
b := container.Builder{
|
||||
Root: root,
|
||||
Root: root,
|
||||
Mounts: mt,
|
||||
CredGenerator: credG,
|
||||
CloneFlags: forkexec.UnshareFlags,
|
||||
}
|
||||
|
||||
m, err := b.Build()
|
||||
@ -275,10 +316,6 @@ func start() (*runner.Result, error) {
|
||||
return nil, fmt.Errorf("cannot make temp root for new namespace")
|
||||
}
|
||||
defer os.RemoveAll(root)
|
||||
mounts, err := mount.NewDefaultBuilder().WithBind(root, "w", true).Build(true)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot make rootfs mounts")
|
||||
}
|
||||
r = &unshare.Runner{
|
||||
Args: args,
|
||||
Env: []string{pathEnv},
|
||||
@ -289,7 +326,7 @@ func start() (*runner.Result, error) {
|
||||
Limit: limit,
|
||||
Seccomp: filter,
|
||||
Root: root,
|
||||
Mounts: mounts,
|
||||
Mounts: mt,
|
||||
ShowDetails: showDetails,
|
||||
SyncFunc: syncFunc,
|
||||
HostName: "run_program",
|
||||
@ -414,3 +451,19 @@ func getStatus(s runner.Status) int {
|
||||
return int(StatusFatal)
|
||||
}
|
||||
}
|
||||
|
||||
type credGen struct {
|
||||
cur uint32
|
||||
}
|
||||
|
||||
func newCredGen() *credGen {
|
||||
return &credGen{cur: 10000}
|
||||
}
|
||||
|
||||
func (c *credGen) Get() syscall.Credential {
|
||||
n := atomic.AddUint32(&c.cur, 1)
|
||||
return syscall.Credential{
|
||||
Uid: n,
|
||||
Gid: n,
|
||||
}
|
||||
}
|
||||
|
||||
@ -76,6 +76,8 @@ func (c *containerServer) handleExecve(cmd *execCmd, msg *unixsocket.Msg) error
|
||||
DropCaps: true,
|
||||
SyncFunc: syncFunc,
|
||||
Credential: cred,
|
||||
|
||||
UnshareCgroupAfterSync: true,
|
||||
}
|
||||
// starts the runner, error is handled same as wait4 to make communication equal
|
||||
pid, err := r.Start()
|
||||
|
||||
@ -34,6 +34,9 @@ type Builder struct {
|
||||
|
||||
// CredGenerator defines a credential generator used to create new container
|
||||
CredGenerator CredGenerator
|
||||
|
||||
// Clone flags defines unshare clone flag to create container
|
||||
CloneFlags uintptr
|
||||
}
|
||||
|
||||
// CredGenerator generates uid / gid credential used by container
|
||||
@ -130,13 +133,20 @@ func (b *Builder) Build() (Environment, error) {
|
||||
uidMap, gidMap = getIDMapping(&cred)
|
||||
}
|
||||
|
||||
var cloneFlag uintptr
|
||||
if b.CloneFlags == 0 {
|
||||
cloneFlag = forkexec.UnshareFlags
|
||||
} else {
|
||||
cloneFlag = b.CloneFlags & forkexec.UnshareFlags
|
||||
}
|
||||
|
||||
r := &forkexec.Runner{
|
||||
Args: []string{os.Args[0], initArg},
|
||||
Env: []string{PathEnv},
|
||||
ExecFile: execFile.Fd(),
|
||||
Files: files,
|
||||
WorkDir: containerWD,
|
||||
CloneFlags: forkexec.UnshareFlags,
|
||||
CloneFlags: cloneFlag,
|
||||
Mounts: mounts,
|
||||
HostName: containerName,
|
||||
DomainName: containerName,
|
||||
|
||||
@ -49,3 +49,17 @@ var (
|
||||
Inheritable: 0,
|
||||
}
|
||||
)
|
||||
|
||||
const (
|
||||
_SECURE_NOROOT = 1 << iota
|
||||
_SECURE_NOROOT_LOCKED
|
||||
|
||||
_SECURE_NO_SETUID_FIXUP
|
||||
_SECURE_NO_SETUID_FIXUP_LOCKED
|
||||
|
||||
_SECURE_KEEP_CAPS
|
||||
_SECURE_KEEP_CAPS_LOCKED
|
||||
|
||||
_SECURE_NO_CAP_AMBIENT_RAISE
|
||||
_SECURE_NO_CAP_AMBIENT_RAISE_LOCKED
|
||||
)
|
||||
|
||||
@ -73,6 +73,13 @@ func forkAndExecInChild(r *Runner, argv0 *byte, argv, env []*byte, workdir, host
|
||||
goto childerror
|
||||
}
|
||||
|
||||
// keep capabilities through set_uid / set_gid calls (make sure we can use unshare cgroup), later dropped
|
||||
_, _, err1 = syscall.RawSyscall(syscall.SYS_PRCTL, syscall.PR_SET_SECUREBITS,
|
||||
_SECURE_KEEP_CAPS_LOCKED|_SECURE_NO_SETUID_FIXUP|_SECURE_NO_SETUID_FIXUP_LOCKED, 0)
|
||||
if err1 != 0 {
|
||||
goto childerror
|
||||
}
|
||||
|
||||
// set the credential for the child process(exec_linux.go)
|
||||
if cred := r.Credential; cred != nil {
|
||||
ngroups := uintptr(len(cred.Groups))
|
||||
@ -291,7 +298,13 @@ func forkAndExecInChild(r *Runner, argv0 *byte, argv, env []*byte, workdir, host
|
||||
}
|
||||
|
||||
// Drop all capabilities
|
||||
if r.DropCaps {
|
||||
if (r.Credential != nil || r.DropCaps) && !r.UnshareCgroupAfterSync {
|
||||
// make sure the children have no privilege at all
|
||||
_, _, err1 = syscall.RawSyscall(syscall.SYS_PRCTL, syscall.PR_SET_SECUREBITS,
|
||||
_SECURE_KEEP_CAPS_LOCKED|_SECURE_NO_SETUID_FIXUP|_SECURE_NO_SETUID_FIXUP_LOCKED|_SECURE_NOROOT|_SECURE_NOROOT_LOCKED, 0)
|
||||
if err1 != 0 {
|
||||
goto childerror
|
||||
}
|
||||
_, _, err1 = syscall.RawSyscall(syscall.SYS_CAPSET, uintptr(unsafe.Pointer(&dropCapHeader)), uintptr(unsafe.Pointer(&dropCapData)), 0)
|
||||
if err1 != 0 {
|
||||
goto childerror
|
||||
@ -311,6 +324,26 @@ func forkAndExecInChild(r *Runner, argv0 *byte, argv, env []*byte, workdir, host
|
||||
goto childerror
|
||||
}
|
||||
|
||||
// unshare cgroup namespace
|
||||
if r.UnshareCgroupAfterSync {
|
||||
r1, _, err1 = syscall.RawSyscall(syscall.SYS_UNSHARE, uintptr(unix.CLONE_NEWCGROUP), 0, 0)
|
||||
if err1 != 0 {
|
||||
goto childerror
|
||||
}
|
||||
if r.DropCaps || r.Credential != nil {
|
||||
// make sure the children have no privilege at all
|
||||
_, _, err1 = syscall.RawSyscall(syscall.SYS_PRCTL, syscall.PR_SET_SECUREBITS,
|
||||
_SECURE_KEEP_CAPS_LOCKED|_SECURE_NO_SETUID_FIXUP|_SECURE_NO_SETUID_FIXUP_LOCKED|_SECURE_NOROOT|_SECURE_NOROOT_LOCKED, 0)
|
||||
if err1 != 0 {
|
||||
goto childerror
|
||||
}
|
||||
_, _, err1 = syscall.RawSyscall(syscall.SYS_CAPSET, uintptr(unsafe.Pointer(&dropCapHeader)), uintptr(unsafe.Pointer(&dropCapData)), 0)
|
||||
if err1 != 0 {
|
||||
goto childerror
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
_, _, err1 = syscall.RawSyscall(syscall.SYS_PTRACE, uintptr(syscall.PTRACE_TRACEME), 0, 0)
|
||||
if err1 != 0 {
|
||||
goto childerror
|
||||
@ -329,7 +362,7 @@ func forkAndExecInChild(r *Runner, argv0 *byte, argv, env []*byte, workdir, host
|
||||
}
|
||||
|
||||
// Load seccomp, stop and wait for tracer
|
||||
if r.Seccomp != nil {
|
||||
if r.Seccomp != nil && (!r.UnshareCgroupAfterSync || r.Ptrace) {
|
||||
// If execve is seccomp trapped, then tracee stop is necessary
|
||||
// otherwise execve will fail due to ENOSYS
|
||||
// Do getpid and kill to send SYS_KILL to self
|
||||
@ -354,6 +387,33 @@ func forkAndExecInChild(r *Runner, argv0 *byte, argv, env []*byte, workdir, host
|
||||
if r1 == 0 || err1 != 0 {
|
||||
goto childerror
|
||||
}
|
||||
|
||||
// unshare cgroup namespace
|
||||
if r.UnshareCgroupAfterSync {
|
||||
r1, _, err1 = syscall.RawSyscall(syscall.SYS_UNSHARE, uintptr(unix.CLONE_NEWCGROUP), 0, 0)
|
||||
if err1 != 0 {
|
||||
goto childerror
|
||||
}
|
||||
if r.DropCaps || r.Credential != nil {
|
||||
// make sure the children have no privilege at all
|
||||
_, _, err1 = syscall.RawSyscall(syscall.SYS_PRCTL, syscall.PR_SET_SECUREBITS,
|
||||
_SECURE_KEEP_CAPS_LOCKED|_SECURE_NO_SETUID_FIXUP|_SECURE_NO_SETUID_FIXUP_LOCKED|_SECURE_NOROOT|_SECURE_NOROOT_LOCKED, 0)
|
||||
if err1 != 0 {
|
||||
goto childerror
|
||||
}
|
||||
_, _, err1 = syscall.RawSyscall(syscall.SYS_CAPSET, uintptr(unsafe.Pointer(&dropCapHeader)), uintptr(unsafe.Pointer(&dropCapData)), 0)
|
||||
if err1 != 0 {
|
||||
goto childerror
|
||||
}
|
||||
}
|
||||
if r.Seccomp != nil {
|
||||
// Load seccomp filter
|
||||
_, _, err1 = syscall.RawSyscall(unix.SYS_SECCOMP, SECCOMP_SET_MODE_FILTER, SECCOMP_FILTER_FLAG_TSYNC, uintptr(unsafe.Pointer(r.Seccomp)))
|
||||
if err1 != 0 {
|
||||
goto childerror
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Enable ptrace if no seccomp is needed
|
||||
|
||||
@ -96,4 +96,8 @@ type Runner struct {
|
||||
// parent will signal child to stop and report the error
|
||||
// SyncFunc is called right before execve, thus it could track cpu more accurately
|
||||
SyncFunc func(int) error
|
||||
|
||||
// UnshareCgroupAfterSync specifies whether to unshare cgroup namespace after
|
||||
// sync (the syncFunc might be add the child to the cgroup)
|
||||
UnshareCgroupAfterSync bool
|
||||
}
|
||||
|
||||
@ -99,7 +99,7 @@ func (b *Builder) WithProc() *Builder {
|
||||
Source: "proc",
|
||||
Target: "proc",
|
||||
FsType: "proc",
|
||||
Flags: unix.MS_NOSUID,
|
||||
Flags: unix.MS_NOSUID | unix.MS_RDONLY,
|
||||
})
|
||||
return b
|
||||
}
|
||||
|
||||
@ -20,6 +20,8 @@ func (r *Runner) Run(c context.Context) <-chan runner.Result {
|
||||
Seccomp: r.Seccomp.SockFprog(),
|
||||
Ptrace: true,
|
||||
SyncFunc: r.SyncFunc,
|
||||
|
||||
UnshareCgroupAfterSync: true,
|
||||
}
|
||||
|
||||
th := &tracerHandler{
|
||||
|
||||
@ -46,6 +46,8 @@ func (r *Runner) trace(c context.Context) (result runner.Result) {
|
||||
PivotRoot: r.Root,
|
||||
DropCaps: true,
|
||||
SyncFunc: r.SyncFunc,
|
||||
|
||||
UnshareCgroupAfterSync: true,
|
||||
}
|
||||
|
||||
var (
|
||||
|
||||
Loading…
Reference in New Issue
Block a user