mirror of
https://github.com/criyle/go-sandbox.git
synced 2025-11-04 14:49:53 +08:00
Add ability unshare cgroup after sync
This commit is contained in:
parent
cfa40fa36a
commit
5f133175fd
@ -8,11 +8,14 @@ import (
|
|||||||
"io/ioutil"
|
"io/ioutil"
|
||||||
"os"
|
"os"
|
||||||
"os/signal"
|
"os/signal"
|
||||||
|
"sync/atomic"
|
||||||
|
"syscall"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/criyle/go-sandbox/config"
|
"github.com/criyle/go-sandbox/config"
|
||||||
"github.com/criyle/go-sandbox/container"
|
"github.com/criyle/go-sandbox/container"
|
||||||
"github.com/criyle/go-sandbox/pkg/cgroup"
|
"github.com/criyle/go-sandbox/pkg/cgroup"
|
||||||
|
"github.com/criyle/go-sandbox/pkg/forkexec"
|
||||||
"github.com/criyle/go-sandbox/pkg/memfd"
|
"github.com/criyle/go-sandbox/pkg/memfd"
|
||||||
"github.com/criyle/go-sandbox/pkg/mount"
|
"github.com/criyle/go-sandbox/pkg/mount"
|
||||||
"github.com/criyle/go-sandbox/pkg/rlimit"
|
"github.com/criyle/go-sandbox/pkg/rlimit"
|
||||||
@ -30,7 +33,7 @@ const (
|
|||||||
|
|
||||||
var (
|
var (
|
||||||
addReadable, addWritable, addRawReadable, addRawWritable arrayFlags
|
addReadable, addWritable, addRawReadable, addRawWritable arrayFlags
|
||||||
allowProc, unsafe, showDetails, useCGroup, memfile bool
|
allowProc, unsafe, showDetails, useCGroup, memfile, cred bool
|
||||||
timeLimit, realTimeLimit, memoryLimit, outputLimit, stackLimit uint64
|
timeLimit, realTimeLimit, memoryLimit, outputLimit, stackLimit uint64
|
||||||
inputFileName, outputFileName, errorFileName, workPath, runt string
|
inputFileName, outputFileName, errorFileName, workPath, runt string
|
||||||
|
|
||||||
@ -72,6 +75,7 @@ func main() {
|
|||||||
flag.BoolVar(&useCGroup, "cgroup", false, "Use cgroup to colloct resource usage")
|
flag.BoolVar(&useCGroup, "cgroup", false, "Use cgroup to colloct resource usage")
|
||||||
flag.BoolVar(&memfile, "memfd", false, "Use memfd as exec file")
|
flag.BoolVar(&memfile, "memfd", false, "Use memfd as exec file")
|
||||||
flag.StringVar(&runt, "runner", "ptrace", "Runner for the program (ptrace, ns, container)")
|
flag.StringVar(&runt, "runner", "ptrace", "Runner for the program (ptrace, ns, container)")
|
||||||
|
flag.BoolVar(&cred, "cred", false, "Generate credential for containers (uid=10000)")
|
||||||
flag.Parse()
|
flag.Parse()
|
||||||
|
|
||||||
args = flag.Args()
|
args = flag.Args()
|
||||||
@ -155,6 +159,35 @@ func start() (*runner.Result, error) {
|
|||||||
addWrite := filehandler.GetExtraSet(addWritable, addRawWritable)
|
addWrite := filehandler.GetExtraSet(addWritable, addRawWritable)
|
||||||
args, allow, trace, h := config.GetConf(pType, workPath, args, addRead, addWrite, allowProc)
|
args, allow, trace, h := config.GetConf(pType, workPath, args, addRead, addWrite, allowProc)
|
||||||
|
|
||||||
|
mb := mount.NewBuilder().
|
||||||
|
// basic exec and lib
|
||||||
|
WithBind("/bin", "bin", true).
|
||||||
|
WithBind("/lib", "lib", true).
|
||||||
|
WithBind("/lib64", "lib64", true).
|
||||||
|
WithBind("/usr", "usr", true).
|
||||||
|
// java wants /proc/self/exe as it need relative path for lib
|
||||||
|
// however, /proc gives interface like /proc/1/fd/3 ..
|
||||||
|
// it is fine since open that file will be a EPERM
|
||||||
|
// changing the fs uid and gid would be a good idea
|
||||||
|
WithProc().
|
||||||
|
// some compiler have multiple version
|
||||||
|
WithBind("/etc/alternatives", "etc/alternatives", true).
|
||||||
|
// fpc wants /etc/fpc.cfg
|
||||||
|
WithBind("/etc/fpc.cfg", "etc/fpc.cfg", true).
|
||||||
|
// go wants /dev/null
|
||||||
|
WithBind("/dev/null", "dev/null", false).
|
||||||
|
// ghc wants /var/lib/ghc
|
||||||
|
WithBind("/var/lib/ghc", "var/lib/ghc", true).
|
||||||
|
// work dir
|
||||||
|
WithTmpfs("w", "size=8m,nr_inodes=4k").
|
||||||
|
// tmp dir
|
||||||
|
WithTmpfs("tmp", "size=8m,nr_inodes=4k")
|
||||||
|
|
||||||
|
mt, err := mb.Build(true)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
if useCGroup {
|
if useCGroup {
|
||||||
b, err := cgroup.NewBuilder("runprog").WithCPUAcct().WithMemory().WithPids().FilterByEnv()
|
b, err := cgroup.NewBuilder("runprog").WithCPUAcct().WithMemory().WithPids().FilterByEnv()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@ -237,8 +270,16 @@ func start() (*runner.Result, error) {
|
|||||||
}
|
}
|
||||||
defer os.RemoveAll(root)
|
defer os.RemoveAll(root)
|
||||||
|
|
||||||
|
var credG container.CredGenerator
|
||||||
|
if cred {
|
||||||
|
credG = newCredGen()
|
||||||
|
}
|
||||||
|
|
||||||
b := container.Builder{
|
b := container.Builder{
|
||||||
Root: root,
|
Root: root,
|
||||||
|
Mounts: mt,
|
||||||
|
CredGenerator: credG,
|
||||||
|
CloneFlags: forkexec.UnshareFlags,
|
||||||
}
|
}
|
||||||
|
|
||||||
m, err := b.Build()
|
m, err := b.Build()
|
||||||
@ -275,10 +316,6 @@ func start() (*runner.Result, error) {
|
|||||||
return nil, fmt.Errorf("cannot make temp root for new namespace")
|
return nil, fmt.Errorf("cannot make temp root for new namespace")
|
||||||
}
|
}
|
||||||
defer os.RemoveAll(root)
|
defer os.RemoveAll(root)
|
||||||
mounts, err := mount.NewDefaultBuilder().WithBind(root, "w", true).Build(true)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("cannot make rootfs mounts")
|
|
||||||
}
|
|
||||||
r = &unshare.Runner{
|
r = &unshare.Runner{
|
||||||
Args: args,
|
Args: args,
|
||||||
Env: []string{pathEnv},
|
Env: []string{pathEnv},
|
||||||
@ -289,7 +326,7 @@ func start() (*runner.Result, error) {
|
|||||||
Limit: limit,
|
Limit: limit,
|
||||||
Seccomp: filter,
|
Seccomp: filter,
|
||||||
Root: root,
|
Root: root,
|
||||||
Mounts: mounts,
|
Mounts: mt,
|
||||||
ShowDetails: showDetails,
|
ShowDetails: showDetails,
|
||||||
SyncFunc: syncFunc,
|
SyncFunc: syncFunc,
|
||||||
HostName: "run_program",
|
HostName: "run_program",
|
||||||
@ -414,3 +451,19 @@ func getStatus(s runner.Status) int {
|
|||||||
return int(StatusFatal)
|
return int(StatusFatal)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type credGen struct {
|
||||||
|
cur uint32
|
||||||
|
}
|
||||||
|
|
||||||
|
func newCredGen() *credGen {
|
||||||
|
return &credGen{cur: 10000}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *credGen) Get() syscall.Credential {
|
||||||
|
n := atomic.AddUint32(&c.cur, 1)
|
||||||
|
return syscall.Credential{
|
||||||
|
Uid: n,
|
||||||
|
Gid: n,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@ -76,6 +76,8 @@ func (c *containerServer) handleExecve(cmd *execCmd, msg *unixsocket.Msg) error
|
|||||||
DropCaps: true,
|
DropCaps: true,
|
||||||
SyncFunc: syncFunc,
|
SyncFunc: syncFunc,
|
||||||
Credential: cred,
|
Credential: cred,
|
||||||
|
|
||||||
|
UnshareCgroupAfterSync: true,
|
||||||
}
|
}
|
||||||
// starts the runner, error is handled same as wait4 to make communication equal
|
// starts the runner, error is handled same as wait4 to make communication equal
|
||||||
pid, err := r.Start()
|
pid, err := r.Start()
|
||||||
|
|||||||
@ -34,6 +34,9 @@ type Builder struct {
|
|||||||
|
|
||||||
// CredGenerator defines a credential generator used to create new container
|
// CredGenerator defines a credential generator used to create new container
|
||||||
CredGenerator CredGenerator
|
CredGenerator CredGenerator
|
||||||
|
|
||||||
|
// Clone flags defines unshare clone flag to create container
|
||||||
|
CloneFlags uintptr
|
||||||
}
|
}
|
||||||
|
|
||||||
// CredGenerator generates uid / gid credential used by container
|
// CredGenerator generates uid / gid credential used by container
|
||||||
@ -130,13 +133,20 @@ func (b *Builder) Build() (Environment, error) {
|
|||||||
uidMap, gidMap = getIDMapping(&cred)
|
uidMap, gidMap = getIDMapping(&cred)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
var cloneFlag uintptr
|
||||||
|
if b.CloneFlags == 0 {
|
||||||
|
cloneFlag = forkexec.UnshareFlags
|
||||||
|
} else {
|
||||||
|
cloneFlag = b.CloneFlags & forkexec.UnshareFlags
|
||||||
|
}
|
||||||
|
|
||||||
r := &forkexec.Runner{
|
r := &forkexec.Runner{
|
||||||
Args: []string{os.Args[0], initArg},
|
Args: []string{os.Args[0], initArg},
|
||||||
Env: []string{PathEnv},
|
Env: []string{PathEnv},
|
||||||
ExecFile: execFile.Fd(),
|
ExecFile: execFile.Fd(),
|
||||||
Files: files,
|
Files: files,
|
||||||
WorkDir: containerWD,
|
WorkDir: containerWD,
|
||||||
CloneFlags: forkexec.UnshareFlags,
|
CloneFlags: cloneFlag,
|
||||||
Mounts: mounts,
|
Mounts: mounts,
|
||||||
HostName: containerName,
|
HostName: containerName,
|
||||||
DomainName: containerName,
|
DomainName: containerName,
|
||||||
|
|||||||
@ -49,3 +49,17 @@ var (
|
|||||||
Inheritable: 0,
|
Inheritable: 0,
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
_SECURE_NOROOT = 1 << iota
|
||||||
|
_SECURE_NOROOT_LOCKED
|
||||||
|
|
||||||
|
_SECURE_NO_SETUID_FIXUP
|
||||||
|
_SECURE_NO_SETUID_FIXUP_LOCKED
|
||||||
|
|
||||||
|
_SECURE_KEEP_CAPS
|
||||||
|
_SECURE_KEEP_CAPS_LOCKED
|
||||||
|
|
||||||
|
_SECURE_NO_CAP_AMBIENT_RAISE
|
||||||
|
_SECURE_NO_CAP_AMBIENT_RAISE_LOCKED
|
||||||
|
)
|
||||||
|
|||||||
@ -73,6 +73,13 @@ func forkAndExecInChild(r *Runner, argv0 *byte, argv, env []*byte, workdir, host
|
|||||||
goto childerror
|
goto childerror
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// keep capabilities through set_uid / set_gid calls (make sure we can use unshare cgroup), later dropped
|
||||||
|
_, _, err1 = syscall.RawSyscall(syscall.SYS_PRCTL, syscall.PR_SET_SECUREBITS,
|
||||||
|
_SECURE_KEEP_CAPS_LOCKED|_SECURE_NO_SETUID_FIXUP|_SECURE_NO_SETUID_FIXUP_LOCKED, 0)
|
||||||
|
if err1 != 0 {
|
||||||
|
goto childerror
|
||||||
|
}
|
||||||
|
|
||||||
// set the credential for the child process(exec_linux.go)
|
// set the credential for the child process(exec_linux.go)
|
||||||
if cred := r.Credential; cred != nil {
|
if cred := r.Credential; cred != nil {
|
||||||
ngroups := uintptr(len(cred.Groups))
|
ngroups := uintptr(len(cred.Groups))
|
||||||
@ -291,7 +298,13 @@ func forkAndExecInChild(r *Runner, argv0 *byte, argv, env []*byte, workdir, host
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Drop all capabilities
|
// Drop all capabilities
|
||||||
if r.DropCaps {
|
if (r.Credential != nil || r.DropCaps) && !r.UnshareCgroupAfterSync {
|
||||||
|
// make sure the children have no privilege at all
|
||||||
|
_, _, err1 = syscall.RawSyscall(syscall.SYS_PRCTL, syscall.PR_SET_SECUREBITS,
|
||||||
|
_SECURE_KEEP_CAPS_LOCKED|_SECURE_NO_SETUID_FIXUP|_SECURE_NO_SETUID_FIXUP_LOCKED|_SECURE_NOROOT|_SECURE_NOROOT_LOCKED, 0)
|
||||||
|
if err1 != 0 {
|
||||||
|
goto childerror
|
||||||
|
}
|
||||||
_, _, err1 = syscall.RawSyscall(syscall.SYS_CAPSET, uintptr(unsafe.Pointer(&dropCapHeader)), uintptr(unsafe.Pointer(&dropCapData)), 0)
|
_, _, err1 = syscall.RawSyscall(syscall.SYS_CAPSET, uintptr(unsafe.Pointer(&dropCapHeader)), uintptr(unsafe.Pointer(&dropCapData)), 0)
|
||||||
if err1 != 0 {
|
if err1 != 0 {
|
||||||
goto childerror
|
goto childerror
|
||||||
@ -311,6 +324,26 @@ func forkAndExecInChild(r *Runner, argv0 *byte, argv, env []*byte, workdir, host
|
|||||||
goto childerror
|
goto childerror
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// unshare cgroup namespace
|
||||||
|
if r.UnshareCgroupAfterSync {
|
||||||
|
r1, _, err1 = syscall.RawSyscall(syscall.SYS_UNSHARE, uintptr(unix.CLONE_NEWCGROUP), 0, 0)
|
||||||
|
if err1 != 0 {
|
||||||
|
goto childerror
|
||||||
|
}
|
||||||
|
if r.DropCaps || r.Credential != nil {
|
||||||
|
// make sure the children have no privilege at all
|
||||||
|
_, _, err1 = syscall.RawSyscall(syscall.SYS_PRCTL, syscall.PR_SET_SECUREBITS,
|
||||||
|
_SECURE_KEEP_CAPS_LOCKED|_SECURE_NO_SETUID_FIXUP|_SECURE_NO_SETUID_FIXUP_LOCKED|_SECURE_NOROOT|_SECURE_NOROOT_LOCKED, 0)
|
||||||
|
if err1 != 0 {
|
||||||
|
goto childerror
|
||||||
|
}
|
||||||
|
_, _, err1 = syscall.RawSyscall(syscall.SYS_CAPSET, uintptr(unsafe.Pointer(&dropCapHeader)), uintptr(unsafe.Pointer(&dropCapData)), 0)
|
||||||
|
if err1 != 0 {
|
||||||
|
goto childerror
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
_, _, err1 = syscall.RawSyscall(syscall.SYS_PTRACE, uintptr(syscall.PTRACE_TRACEME), 0, 0)
|
_, _, err1 = syscall.RawSyscall(syscall.SYS_PTRACE, uintptr(syscall.PTRACE_TRACEME), 0, 0)
|
||||||
if err1 != 0 {
|
if err1 != 0 {
|
||||||
goto childerror
|
goto childerror
|
||||||
@ -329,7 +362,7 @@ func forkAndExecInChild(r *Runner, argv0 *byte, argv, env []*byte, workdir, host
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Load seccomp, stop and wait for tracer
|
// Load seccomp, stop and wait for tracer
|
||||||
if r.Seccomp != nil {
|
if r.Seccomp != nil && (!r.UnshareCgroupAfterSync || r.Ptrace) {
|
||||||
// If execve is seccomp trapped, then tracee stop is necessary
|
// If execve is seccomp trapped, then tracee stop is necessary
|
||||||
// otherwise execve will fail due to ENOSYS
|
// otherwise execve will fail due to ENOSYS
|
||||||
// Do getpid and kill to send SYS_KILL to self
|
// Do getpid and kill to send SYS_KILL to self
|
||||||
@ -354,6 +387,33 @@ func forkAndExecInChild(r *Runner, argv0 *byte, argv, env []*byte, workdir, host
|
|||||||
if r1 == 0 || err1 != 0 {
|
if r1 == 0 || err1 != 0 {
|
||||||
goto childerror
|
goto childerror
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// unshare cgroup namespace
|
||||||
|
if r.UnshareCgroupAfterSync {
|
||||||
|
r1, _, err1 = syscall.RawSyscall(syscall.SYS_UNSHARE, uintptr(unix.CLONE_NEWCGROUP), 0, 0)
|
||||||
|
if err1 != 0 {
|
||||||
|
goto childerror
|
||||||
|
}
|
||||||
|
if r.DropCaps || r.Credential != nil {
|
||||||
|
// make sure the children have no privilege at all
|
||||||
|
_, _, err1 = syscall.RawSyscall(syscall.SYS_PRCTL, syscall.PR_SET_SECUREBITS,
|
||||||
|
_SECURE_KEEP_CAPS_LOCKED|_SECURE_NO_SETUID_FIXUP|_SECURE_NO_SETUID_FIXUP_LOCKED|_SECURE_NOROOT|_SECURE_NOROOT_LOCKED, 0)
|
||||||
|
if err1 != 0 {
|
||||||
|
goto childerror
|
||||||
|
}
|
||||||
|
_, _, err1 = syscall.RawSyscall(syscall.SYS_CAPSET, uintptr(unsafe.Pointer(&dropCapHeader)), uintptr(unsafe.Pointer(&dropCapData)), 0)
|
||||||
|
if err1 != 0 {
|
||||||
|
goto childerror
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if r.Seccomp != nil {
|
||||||
|
// Load seccomp filter
|
||||||
|
_, _, err1 = syscall.RawSyscall(unix.SYS_SECCOMP, SECCOMP_SET_MODE_FILTER, SECCOMP_FILTER_FLAG_TSYNC, uintptr(unsafe.Pointer(r.Seccomp)))
|
||||||
|
if err1 != 0 {
|
||||||
|
goto childerror
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Enable ptrace if no seccomp is needed
|
// Enable ptrace if no seccomp is needed
|
||||||
|
|||||||
@ -96,4 +96,8 @@ type Runner struct {
|
|||||||
// parent will signal child to stop and report the error
|
// parent will signal child to stop and report the error
|
||||||
// SyncFunc is called right before execve, thus it could track cpu more accurately
|
// SyncFunc is called right before execve, thus it could track cpu more accurately
|
||||||
SyncFunc func(int) error
|
SyncFunc func(int) error
|
||||||
|
|
||||||
|
// UnshareCgroupAfterSync specifies whether to unshare cgroup namespace after
|
||||||
|
// sync (the syncFunc might be add the child to the cgroup)
|
||||||
|
UnshareCgroupAfterSync bool
|
||||||
}
|
}
|
||||||
|
|||||||
@ -99,7 +99,7 @@ func (b *Builder) WithProc() *Builder {
|
|||||||
Source: "proc",
|
Source: "proc",
|
||||||
Target: "proc",
|
Target: "proc",
|
||||||
FsType: "proc",
|
FsType: "proc",
|
||||||
Flags: unix.MS_NOSUID,
|
Flags: unix.MS_NOSUID | unix.MS_RDONLY,
|
||||||
})
|
})
|
||||||
return b
|
return b
|
||||||
}
|
}
|
||||||
|
|||||||
@ -20,6 +20,8 @@ func (r *Runner) Run(c context.Context) <-chan runner.Result {
|
|||||||
Seccomp: r.Seccomp.SockFprog(),
|
Seccomp: r.Seccomp.SockFprog(),
|
||||||
Ptrace: true,
|
Ptrace: true,
|
||||||
SyncFunc: r.SyncFunc,
|
SyncFunc: r.SyncFunc,
|
||||||
|
|
||||||
|
UnshareCgroupAfterSync: true,
|
||||||
}
|
}
|
||||||
|
|
||||||
th := &tracerHandler{
|
th := &tracerHandler{
|
||||||
|
|||||||
@ -46,6 +46,8 @@ func (r *Runner) trace(c context.Context) (result runner.Result) {
|
|||||||
PivotRoot: r.Root,
|
PivotRoot: r.Root,
|
||||||
DropCaps: true,
|
DropCaps: true,
|
||||||
SyncFunc: r.SyncFunc,
|
SyncFunc: r.SyncFunc,
|
||||||
|
|
||||||
|
UnshareCgroupAfterSync: true,
|
||||||
}
|
}
|
||||||
|
|
||||||
var (
|
var (
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user