Remove types pkg & update documents

- move types pkg into runner
- add documentations for cgroup & container
- add benchmakr for cgroup & container
This commit is contained in:
criyle 2020-03-02 03:14:13 -05:00
parent e8260dde37
commit 4f2257a187
28 changed files with 469 additions and 249 deletions

View File

@ -43,7 +43,7 @@ Default file access syscall check:
2. Use Linux Control Groups to limit & acct CPU & memory (elimilate wait4.rusage)
3. Container tech with execveat memfd, sethostname, setdomainname
## Design (in progress)
## Design
### Result Status
@ -86,7 +86,7 @@ Configured runner to run the program. `Context` is used to cancel (control time
``` go
type Runner interface {
Run(context.Context) <-chan types.Result
Run(context.Context) <-chan runner.Result
}
```
@ -144,7 +144,7 @@ type Environment interface {
Open([]OpenCmd) ([]*os.File, error)
Delete(p string) error
Reset() error
Execve(context.Context, ExecveParam) <-chan types.Result
Execve(context.Context, ExecveParam) <-chan runner.Result
Destroy() error
}
```
@ -170,7 +170,6 @@ type Environment interface {
- filehandler: an example implementation of UOJ file set
- unshare: wrapper to call forkexec and unshared namespaces
- ptracer: ptrace tracer and provides syscall trap filter context
- types: provides general res / result data structures
## Executable
@ -180,35 +179,50 @@ type Environment interface {
- config/config.go: all configs toward running specs (similar to UOJ)
## Benchmarks (MacOS docker amd64 / native arm64)
## Benchmarks
- 1ms / 2ms: fork, unshare pid / user / cgroup
- 4ms / 8ms: run inside pre-forked container
- 50ms / 25ms: unshare ipc / mount
- 100ms / 44ms: unshare pid & user & cgroup & mount & pivot root
- 400ms / 63ms: unshare net
- 800ms / 170ms: unshare all
- 880ms / 170ms: unshare all & pivot root
It seems unshare net or ipc takes time, maybe limits action by seccomp instead.
Pre-forked container also saves time for container creation / cleanup.
### ForkExec
```bash
$ go test -bench . -benchtime 10s
goos: linux
goarch: amd64
pkg: github.com/criyle/go-sandbox/pkg/forkexec
BenchmarkSimpleFork-4 12789 870486 ns/op
BenchmarkUnsharePid-4 13172 917304 ns/op
BenchmarkUnshareUser-4 13148 927952 ns/op
BenchmarkUnshareUts-4 13170 884606 ns/op
BenchmarkUnshareCgroup-4 13650 895186 ns/op
BenchmarkUnshareIpc-4 196 66418708 ns/op
BenchmarkUnshareMount-4 243 46957682 ns/op
BenchmarkUnshareNet-4 100 411869776 ns/op
BenchmarkFastUnshareMountPivot-4 120 107310917 ns/op
BenchmarkUnshareAll-4 100 837352275 ns/op
BenchmarkUnshareMountPivot-4 12 913099234 ns/op
BenchmarkSimpleFork-4 12409 996096 ns/op
BenchmarkUnsharePid-4 10000 1065168 ns/op
BenchmarkUnshareUser-4 10000 1061770 ns/op
BenchmarkUnshareUts-4 10000 1056558 ns/op
BenchmarkUnshareCgroup-4 10000 1049446 ns/op
BenchmarkUnshareIpc-4 709 16114052 ns/op
BenchmarkUnshareMount-4 745 16207754 ns/op
BenchmarkUnshareNet-4 3643 3492924 ns/op
BenchmarkFastUnshareMountPivot-4 612 20967318 ns/op
BenchmarkUnshareAll-4 837 14047995 ns/op
BenchmarkUnshareMountPivot-4 488 24198331 ns/op
PASS
ok github.com/criyle/go-sandbox/pkg/forkexec 300.744s
ok github.com/criyle/go-sandbox/pkg/forkexec 147.186s
```
### Container
```bash
$ go test -bench . -benchtime 10s
goos: linux
goarch: amd64
pkg: github.com/criyle/go-sandbox/container
BenchmarkContainer-4 5907 2062070 ns/op
PASS
ok github.com/criyle/go-sandbox/container 21.763s
```
### Cgroup
```bash
$ go test -bench . -benchtime 10s
goos: linux
goarch: amd64
pkg: github.com/criyle/go-sandbox/pkg/cgroup
BenchmarkCgroup-4 50283 245094 ns/op
PASS
ok github.com/criyle/go-sandbox/pkg/cgroup 14.744s
```

View File

@ -1,3 +1,4 @@
// Command runprog executes program defined restricted environment including seccomp-ptraced, namespaced and containerized.
package main
import (
@ -21,7 +22,6 @@ import (
"github.com/criyle/go-sandbox/runner/ptrace"
"github.com/criyle/go-sandbox/runner/ptrace/filehandler"
"github.com/criyle/go-sandbox/runner/unshare"
"github.com/criyle/go-sandbox/types"
)
const (
@ -108,24 +108,24 @@ func main() {
rt, err := start()
if rt == nil {
rt = &types.Result{
Status: types.StatusRunnerError,
rt = &runner.Result{
Status: runner.StatusRunnerError,
}
}
if err == nil && rt.Status != types.StatusNormal {
if err == nil && rt.Status != runner.StatusNormal {
err = rt.Status
}
debug("setupTime: ", rt.SetUpTime)
debug("runningTime: ", rt.RunningTime)
if err != nil {
debug(err)
c, ok := err.(types.Status)
c, ok := err.(runner.Status)
if !ok {
c = types.StatusRunnerError
c = runner.StatusRunnerError
}
// Handle fatal error from trace
fmt.Fprintf(f, "%d %d %d %d\n", getStatus(c), int(rt.Time/time.Millisecond), uint64(rt.Memory)>>10, rt.ExitStatus)
if c == types.StatusRunnerError {
if c == runner.StatusRunnerError {
os.Exit(1)
}
} else {
@ -138,17 +138,17 @@ type containerRunner struct {
container.ExecveParam
}
func (r *containerRunner) Run(c context.Context) <-chan types.Result {
func (r *containerRunner) Run(c context.Context) <-chan runner.Result {
return r.Environment.Execve(c, r.ExecveParam)
}
func start() (*types.Result, error) {
func start() (*runner.Result, error) {
var (
runner runner.Runner
cg *cgroup.CGroup
r runner.Runner
cg *cgroup.Cgroup
err error
execFile uintptr
rt types.Result
rt runner.Result
)
addRead := filehandler.GetExtraSet(addReadable, addRawReadable)
@ -225,9 +225,9 @@ func start() (*types.Result, error) {
actionDefault = seccomp.ActionTrace.WithReturnCode(seccomp.MsgDisallow)
}
limit := types.Limit{
limit := runner.Limit{
TimeLimit: time.Duration(timeLimit) * time.Second,
MemoryLimit: types.Size(memoryLimit << 20),
MemoryLimit: runner.Size(memoryLimit << 20),
}
if runt == "container" {
@ -250,12 +250,12 @@ func start() (*types.Result, error) {
if err != nil {
return nil, fmt.Errorf("failed to ping container: %v", err)
}
runner = &containerRunner{
r = &containerRunner{
Environment: m,
ExecveParam: container.ExecveParam{
Args: args,
Env: []string{pathEnv},
Fds: fds,
Files: fds,
ExecFile: execFile,
RLimits: rlims.PrepareRLimit(),
SyncFunc: syncFunc,
@ -279,7 +279,7 @@ func start() (*types.Result, error) {
if err != nil {
return nil, fmt.Errorf("cannot make rootfs mounts")
}
runner = &unshare.Runner{
r = &unshare.Runner{
Args: args,
Env: []string{pathEnv},
ExecFile: execFile,
@ -305,7 +305,7 @@ func start() (*types.Result, error) {
if err != nil {
return nil, fmt.Errorf("failed to create seccomp filter %v", err)
}
runner = &ptrace.Runner{
r = &ptrace.Runner{
Args: args,
Env: []string{pathEnv},
ExecFile: execFile,
@ -332,14 +332,14 @@ func start() (*types.Result, error) {
c, cancel := context.WithTimeout(context.Background(), time.Duration(int64(realTimeLimit)*int64(time.Second)))
defer cancel()
s := runner.Run(c)
s := r.Run(c)
rTime := time.Now()
select {
case <-sig:
cancel()
rt = <-s
rt.Status = types.StatusRunnerError
rt.Status = runner.StatusRunnerError
case rt = <-s:
}
@ -367,7 +367,7 @@ func start() (*types.Result, error) {
}
debug("cgroup: cpu: ", cpu, " memory: ", memory, "cache: ", cache)
rt.Time = time.Duration(cpu)
rt.Memory = types.Size(memory - cache)
rt.Memory = runner.Size(memory - cache)
debug("cgroup:", rt)
}
return &rt, nil
@ -394,21 +394,21 @@ const (
StatusFatal // 7
)
func getStatus(s types.Status) int {
func getStatus(s runner.Status) int {
switch s {
case types.StatusNormal:
case runner.StatusNormal:
return int(StatusNormal)
case types.StatusInvalid:
case runner.StatusInvalid:
return int(StatusInvalid)
case types.StatusTimeLimitExceeded:
case runner.StatusTimeLimitExceeded:
return int(StatusTLE)
case types.StatusMemoryLimitExceeded:
case runner.StatusMemoryLimitExceeded:
return int(StatusMLE)
case types.StatusOutputLimitExceeded:
case runner.StatusOutputLimitExceeded:
return int(StatusOLE)
case types.StatusDisallowedSyscall:
case runner.StatusDisallowedSyscall:
return int(StatusBan)
case types.StatusSignalled, types.StatusNonzeroExitStatus:
case runner.StatusSignalled, runner.StatusNonzeroExitStatus:
return int(StatusRE)
default:
return int(StatusFatal)

115
container/benchmark_test.go Normal file
View File

@ -0,0 +1,115 @@
package container
import (
"context"
"errors"
"io/ioutil"
"testing"
"github.com/criyle/go-sandbox/runner"
)
func init() {
Init()
}
func BenchmarkContainer(b *testing.B) {
tmpDir, err := ioutil.TempDir("", "")
if err != nil {
b.Error(err)
return
}
builder := &Builder{
Root: tmpDir,
}
m, err := builder.Build()
if err != nil {
b.Error(err)
return
}
b.Cleanup(func() {
m.Destroy()
})
b.ResetTimer()
for i := 0; i < b.N; i++ {
rt := m.Execve(context.TODO(), ExecveParam{
Args: []string{"/bin/echo"},
Env: []string{"PATH=/bin"},
})
r := <-rt
if r.Status != runner.StatusNormal {
b.Error(r.Status, r.Error)
return
}
}
}
func TestContainerSuccess(t *testing.T) {
m := getEnv(t)
if m == nil {
return
}
rt := m.Execve(context.TODO(), ExecveParam{
Args: []string{"/bin/echo"},
Env: []string{"PATH=/bin"},
})
r := <-rt
if r.Status != runner.StatusNormal {
t.Error(r.Status, r.Error)
return
}
}
func TestContainerNotExists(t *testing.T) {
m := getEnv(t)
if m == nil {
return
}
rt := m.Execve(context.TODO(), ExecveParam{
Args: []string{"not_exists"},
Env: []string{"PATH=/bin"},
})
r := <-rt
if r.Status != runner.StatusRunnerError {
t.Error(r.Status, r.Error)
return
}
}
func TestContainerSyncFuncFail(t *testing.T) {
m := getEnv(t)
if m == nil {
return
}
err := errors.New("test error")
rt := m.Execve(context.TODO(), ExecveParam{
Args: []string{"/bin/echo"},
Env: []string{"PATH=/bin"},
SyncFunc: func(pid int) error {
return err
},
})
r := <-rt
if r.Status != runner.StatusRunnerError {
t.Error(r.Status, r.Error)
return
}
}
func getEnv(t *testing.T) Environment {
tmpDir, err := ioutil.TempDir("", "")
if err != nil {
t.Error(err)
return nil
}
builder := &Builder{
Root: tmpDir,
}
m, err := builder.Build()
if err != nil {
t.Error(err)
return nil
}
return m
}

View File

@ -7,7 +7,7 @@ import (
"github.com/criyle/go-sandbox/pkg/forkexec"
"github.com/criyle/go-sandbox/pkg/unixsocket"
"github.com/criyle/go-sandbox/types"
"github.com/criyle/go-sandbox/runner"
)
func (c *containerServer) handleExecve(cmd *execCmd, msg *unixsocket.Msg) error {
@ -118,14 +118,14 @@ func (c *containerServer) handleExecve(cmd *execCmd, msg *unixsocket.Msg) error
if err != nil {
c.sendErrorReply("execve: wait4 %v", err)
} else {
status := types.StatusNormal
status := runner.StatusNormal
userTime := time.Duration(rusage.Utime.Nano()) // ns
userMem := types.Size(rusage.Maxrss << 10) // bytes
userMem := runner.Size(rusage.Maxrss << 10) // bytes
switch {
case wstatus.Exited():
exitStatus := wstatus.ExitStatus()
if exitStatus != 0 {
status = types.StatusNonzeroExitStatus
status = runner.StatusNonzeroExitStatus
}
c.sendReply(&reply{
ExecReply: &execReply{
@ -140,13 +140,13 @@ func (c *containerServer) handleExecve(cmd *execCmd, msg *unixsocket.Msg) error
switch wstatus.Signal() {
// kill signal treats as TLE
case syscall.SIGXCPU, syscall.SIGKILL:
status = types.StatusTimeLimitExceeded
status = runner.StatusTimeLimitExceeded
case syscall.SIGXFSZ:
status = types.StatusOutputLimitExceeded
status = runner.StatusOutputLimitExceeded
case syscall.SIGSYS:
status = types.StatusDisallowedSyscall
status = runner.StatusDisallowedSyscall
default:
status = types.StatusSignalled
status = runner.StatusSignalled
}
c.sendReply(&reply{
ExecReply: &execReply{

View File

@ -12,33 +12,33 @@
// Host to container communication protocol is single threaded and always initiated by
// the host:
//
// - ping (alive check):
// - reply: pong
// - ping (alive check):
// - reply: pong
//
// - conf (set configuration):
// - reply pong
// - conf (set configuration):
// - reply pong
//
// - open (open files in given mode inside container):
// - send: []OpenCmd
// - reply: "success", file fds / "error"
// - open (open files in given mode inside container):
// - send: []OpenCmd
// - reply: "success", file fds / "error"
//
// - delete (unlink file / rmdir dir inside container):
// - send: path
// - reply: "finished" / "error"
// - delete (unlink file / rmdir dir inside container):
// - send: path
// - reply: "finished" / "error"
//
// - reset (clean up container for later use (clear workdir / tmp)):
// - send:
// - reply: "success"
// - reset (clean up container for later use (clear workdir / tmp)):
// - send:
// - reply: "success"
//
// - execve: (execute file inside container):
// - send: argv, env, rLimits, fds
// - reply:
// - success: "success", pid
// - failed: "failed"
// - send (success): "init_finished" (as cmd)
// - reply: "finished" / send: "kill" (as cmd)
// - send: "kill" (as cmd) / reply: "finished"
// - reply:
// - execve: (execute file inside container):
// - send: argv, env, rLimits, fds
// - reply:
// - success: "success", pid
// - failed: "failed"
// - send (success): "init_finished" (as cmd)
// - reply: "finished" / send: "kill" (as cmd)
// - send: "kill" (as cmd) / reply: "finished"
// - reply:
//
// Any socket related error will cause the container exit with all process inside container
package container

View File

@ -8,10 +8,9 @@ import (
"syscall"
"github.com/criyle/go-sandbox/pkg/forkexec"
"github.com/criyle/go-sandbox/pkg/memfd"
"github.com/criyle/go-sandbox/pkg/mount"
"github.com/criyle/go-sandbox/pkg/unixsocket"
"github.com/criyle/go-sandbox/types"
"github.com/criyle/go-sandbox/runner"
"golang.org/x/sys/unix"
)
@ -49,7 +48,7 @@ type Environment interface {
Open([]OpenCmd) ([]*os.File, error)
Delete(p string) error
Reset() error
Execve(context.Context, ExecveParam) <-chan types.Result
Execve(context.Context, ExecveParam) <-chan runner.Result
Destroy() error
}
@ -196,21 +195,12 @@ func (b *Builder) exec() (*os.File, error) {
return OpenCurrentExec()
}
// OpenCurrentExec opens current executable and dup and seal it to memfd
// OpenCurrentExec opens current executable (/proc/self/exe)
func OpenCurrentExec() (*os.File, error) {
self, err := os.Open(currentExec)
if err != nil {
return nil, fmt.Errorf("failed to open %v: %v", currentExec, err)
}
defer self.Close()
execFile, err := memfd.DupToMemfd("init", self)
if err != nil {
return nil, fmt.Errorf("failed to create memfd: %v", err)
}
return execFile, nil
return os.Open(currentExec)
}
// newPassCredSocketPair creates socket pair and let the first socket to receive credential information
func newPassCredSocketPair() (*unixsocket.Socket, *unixsocket.Socket, error) {
ins, outs, err := unixsocket.NewSocketPair()
if err != nil {
@ -221,11 +211,6 @@ func newPassCredSocketPair() (*unixsocket.Socket, *unixsocket.Socket, error) {
outs.Close()
return nil, nil, err
}
if err = outs.SetPassCred(1); err != nil {
ins.Close()
outs.Close()
return nil, nil, err
}
return ins, outs, nil
}

View File

@ -7,37 +7,42 @@ import (
"github.com/criyle/go-sandbox/pkg/rlimit"
"github.com/criyle/go-sandbox/pkg/unixsocket"
"github.com/criyle/go-sandbox/types"
"github.com/criyle/go-sandbox/runner"
)
// ExecveParam is parameters to run process inside container
type ExecveParam struct {
// Args holds command line arguments
Args []string
Env []string
Fds []uintptr
// fd parameter of fexecve
// Env specifies the environment of the process
Env []string
// Files specifies file descriptors for the child process
Files []uintptr
// ExecFile specifies file descriptor for executable file using fexecve
ExecFile uintptr
// POSIX Resource limit set by set rlimit
// RLimits specifies POSIX Resource limit through setrlimit
RLimits []rlimit.RLimit
// SyncFunc called with pid before execve (for adding the process to cgroups)
// SyncFunc calls with pid just before execve (for attach the process to cgroups)
SyncFunc func(pid int) error
}
// Execve runs process inside container. It accepts context cancelation as time limit exceeded.
func (c *container) Execve(ctx context.Context, param ExecveParam) <-chan types.Result {
func (c *container) Execve(ctx context.Context, param ExecveParam) <-chan runner.Result {
c.mu.Lock()
sTime := time.Now()
// make sure goroutine not leaked (blocked) even if result is not consumed
result := make(chan types.Result, 1)
result := make(chan runner.Result, 1)
errResult := func(f string, v ...interface{}) <-chan types.Result {
result <- types.Result{
Status: types.StatusRunnerError,
errResult := func(f string, v ...interface{}) <-chan runner.Result {
result <- runner.Result{
Status: runner.StatusRunnerError,
Error: fmt.Sprintf(f, v...),
}
return result
@ -48,7 +53,7 @@ func (c *container) Execve(ctx context.Context, param ExecveParam) <-chan types.
if param.ExecFile > 0 {
files = append(files, int(param.ExecFile))
}
files = append(files, uintptrSliceToInt(param.Fds)...)
files = append(files, uintptrSliceToInt(param.Files)...)
msg := &unixsocket.Msg{
Fds: files,
}
@ -110,28 +115,28 @@ func (c *container) Execve(ctx context.Context, param ExecveParam) <-chan types.
// handle potential error
if err != nil {
result <- types.Result{
Status: types.StatusRunnerError,
result <- runner.Result{
Status: runner.StatusRunnerError,
Error: err.Error(),
}
return
}
if reply2.Error != nil {
result <- types.Result{
Status: types.StatusRunnerError,
result <- runner.Result{
Status: runner.StatusRunnerError,
Error: reply2.Error.Error(),
}
return
}
if reply2.ExecReply == nil {
result <- types.Result{
Status: types.StatusRunnerError,
result <- runner.Result{
Status: runner.StatusRunnerError,
Error: "execve: no reply received",
}
return
}
// emit result after all communication finish
result <- types.Result{
result <- runner.Result{
Status: reply2.ExecReply.Status,
ExitStatus: reply2.ExecReply.ExitStatus,
Time: reply2.ExecReply.Time,

View File

@ -6,7 +6,7 @@ import (
"time"
"github.com/criyle/go-sandbox/pkg/rlimit"
"github.com/criyle/go-sandbox/types"
"github.com/criyle/go-sandbox/runner"
)
// cmd is the control message send into container
@ -64,9 +64,9 @@ type errorReply struct {
// execReply stores execve result
type execReply struct {
ExitStatus int // waitpid exit status
Status types.Status // return status
Status runner.Status // return status
Time time.Duration // waitpid user CPU (ns)
Memory types.Size // waitpid user memory (byte)
Memory runner.Size // waitpid user memory (byte)
}
func (e *errorReply) Error() string {

View File

@ -0,0 +1,66 @@
package cgroup
import "testing"
func BenchmarkCgroup(b *testing.B) {
builder, err := NewBuilder("benchmark").WithCPUAcct().WithMemory().WithPids().FilterByEnv()
if err != nil {
b.Error(err)
return
}
b.ResetTimer()
for i := 0; i < b.N; i++ {
cg, err := builder.Build()
if err != nil {
b.Error(err)
return
}
if err := cg.SetMemoryLimitInBytes(4096); err != nil {
b.Error(err)
return
}
if err := cg.SetPidsMax(1); err != nil {
b.Error(err)
return
}
if _, err := cg.CpuacctUsage(); err != nil {
b.Error(err)
return
}
if _, err := cg.MemoryMaxUsageInBytes(); err != nil {
b.Error(err)
return
}
cg.Destroy()
}
}
func TestCgroup(t *testing.T) {
builder, err := NewBuilder("test").WithCPUAcct().WithMemory().WithPids().FilterByEnv()
if err != nil {
t.Error(err)
return
}
cg, err := builder.Build()
if err != nil {
t.Error(err)
return
}
if err := cg.SetMemoryLimitInBytes(4096); err != nil {
t.Error(err)
return
}
if err := cg.SetPidsMax(1); err != nil {
t.Error(err)
return
}
if _, err := cg.CpuacctUsage(); err != nil {
t.Error(err)
return
}
if _, err := cg.MemoryMaxUsageInBytes(); err != nil {
t.Error(err)
return
}
cg.Destroy()
}

View File

@ -6,20 +6,14 @@ import (
"os"
)
// additional ideas:
// cpu share(not used): cpu.share
// reclaim pages from old process: memory.force_empty
// (tasks kill are managed out of cgroup as freeze takes some time)
// freeze: freezer.state
// CGroup is the combination of sub-cgroups
type CGroup struct {
// Cgroup is the combination of sub-cgroups
type Cgroup struct {
prefix string
cpuacct, memory, pids *SubCGroup
cpuacct, memory, pids *SubCgroup
}
// Build creates new cgrouup directories
func (b *Builder) Build() (cg *CGroup, err error) {
func (b *Builder) Build() (cg *Cgroup, err error) {
var (
cpuacctPath, memoryPath, pidsPath string
)
@ -32,31 +26,31 @@ func (b *Builder) Build() (cg *CGroup, err error) {
}
}()
if b.CPUAcct {
if cpuacctPath, err = CreateSubCGroupPath("cpuacct", b.Prefix); err != nil {
if cpuacctPath, err = CreateSubCgroupPath("cpuacct", b.Prefix); err != nil {
return
}
}
if b.Memory {
if memoryPath, err = CreateSubCGroupPath("memory", b.Prefix); err != nil {
if memoryPath, err = CreateSubCgroupPath("memory", b.Prefix); err != nil {
return
}
}
if b.Pids {
if pidsPath, err = CreateSubCGroupPath("pids", b.Prefix); err != nil {
if pidsPath, err = CreateSubCgroupPath("pids", b.Prefix); err != nil {
return
}
}
return &CGroup{
return &Cgroup{
prefix: b.Prefix,
cpuacct: NewSubCGroup(cpuacctPath),
memory: NewSubCGroup(memoryPath),
pids: NewSubCGroup(pidsPath),
cpuacct: NewSubCgroup(cpuacctPath),
memory: NewSubCgroup(memoryPath),
pids: NewSubCgroup(pidsPath),
}, nil
}
// AddProc writes cgroup.procs to all sub-cgroup
func (c *CGroup) AddProc(pid int) error {
func (c *Cgroup) AddProc(pid int) error {
if err := c.cpuacct.WriteUint(cgroupProcs, uint64(pid)); err != nil {
return err
}
@ -69,8 +63,8 @@ func (c *CGroup) AddProc(pid int) error {
return nil
}
// Destroy removes dir for sub-cggroup, errors are ignored if remove one failed
func (c *CGroup) Destroy() error {
// Destroy removes dir for sub-cgroup, errors are ignored if remove one failed
func (c *Cgroup) Destroy() error {
var err1 error
if err := remove(c.cpuacct.path); err != nil {
err1 = err
@ -85,37 +79,37 @@ func (c *CGroup) Destroy() error {
}
// CpuacctUsage read cpuacct.usage in ns
func (c *CGroup) CpuacctUsage() (uint64, error) {
func (c *Cgroup) CpuacctUsage() (uint64, error) {
return c.cpuacct.ReadUint("cpuacct.usage")
}
// MemoryMaxUsageInBytes read memory.max_usage_in_bytes
func (c *CGroup) MemoryMaxUsageInBytes() (uint64, error) {
func (c *Cgroup) MemoryMaxUsageInBytes() (uint64, error) {
return c.memory.ReadUint("memory.max_usage_in_bytes")
}
// SetMemoryLimitInBytes write memory.limit_in_bytes
func (c *CGroup) SetMemoryLimitInBytes(i uint64) error {
func (c *Cgroup) SetMemoryLimitInBytes(i uint64) error {
return c.memory.WriteUint("memory.limit_in_bytes", i)
}
// SetPidsMax write pids.max
func (c *CGroup) SetPidsMax(i uint64) error {
func (c *Cgroup) SetPidsMax(i uint64) error {
return c.pids.WriteUint("pids.max", i)
}
// SetCpuacctUsage write cpuacct.usage in ns
func (c *CGroup) SetCpuacctUsage(i uint64) error {
func (c *Cgroup) SetCpuacctUsage(i uint64) error {
return c.cpuacct.WriteUint("cpuacct.usage", i)
}
// SetMemoryMaxUsageInBytes write cpuacct.usage in ns
func (c *CGroup) SetMemoryMaxUsageInBytes(i uint64) error {
func (c *Cgroup) SetMemoryMaxUsageInBytes(i uint64) error {
return c.memory.WriteUint("memory.max_usage_in_bytes", i)
}
// FindMemoryStatProperty find certain property from memory.stat
func (c *CGroup) FindMemoryStatProperty(prop string) (uint64, error) {
func (c *Cgroup) FindMemoryStatProperty(prop string) (uint64, error) {
content, err := c.memory.ReadFile("memory.stat")
if err != nil {
return 0, err

View File

@ -1,6 +1,17 @@
// Package cgroup provices builder to create multiple different cgroup-v1 sub groups
// under systemd defined path (i.e. /sys/fs/cgroup).
// under systemd defined mount path (i.e.,sys/fs/cgroup).
//
// current available cgroups are cpuacct, memory, pids
// not available: cpu, cpuset, devices, freezer, net_cls, blkio, perf_event, net_prio, huge_tlb, rdma
// Current available:
// cpuacct
// memory
// pids
//
// Current not avaliable: cpu, cpuset, devices, freezer, net_cls, blkio, perf_event, net_prio, huge_tlb, rdma
//
// Additional ideas:
//
// cpu share(not used): cpu.share
// reclaim pages from old process: memory.force_empty
// (tasks kill are managed out of cgroup as freeze takes some time)
// freeze: freezer.state
package cgroup

View File

@ -9,31 +9,29 @@ import (
"syscall"
)
// SubCGroup is the sub-cgroup
type SubCGroup struct {
// SubCgroup is the accessor for single cgroup resource with given path
type SubCgroup struct {
path string
}
// ErrNotInitialized returned when trying to read from not initialized cgroup
var ErrNotInitialized = errors.New("cgroup was not initialized")
// NewSubCGroup creates a sug CGroup
func NewSubCGroup(p string) *SubCGroup {
return &SubCGroup{
path: p,
}
// NewSubCgroup creates a cgroup accessor with given path (path needs to be created in advance)
func NewSubCgroup(p string) *SubCgroup {
return &SubCgroup{path: p}
}
// WriteUint writes uint64 into given file
func (c *SubCGroup) WriteUint(filename string, i uint64) error {
func (c *SubCgroup) WriteUint(filename string, i uint64) error {
if c.path == "" {
return nil
}
return c.WriteFile(filename, []byte(strconv.FormatUint(i, 10)))
}
// ReadUint read uint64 into given file
func (c *SubCGroup) ReadUint(filename string) (uint64, error) {
// ReadUint read uint64 from given file
func (c *SubCgroup) ReadUint(filename string) (uint64, error) {
if c.path == "" {
return 0, ErrNotInitialized
}
@ -50,7 +48,7 @@ func (c *SubCGroup) ReadUint(filename string) (uint64, error) {
// WriteFile writes cgroup file and handles potential EINTR error while writes to
// the slow device (cgroup)
func (c *SubCGroup) WriteFile(name string, content []byte) error {
func (c *SubCgroup) WriteFile(name string, content []byte) error {
p := path.Join(c.path, name)
err := ioutil.WriteFile(p, content, 0664)
for err != nil && errors.Is(err, syscall.EINTR) {
@ -61,7 +59,7 @@ func (c *SubCGroup) WriteFile(name string, content []byte) error {
// ReadFile reads cgroup file and handles potential EINTR error while read to
// the slow device (cgroup)
func (c *SubCGroup) ReadFile(name string) ([]byte, error) {
func (c *SubCgroup) ReadFile(name string) ([]byte, error) {
p := path.Join(c.path, name)
data, err := ioutil.ReadFile(p)
for err != nil && errors.Is(err, syscall.EINTR) {

View File

@ -8,7 +8,7 @@ import (
"strings"
)
// EnsureDirExists creates dir if not exists
// EnsureDirExists creates directories if the path not exists
func EnsureDirExists(path string) error {
if _, err := os.Stat(path); os.IsNotExist(err) {
return os.Mkdir(path, os.ModePerm)
@ -16,8 +16,8 @@ func EnsureDirExists(path string) error {
return nil
}
// CreateSubCGroupPath creates path for sub-cgroup
func CreateSubCGroupPath(group, prefix string) (string, error) {
// CreateSubCgroupPath creates path for sub-cgroup with given group and prefix
func CreateSubCgroupPath(group, prefix string) (string, error) {
base := path.Join(basePath, group, prefix)
EnsureDirExists(base)
return ioutil.TempDir(base, "")

View File

@ -6,7 +6,7 @@ import (
"strings"
"syscall"
"github.com/criyle/go-sandbox/types"
"github.com/criyle/go-sandbox/runner"
)
// RLimits defines the rlimit applied by setrlimit syscall to traced process
@ -88,7 +88,7 @@ func (r RLimit) String() string {
case syscall.RLIMIT_AS:
t = "AddressSpace"
}
return fmt.Sprintf("%s[%v:%v]", t, types.Size(r.Rlim.Cur), types.Size(r.Rlim.Max))
return fmt.Sprintf("%s[%v:%v]", t, runner.Size(r.Rlim.Cur), runner.Size(r.Rlim.Max))
}
func (r RLimits) String() string {

View File

@ -1,6 +1,6 @@
package ptracer
import "github.com/criyle/go-sandbox/types"
import "github.com/criyle/go-sandbox/runner"
// TraceAction defines the action returned by TraceHandle
type TraceAction int
@ -18,7 +18,7 @@ const (
type Tracer struct {
Handler
Runner
types.Limit
runner.Limit
}
// Runner represents the process runner

View File

@ -9,12 +9,12 @@ import (
unix "golang.org/x/sys/unix"
"github.com/criyle/go-sandbox/pkg/seccomp"
"github.com/criyle/go-sandbox/types"
"github.com/criyle/go-sandbox/runner"
)
// Trace starts new goroutine and trace runner with ptrace
func (t *Tracer) Trace(c context.Context) <-chan types.Result {
result := make(chan types.Result, 1)
func (t *Tracer) Trace(c context.Context) <-chan runner.Result {
result := make(chan runner.Result, 1)
go func() {
result <- t.TraceRun(c)
}()
@ -23,9 +23,9 @@ func (t *Tracer) Trace(c context.Context) <-chan types.Result {
// TraceRun start and traces all child process by runner in the calling goroutine
// parameter done used to cancel work, start is used notify child starts
func (t *Tracer) TraceRun(c context.Context) (result types.Result) {
func (t *Tracer) TraceRun(c context.Context) (result runner.Result) {
var (
status = types.StatusNormal
status = runner.StatusNormal
wstatus unix.WaitStatus // wait4 wait status
rusage unix.Rusage // wait4 rusage
traced = make(map[int]bool) // store all process that have set ptrace options
@ -44,7 +44,7 @@ func (t *Tracer) TraceRun(c context.Context) (result types.Result) {
t.Handler.Debug("tracer started: ", pgid, err)
if err != nil {
t.Handler.Debug("start tracee failed: ", err)
result.Status = types.StatusRunnerError
result.Status = runner.StatusRunnerError
result.Error = err.Error()
return
}
@ -63,7 +63,7 @@ func (t *Tracer) TraceRun(c context.Context) (result types.Result) {
defer func() {
if err := recover(); err != nil {
t.Handler.Debug("panic: ", err)
result.Status = types.StatusRunnerError
result.Status = runner.StatusRunnerError
result.Error = fmt.Sprintf("%v", err)
}
// kill all tracee upon return
@ -88,7 +88,7 @@ func (t *Tracer) TraceRun(c context.Context) (result types.Result) {
}
if err != nil {
t.Handler.Debug("wait4 failed: ", err)
result.Status = types.StatusRunnerError
result.Status = runner.StatusRunnerError
result.Error = err.Error()
return
}
@ -97,21 +97,21 @@ func (t *Tracer) TraceRun(c context.Context) (result types.Result) {
if pid == pgid {
// update resource usage and check against limits
userTime := time.Duration(rusage.Utime.Nano()) // ns
userMem := types.Size(rusage.Maxrss << 10) // bytes
userMem := runner.Size(rusage.Maxrss << 10) // bytes
// check tle / mle
if userTime > t.Limit.TimeLimit {
status = types.StatusTimeLimitExceeded
status = runner.StatusTimeLimitExceeded
}
if userMem > t.Limit.MemoryLimit {
status = types.StatusMemoryLimitExceeded
status = runner.StatusMemoryLimitExceeded
}
result = types.Result{
result = runner.Result{
Status: status,
Time: userTime,
Memory: userMem,
}
if status != types.StatusNormal {
if status != runner.StatusNormal {
return
}
}
@ -125,13 +125,13 @@ func (t *Tracer) TraceRun(c context.Context) (result types.Result) {
if execved {
result.ExitStatus = wstatus.ExitStatus()
if result.ExitStatus == 0 {
result.Status = types.StatusNormal
result.Status = runner.StatusNormal
} else {
result.Status = types.StatusNonzeroExitStatus
result.Status = runner.StatusNonzeroExitStatus
}
return
}
result.Status = types.StatusRunnerError
result.Status = runner.StatusRunnerError
result.Error = "child process exit before execve"
return
}
@ -143,13 +143,13 @@ func (t *Tracer) TraceRun(c context.Context) (result types.Result) {
delete(traced, pid)
switch sig {
case unix.SIGXCPU, unix.SIGKILL:
status = types.StatusTimeLimitExceeded
status = runner.StatusTimeLimitExceeded
case unix.SIGXFSZ:
status = types.StatusOutputLimitExceeded
status = runner.StatusOutputLimitExceeded
case unix.SIGSYS:
status = types.StatusDisallowedSyscall
status = runner.StatusDisallowedSyscall
default:
status = types.StatusSignalled
status = runner.StatusSignalled
}
result.Status = status
result.ExitStatus = int(sig)
@ -165,7 +165,7 @@ func (t *Tracer) TraceRun(c context.Context) (result types.Result) {
// Ptrace set option valid if the tracee is stopped
err = setPtraceOption(pid)
if err != nil {
result.Status = types.StatusRunnerError
result.Status = runner.StatusRunnerError
result.Error = err.Error()
return
}
@ -179,7 +179,7 @@ func (t *Tracer) TraceRun(c context.Context) (result types.Result) {
// give the customized handle for syscall
err := t.handleTrap(pid)
if err != nil {
result.Status = types.StatusDisallowedSyscall
result.Status = runner.StatusDisallowedSyscall
result.Error = err.Error()
return
}
@ -209,11 +209,11 @@ func (t *Tracer) TraceRun(c context.Context) (result types.Result) {
// check if cpu rlimit hit
switch stopSig {
case unix.SIGXCPU:
status = types.StatusTimeLimitExceeded
status = runner.StatusTimeLimitExceeded
case unix.SIGXFSZ:
status = types.StatusOutputLimitExceeded
status = runner.StatusOutputLimitExceeded
}
if status != types.StatusNormal {
if status != runner.StatusNormal {
result.Status = status
return
}
@ -263,7 +263,7 @@ func (t *Tracer) handleTrap(pid int) error {
return ctx.skipSyscall()
case TraceKill:
return types.StatusDisallowedSyscall
return runner.StatusDisallowedSyscall
}
}

33
runner/doc.go Normal file
View File

@ -0,0 +1,33 @@
// Package runner provides common interface for program runner together with
// common types including Result, Limit, Size and Status.
//
// Status
//
// Status defines the program running result status including
// Normal
// Program Error
// Resource Limit Exceeded (Time / Memory / Output)
// Unauthorized Access (Disallowed Syscall)
// Runtime Error (Signaled / Nonzero Exit Status)
// Program Runner Error
//
// Size
//
// Size defines size in bytes, underlying type is uint64 so it
// is effective to store up to EiB of size
//
// Limit
//
// Limit defines Time & Memory restriction on program runner
//
// Result
//
// Result defines program running result including
// Status, ExitStatus, Detailed Error, Time, Memory,
// SetupTime and RunningTime (in real clock)
//
// Runner
//
// General interface to run a program, including a context
// for canclation
package runner

View File

@ -1,4 +1,4 @@
package types
package runner
import (
"fmt"

View File

@ -8,7 +8,7 @@ import (
"github.com/criyle/go-sandbox/pkg/seccomp/libseccomp"
"github.com/criyle/go-sandbox/ptracer"
"github.com/criyle/go-sandbox/types"
"github.com/criyle/go-sandbox/runner"
)
type tracerHandler struct {
@ -127,7 +127,7 @@ func (h *tracerHandler) GetSyscallName(ctx *ptracer.Context) (string, error) {
func (h *tracerHandler) HandlerDisallow(name string) error {
if !h.Unsafe {
return types.StatusDisallowedSyscall
return runner.StatusDisallowedSyscall
}
return nil
}

View File

@ -5,11 +5,11 @@ import (
"github.com/criyle/go-sandbox/pkg/forkexec"
"github.com/criyle/go-sandbox/ptracer"
"github.com/criyle/go-sandbox/types"
"github.com/criyle/go-sandbox/runner"
)
// Run starts the tracing process
func (r *Runner) Run(c context.Context) <-chan types.Result {
func (r *Runner) Run(c context.Context) <-chan runner.Result {
ch := &forkexec.Runner{
Args: r.Args,
Env: r.Env,
@ -33,6 +33,5 @@ func (r *Runner) Run(c context.Context) <-chan types.Result {
Runner: ch,
Limit: r.Limit,
}
rt := tracer.Trace(c)
return rt
return tracer.Trace(c)
}

View File

@ -6,7 +6,7 @@ import (
"github.com/criyle/go-sandbox/pkg/rlimit"
"github.com/criyle/go-sandbox/pkg/seccomp"
"github.com/criyle/go-sandbox/ptracer"
"github.com/criyle/go-sandbox/types"
"github.com/criyle/go-sandbox/runner"
)
// Runner defines the spec to run a program safely by ptracer
@ -27,7 +27,7 @@ type Runner struct {
RLimits []rlimit.RLimit
// Res limit enforced by tracer
Limit types.Limit
Limit runner.Limit
// Defines seccomp filter for the ptrace runner
// file access syscalls need to set as ActionTrace

View File

@ -1,4 +1,4 @@
package types
package runner
import (
"fmt"

View File

@ -2,11 +2,9 @@ package runner
import (
"context"
"github.com/criyle/go-sandbox/types"
)
// Runner interface defines method to start running
type Runner interface {
Run(context.Context) <-chan types.Result
Run(context.Context) <-chan Result
}

View File

@ -1,4 +1,4 @@
package types
package runner
import "fmt"

View File

@ -1,4 +1,4 @@
package types
package runner
// Status is the result Status
type Status int

3
runner/unshare/doc.go Normal file
View File

@ -0,0 +1,3 @@
// Package unshare implements runner that uses Linux unshare syscall & mount namespace & rlimit
// to restrict program access
package unshare

View File

@ -11,7 +11,7 @@ import (
"golang.org/x/sys/unix"
"github.com/criyle/go-sandbox/pkg/forkexec"
"github.com/criyle/go-sandbox/types"
"github.com/criyle/go-sandbox/runner"
)
const (
@ -20,7 +20,16 @@ const (
)
// Run starts the unshared process
func (r *Runner) Run(c context.Context) <-chan types.Result {
func (r *Runner) Run(c context.Context) <-chan runner.Result {
result := make(chan runner.Result, 1)
go func() {
result <- r.trace(c)
}()
return result
}
// Trace tracks child processes
func (r *Runner) trace(c context.Context) (result runner.Result) {
ch := &forkexec.Runner{
Args: r.Args,
Env: r.Env,
@ -39,39 +48,29 @@ func (r *Runner) Run(c context.Context) <-chan types.Result {
SyncFunc: r.SyncFunc,
}
result := make(chan types.Result, 1)
go func() {
result <- r.trace(c, ch)
}()
return result
}
// Trace tracks child processes
func (r *Runner) trace(c context.Context, runner *forkexec.Runner) (result types.Result) {
var (
wstatus unix.WaitStatus // wait4 wait status
rusage unix.Rusage // wait4 rusage
status = types.StatusNormal
status = runner.StatusNormal
sTime = time.Now() // start time
fTime time.Time // finish time for setup
)
// Start the runner
pgid, err := runner.Start()
pgid, err := ch.Start()
r.println("Starts: ", pgid, err)
if err != nil {
result.Status = types.StatusRunnerError
result.Status = runner.StatusRunnerError
result.Error = err.Error()
return
}
cc, cancel := context.WithCancel(c)
ctx, cancel := context.WithCancel(c)
defer cancel()
// handle cancel
go func() {
<-cc.Done()
<-ctx.Done()
killAll(pgid)
}()
@ -91,37 +90,37 @@ func (r *Runner) trace(c context.Context, runner *forkexec.Runner) (result types
}
r.println("wait4: ", wstatus)
if err != nil {
result.Status = types.StatusRunnerError
result.Status = runner.StatusRunnerError
result.Error = err.Error()
return
}
// update resource usage and check against limits
userTime := time.Duration(rusage.Utime.Nano()) // ns
userMem := types.Size(rusage.Maxrss << 10) // bytes
userMem := runner.Size(rusage.Maxrss << 10) // bytes
// check tle / mle
if userTime > r.Limit.TimeLimit {
status = types.StatusTimeLimitExceeded
status = runner.StatusTimeLimitExceeded
}
if userMem > r.Limit.MemoryLimit {
status = types.StatusMemoryLimitExceeded
status = runner.StatusMemoryLimitExceeded
}
result = types.Result{
result = runner.Result{
Status: status,
Time: userTime,
Memory: userMem,
}
if status != types.StatusNormal {
if status != runner.StatusNormal {
return
}
switch {
case wstatus.Exited():
result.Status = types.StatusNormal
result.Status = runner.StatusNormal
result.ExitStatus = wstatus.ExitStatus()
if result.ExitStatus != 0 {
result.Status = types.StatusNonzeroExitStatus
result.Status = runner.StatusNonzeroExitStatus
}
return
@ -129,13 +128,13 @@ func (r *Runner) trace(c context.Context, runner *forkexec.Runner) (result types
sig := wstatus.Signal()
switch sig {
case unix.SIGXCPU, unix.SIGKILL:
status = types.StatusTimeLimitExceeded
status = runner.StatusTimeLimitExceeded
case unix.SIGXFSZ:
status = types.StatusOutputLimitExceeded
status = runner.StatusOutputLimitExceeded
case unix.SIGSYS:
status = types.StatusDisallowedSyscall
status = runner.StatusDisallowedSyscall
default:
status = types.StatusSignalled
status = runner.StatusSignalled
}
result.Status = status
result.ExitStatus = int(sig)

View File

@ -4,7 +4,7 @@ import (
"github.com/criyle/go-sandbox/pkg/mount"
"github.com/criyle/go-sandbox/pkg/rlimit"
"github.com/criyle/go-sandbox/pkg/seccomp"
"github.com/criyle/go-sandbox/types"
"github.com/criyle/go-sandbox/runner"
)
// Runner runs program in unshared namespaces
@ -26,7 +26,7 @@ type Runner struct {
RLimits []rlimit.RLimit
// Resource limit enforced by tracer
Limit types.Limit
Limit runner.Limit
// Seccomp defines the seccomp filter attach to the process (should be whitelist only)
Seccomp seccomp.Filter