From 5e5b00688a71156004751d6fcc9dd973d5b2a7d6 Mon Sep 17 00:00:00 2001 From: criyle Date: Fri, 24 Dec 2021 10:42:42 +0000 Subject: [PATCH] cgroup: implement cgroup v2 --- cmd/runprog/main_linux.go | 25 +++++++--- pkg/cgroup/builder.go | 102 +++++++++++++++++++++++++------------- pkg/cgroup/cgroup_info.go | 25 ++++++++-- pkg/cgroup/consts.go | 4 +- pkg/cgroup/doc.go | 2 +- pkg/cgroup/utils.go | 7 +-- pkg/cgroup/v1.go | 6 +-- pkg/cgroup/v2.go | 23 +++------ 8 files changed, 124 insertions(+), 70 deletions(-) diff --git a/cmd/runprog/main_linux.go b/cmd/runprog/main_linux.go index 1fbcfac..9a02db1 100644 --- a/cmd/runprog/main_linux.go +++ b/cmd/runprog/main_linux.go @@ -3,6 +3,7 @@ package main import ( "context" + "errors" "flag" "fmt" "io" @@ -19,6 +20,7 @@ import ( "github.com/criyle/go-sandbox/pkg/memfd" "github.com/criyle/go-sandbox/pkg/mount" "github.com/criyle/go-sandbox/pkg/rlimit" + "github.com/criyle/go-sandbox/pkg/seccomp" "github.com/criyle/go-sandbox/pkg/seccomp/libseccomp" "github.com/criyle/go-sandbox/runner" "github.com/criyle/go-sandbox/runner/ptrace" @@ -119,12 +121,14 @@ func main() { c = runner.StatusRunnerError } // Handle fatal error from trace - fmt.Fprintf(f, "%d %d %d %d\n", getStatus(c), int(rt.Time/time.Millisecond), uint64(rt.Memory)>>10, rt.ExitStatus) + fmt.Fprintf(f, "%d %d %d %d\n", getStatus(c), + int(rt.Time.Round(time.Millisecond)/time.Millisecond), uint64(rt.Memory)>>10, rt.ExitStatus) if c == runner.StatusRunnerError { os.Exit(1) } } else { - fmt.Fprintf(f, "%d %d %d %d\n", 0, int(rt.Time/time.Millisecond), uint64(rt.Memory)>>10, rt.ExitStatus) + fmt.Fprintf(f, "%d %d %d %d\n", 0, + int(rt.Time.Round(time.Millisecond)/time.Millisecond), uint64(rt.Memory)>>10, rt.ExitStatus) } } @@ -268,9 +272,13 @@ func start() (*runner.Result, error) { Trace: trace, Default: actionDefault, } - filter, err := builder.Build() - if err != nil { - return nil, fmt.Errorf("failed to create seccomp filter %v", err) + // do not build filter for container unsafe since seccomp is not compatible with aarch64 syscalls + var filter seccomp.Filter + if !unsafe || runt != "container" { + filter, err = builder.Build() + if err != nil { + return nil, fmt.Errorf("failed to create seccomp filter %v", err) + } } limit := runner.Limit{ @@ -409,13 +417,16 @@ func start() (*runner.Result, error) { if err != nil { return nil, fmt.Errorf("cgroup cpu: %v", err) } + // max memory usage may not exist in cgroup v2 memory, err := cg.MemoryMaxUsage() - if err != nil { + if err != nil && !errors.Is(err, os.ErrNotExist) { return nil, fmt.Errorf("cgroup memory: %v", err) } debug("cgroup: cpu: ", cpu, " memory: ", memory) rt.Time = time.Duration(cpu) - rt.Memory = runner.Size(memory) + if memory > 0 { + rt.Memory = runner.Size(memory) + } debug("cgroup:", rt) } return &rt, nil diff --git a/pkg/cgroup/builder.go b/pkg/cgroup/builder.go index 6c06e75..58143c8 100644 --- a/pkg/cgroup/builder.go +++ b/pkg/cgroup/builder.go @@ -8,6 +8,7 @@ import ( "path" "strconv" "strings" + "sync" "golang.org/x/sys/unix" ) @@ -18,6 +19,9 @@ type Builder struct { Prefix string Type CgroupType + v2initOnce sync.Once + v2initErr error + CPU bool CPUSet bool CPUAcct bool @@ -43,7 +47,7 @@ func (t CgroupType) String() string { } } -// NewBuilder return a dumb builder without any sub-cgroup +// NewBuilder return a dumb builder without any controller func NewBuilder(prefix string) *Builder { return &Builder{ Prefix: prefix, @@ -60,8 +64,9 @@ func (b *Builder) DetectType() *Builder { } if st.Type == unix.CGROUP2_SUPER_MAGIC { b.Type = CgroupTypeV2 + } else { + b.Type = CgroupTypeV1 } - b.Type = CgroupTypeV1 return b } @@ -102,7 +107,7 @@ func (b *Builder) WithPids() *Builder { // FilterByEnv reads /proc/cgroups and filter out non-exists ones func (b *Builder) FilterByEnv() (*Builder, error) { - m, err := GetAllSubCgroup() + m, err := b.getAvailableController() if err != nil { return b, err } @@ -114,9 +119,17 @@ func (b *Builder) FilterByEnv() (*Builder, error) { return b, nil } -// String prints the build properties -func (b *Builder) String() string { - s := make([]string, 0, 3) +func (b *Builder) getAvailableController() (map[string]bool, error) { + switch b.Type { + case CgroupTypeV1: + return GetAvailableControllerV1() + case CgroupTypeV2: + return GetAvailableControllerV2() + } + return nil, os.ErrInvalid +} + +func (b *Builder) loopControllerNames(f func(name string)) { for _, t := range []struct { name string enabled bool @@ -128,9 +141,22 @@ func (b *Builder) String() string { {"pids", b.Pids}, } { if t.enabled { - s = append(s, t.name) + f(t.name) } } +} + +func (b *Builder) controllerNames() []string { + s := make([]string, 0, 5) + b.loopControllerNames(func(name string) { + s = append(s, name) + }) + return s +} + +// String prints the build properties +func (b *Builder) String() string { + s := b.controllerNames() return fmt.Sprintf("cgroup builder(%v): [%s]", b.Type, strings.Join(s, ", ")) } @@ -171,42 +197,50 @@ func (b *Builder) Random(pattern string) (Cgroup, error) { } func (b *Builder) buildV2(name string) (cg Cgroup, err error) { - var s []string - for _, t := range []struct { - name string - enabled bool - }{ - {"cpu", b.CPU}, - {"cpuset", b.CPUSet}, - {"cpuacct", b.CPUAcct}, - {"memory", b.Memory}, - {"pids", b.Pids}, - } { - if t.enabled { - s = append(s, t.name) - } - } - controlMsg := []byte("+" + strings.Join(s, " +")) - // make prefix if not exist - prefix := path.Join(basePath, b.Prefix) - if err := os.Mkdir(prefix, dirPerm); err == nil { - if err := writeFile(path.Join(prefix, cgroupControl), controlMsg, filePerm); err != nil { - return nil, err - } + if err := b.ensurePrefixV2Once(); err != nil { + return nil, err } + p := path.Join(basePath, b.Prefix, name) + defer func() { + if err != nil { + remove(p) + } + }() + // make dir - p := path.Join(basePath, b.Prefix, name) if err := os.Mkdir(p, dirPerm); err != nil { return nil, err } - if err := writeFile(path.Join(p, cgroupControl), controlMsg, filePerm); err != nil { - return nil, err - } return &CgroupV2{p}, nil } +func (b *Builder) ensurePrefixV2Once() error { + b.v2initOnce.Do(func() { + b.v2initErr = b.ensurePrefixV2() + }) + return b.v2initErr +} + +func (b *Builder) ensurePrefixV2() error { + s := b.controllerNames() + controlMsg := []byte("+" + strings.Join(s, " +")) + + // start from base dir + entries := strings.Split(b.Prefix, "/") + current := basePath + for _, e := range entries { + current += e + if err := os.Mkdir(current, dirPerm); err == nil { + if err := writeFile(path.Join(current, cgroupSubtreeControl), controlMsg, filePerm); err != nil { + return err + } + } + } + return nil +} + func (b *Builder) buildV1(name string) (cg Cgroup, err error) { v1 := &CgroupV1{prefix: b.Prefix} @@ -235,7 +269,7 @@ func (b *Builder) buildV1(name string) (cg Cgroup, err error) { } var path string - path, err = CreateV1SubCgroupPathName(c.name, b.Prefix, name) + path, err = CreateV1ControllerPathName(c.name, b.Prefix, name) *c.cg = NewV1Controller(path) if errors.Is(err, os.ErrExist) { // do not ignore first time error, which means collapse diff --git a/pkg/cgroup/cgroup_info.go b/pkg/cgroup/cgroup_info.go index 5b591e3..f4dc91e 100644 --- a/pkg/cgroup/cgroup_info.go +++ b/pkg/cgroup/cgroup_info.go @@ -3,6 +3,7 @@ package cgroup import ( "bufio" "os" + "path" "strconv" "strings" ) @@ -14,8 +15,8 @@ type Info struct { Enabled bool } -// GetCgroupInfo read /proc/cgroups and return the result -func GetCgroupInfo() (map[string]Info, error) { +// GetCgroupV1Info read /proc/cgroups and return the result +func GetCgroupV1Info() (map[string]Info, error) { f, err := os.Open(procCgroupsPath) if err != nil { return nil, err @@ -57,9 +58,9 @@ func GetCgroupInfo() (map[string]Info, error) { return rt, nil } -// GetAllSubCgroup reads /proc/cgroups and get all available sub-cgroup as set -func GetAllSubCgroup() (map[string]bool, error) { - info, err := GetCgroupInfo() +// GetAvailableControllerV1 reads /proc/cgroups and get all available controller as set +func GetAvailableControllerV1() (map[string]bool, error) { + info, err := GetCgroupV1Info() if err != nil { return nil, err } @@ -73,3 +74,17 @@ func GetAllSubCgroup() (map[string]bool, error) { } return rt, nil } + +// GetAvailableControllerV2 reads /sys/fs/cgroup/cgroup.controllers to get all controller +func GetAvailableControllerV2() (map[string]bool, error) { + c, err := readFile(path.Join(basePath, cgroupControllers)) + if err != nil { + return nil, err + } + m := make(map[string]bool) + f := strings.Fields(string(c)) + for _, v := range f { + m[v] = true + } + return m, nil +} diff --git a/pkg/cgroup/consts.go b/pkg/cgroup/consts.go index 18bbbe2..24a6496 100644 --- a/pkg/cgroup/consts.go +++ b/pkg/cgroup/consts.go @@ -4,9 +4,11 @@ const ( // systemd mounted cgroups basePath = "/sys/fs/cgroup" cgroupProcs = "cgroup.procs" - cgroupControl = "cgroup.subtree_control" procCgroupsPath = "/proc/cgroups" + cgroupSubtreeControl = "cgroup.subtree_control" + cgroupControllers = "cgroup.controllers" + filePerm = 0644 dirPerm = 0755 ) diff --git a/pkg/cgroup/doc.go b/pkg/cgroup/doc.go index 8ad4a51..45e9d4f 100644 --- a/pkg/cgroup/doc.go +++ b/pkg/cgroup/doc.go @@ -2,7 +2,7 @@ // under systemd defined mount path (i.e.,sys/fs/cgroup) including v1 and // v2 implementation. // -// Available sub-systems: +// Available cgroup controller: // cpu // cpuset // cpuacct diff --git a/pkg/cgroup/utils.go b/pkg/cgroup/utils.go index 37b87a9..0a16a69 100644 --- a/pkg/cgroup/utils.go +++ b/pkg/cgroup/utils.go @@ -17,14 +17,15 @@ func EnsureDirExists(path string) error { return os.ErrExist } -// CreateSubCgroupPath creates path for sub-cgroup with given group and prefix -func CreateV1SubCgroupPath(controller, prefix string) (string, error) { +// CreateSubCgroupPath creates path for controller with given group and prefix +func CreateV1ControllerPath(controller, prefix string) (string, error) { base := path.Join(basePath, controller, prefix) EnsureDirExists(base) return os.MkdirTemp(base, "") } -func CreateV1SubCgroupPathName(controller, prefix, name string) (string, error) { +// CreateV1ControllerPathName create path for controller with given group, prefix and name +func CreateV1ControllerPathName(controller, prefix, name string) (string, error) { p := path.Join(basePath, controller, prefix, name) return p, EnsureDirExists(p) } diff --git a/pkg/cgroup/v1.go b/pkg/cgroup/v1.go index 716db2e..6669e9f 100644 --- a/pkg/cgroup/v1.go +++ b/pkg/cgroup/v1.go @@ -9,7 +9,7 @@ import ( var _ Cgroup = &CgroupV1{} -// CgroupV1 is the combination of sub-cgroups +// CgroupV1 is the combination of v1 controllers type CgroupV1 struct { prefix string @@ -22,7 +22,7 @@ type CgroupV1 struct { all []*v1controller } -// AddProc writes cgroup.procs to all sub-cgroup +// AddProc writes cgroup.procs to all controller func (c *CgroupV1) AddProc(pid int) error { for _, s := range c.all { if err := s.WriteUint(cgroupProcs, uint64(pid)); err != nil { @@ -32,7 +32,7 @@ func (c *CgroupV1) AddProc(pid int) error { return nil } -// Destroy removes dir for sub-cgroup, errors are ignored if remove one failed +// Destroy removes dir for controller, errors are ignored if remove one failed func (c *CgroupV1) Destroy() error { var err1 error for _, s := range c.all { diff --git a/pkg/cgroup/v2.go b/pkg/cgroup/v2.go index 8b058d1..6bd6289 100644 --- a/pkg/cgroup/v2.go +++ b/pkg/cgroup/v2.go @@ -3,7 +3,6 @@ package cgroup import ( "bufio" "bytes" - "errors" "os" "path" "strconv" @@ -49,36 +48,28 @@ func (c *CgroupV2) MemoryUsage() (uint64, error) { return c.ReadUint("memory.current") } -// not exist, use rusage.max_rss instead +// MemoryMaxUsage not exist, use rusage.max_rss instead func (c *CgroupV2) MemoryMaxUsage() (uint64, error) { return 0, os.ErrNotExist } -// cpu.max quota period +// SetCPUBandwidth set cpu.max quota period func (c *CgroupV2) SetCPUBandwidth(quota, period uint64) error { content := strconv.FormatUint(quota, 10) + " " + strconv.FormatUint(period, 10) - err := c.WriteFile("cpu.max", []byte(content)) - if errors.Is(err, os.ErrNotExist) { - return nil - } - return err + return c.WriteFile("cpu.max", []byte(content)) } -// cpuset.cpus +// SetCPUSet sets cpuset.cpus func (c *CgroupV2) SetCPUSet(content []byte) error { - err := c.WriteFile("cpuset.cpus", content) - if errors.Is(err, os.ErrNotExist) { - return nil - } - return err + return c.WriteFile("cpuset.cpus", content) } -// memory.max +// SetMemoryLimit memory.max func (c *CgroupV2) SetMemoryLimit(l uint64) error { return c.WriteUint("memory.max", l) } -// pids.max +// SetProcLimit pids.max func (c *CgroupV2) SetProcLimit(l uint64) error { return c.WriteUint("pids.max", l) }