cgroup: implement cgroup v2

This commit is contained in:
criyle 2021-12-24 10:42:42 +00:00
parent 2889743b71
commit 5e5b00688a
8 changed files with 124 additions and 70 deletions

View File

@ -3,6 +3,7 @@ package main
import ( import (
"context" "context"
"errors"
"flag" "flag"
"fmt" "fmt"
"io" "io"
@ -19,6 +20,7 @@ import (
"github.com/criyle/go-sandbox/pkg/memfd" "github.com/criyle/go-sandbox/pkg/memfd"
"github.com/criyle/go-sandbox/pkg/mount" "github.com/criyle/go-sandbox/pkg/mount"
"github.com/criyle/go-sandbox/pkg/rlimit" "github.com/criyle/go-sandbox/pkg/rlimit"
"github.com/criyle/go-sandbox/pkg/seccomp"
"github.com/criyle/go-sandbox/pkg/seccomp/libseccomp" "github.com/criyle/go-sandbox/pkg/seccomp/libseccomp"
"github.com/criyle/go-sandbox/runner" "github.com/criyle/go-sandbox/runner"
"github.com/criyle/go-sandbox/runner/ptrace" "github.com/criyle/go-sandbox/runner/ptrace"
@ -119,12 +121,14 @@ func main() {
c = runner.StatusRunnerError c = runner.StatusRunnerError
} }
// Handle fatal error from trace // Handle fatal error from trace
fmt.Fprintf(f, "%d %d %d %d\n", getStatus(c), int(rt.Time/time.Millisecond), uint64(rt.Memory)>>10, rt.ExitStatus) fmt.Fprintf(f, "%d %d %d %d\n", getStatus(c),
int(rt.Time.Round(time.Millisecond)/time.Millisecond), uint64(rt.Memory)>>10, rt.ExitStatus)
if c == runner.StatusRunnerError { if c == runner.StatusRunnerError {
os.Exit(1) os.Exit(1)
} }
} else { } else {
fmt.Fprintf(f, "%d %d %d %d\n", 0, int(rt.Time/time.Millisecond), uint64(rt.Memory)>>10, rt.ExitStatus) fmt.Fprintf(f, "%d %d %d %d\n", 0,
int(rt.Time.Round(time.Millisecond)/time.Millisecond), uint64(rt.Memory)>>10, rt.ExitStatus)
} }
} }
@ -268,9 +272,13 @@ func start() (*runner.Result, error) {
Trace: trace, Trace: trace,
Default: actionDefault, Default: actionDefault,
} }
filter, err := builder.Build() // do not build filter for container unsafe since seccomp is not compatible with aarch64 syscalls
if err != nil { var filter seccomp.Filter
return nil, fmt.Errorf("failed to create seccomp filter %v", err) if !unsafe || runt != "container" {
filter, err = builder.Build()
if err != nil {
return nil, fmt.Errorf("failed to create seccomp filter %v", err)
}
} }
limit := runner.Limit{ limit := runner.Limit{
@ -409,13 +417,16 @@ func start() (*runner.Result, error) {
if err != nil { if err != nil {
return nil, fmt.Errorf("cgroup cpu: %v", err) return nil, fmt.Errorf("cgroup cpu: %v", err)
} }
// max memory usage may not exist in cgroup v2
memory, err := cg.MemoryMaxUsage() memory, err := cg.MemoryMaxUsage()
if err != nil { if err != nil && !errors.Is(err, os.ErrNotExist) {
return nil, fmt.Errorf("cgroup memory: %v", err) return nil, fmt.Errorf("cgroup memory: %v", err)
} }
debug("cgroup: cpu: ", cpu, " memory: ", memory) debug("cgroup: cpu: ", cpu, " memory: ", memory)
rt.Time = time.Duration(cpu) rt.Time = time.Duration(cpu)
rt.Memory = runner.Size(memory) if memory > 0 {
rt.Memory = runner.Size(memory)
}
debug("cgroup:", rt) debug("cgroup:", rt)
} }
return &rt, nil return &rt, nil

View File

@ -8,6 +8,7 @@ import (
"path" "path"
"strconv" "strconv"
"strings" "strings"
"sync"
"golang.org/x/sys/unix" "golang.org/x/sys/unix"
) )
@ -18,6 +19,9 @@ type Builder struct {
Prefix string Prefix string
Type CgroupType Type CgroupType
v2initOnce sync.Once
v2initErr error
CPU bool CPU bool
CPUSet bool CPUSet bool
CPUAcct bool CPUAcct bool
@ -43,7 +47,7 @@ func (t CgroupType) String() string {
} }
} }
// NewBuilder return a dumb builder without any sub-cgroup // NewBuilder return a dumb builder without any controller
func NewBuilder(prefix string) *Builder { func NewBuilder(prefix string) *Builder {
return &Builder{ return &Builder{
Prefix: prefix, Prefix: prefix,
@ -60,8 +64,9 @@ func (b *Builder) DetectType() *Builder {
} }
if st.Type == unix.CGROUP2_SUPER_MAGIC { if st.Type == unix.CGROUP2_SUPER_MAGIC {
b.Type = CgroupTypeV2 b.Type = CgroupTypeV2
} else {
b.Type = CgroupTypeV1
} }
b.Type = CgroupTypeV1
return b return b
} }
@ -102,7 +107,7 @@ func (b *Builder) WithPids() *Builder {
// FilterByEnv reads /proc/cgroups and filter out non-exists ones // FilterByEnv reads /proc/cgroups and filter out non-exists ones
func (b *Builder) FilterByEnv() (*Builder, error) { func (b *Builder) FilterByEnv() (*Builder, error) {
m, err := GetAllSubCgroup() m, err := b.getAvailableController()
if err != nil { if err != nil {
return b, err return b, err
} }
@ -114,9 +119,17 @@ func (b *Builder) FilterByEnv() (*Builder, error) {
return b, nil return b, nil
} }
// String prints the build properties func (b *Builder) getAvailableController() (map[string]bool, error) {
func (b *Builder) String() string { switch b.Type {
s := make([]string, 0, 3) case CgroupTypeV1:
return GetAvailableControllerV1()
case CgroupTypeV2:
return GetAvailableControllerV2()
}
return nil, os.ErrInvalid
}
func (b *Builder) loopControllerNames(f func(name string)) {
for _, t := range []struct { for _, t := range []struct {
name string name string
enabled bool enabled bool
@ -128,9 +141,22 @@ func (b *Builder) String() string {
{"pids", b.Pids}, {"pids", b.Pids},
} { } {
if t.enabled { if t.enabled {
s = append(s, t.name) f(t.name)
} }
} }
}
func (b *Builder) controllerNames() []string {
s := make([]string, 0, 5)
b.loopControllerNames(func(name string) {
s = append(s, name)
})
return s
}
// String prints the build properties
func (b *Builder) String() string {
s := b.controllerNames()
return fmt.Sprintf("cgroup builder(%v): [%s]", b.Type, strings.Join(s, ", ")) return fmt.Sprintf("cgroup builder(%v): [%s]", b.Type, strings.Join(s, ", "))
} }
@ -171,42 +197,50 @@ func (b *Builder) Random(pattern string) (Cgroup, error) {
} }
func (b *Builder) buildV2(name string) (cg Cgroup, err error) { func (b *Builder) buildV2(name string) (cg Cgroup, err error) {
var s []string
for _, t := range []struct {
name string
enabled bool
}{
{"cpu", b.CPU},
{"cpuset", b.CPUSet},
{"cpuacct", b.CPUAcct},
{"memory", b.Memory},
{"pids", b.Pids},
} {
if t.enabled {
s = append(s, t.name)
}
}
controlMsg := []byte("+" + strings.Join(s, " +"))
// make prefix if not exist // make prefix if not exist
prefix := path.Join(basePath, b.Prefix) if err := b.ensurePrefixV2Once(); err != nil {
if err := os.Mkdir(prefix, dirPerm); err == nil { return nil, err
if err := writeFile(path.Join(prefix, cgroupControl), controlMsg, filePerm); err != nil {
return nil, err
}
} }
p := path.Join(basePath, b.Prefix, name)
defer func() {
if err != nil {
remove(p)
}
}()
// make dir // make dir
p := path.Join(basePath, b.Prefix, name)
if err := os.Mkdir(p, dirPerm); err != nil { if err := os.Mkdir(p, dirPerm); err != nil {
return nil, err return nil, err
} }
if err := writeFile(path.Join(p, cgroupControl), controlMsg, filePerm); err != nil {
return nil, err
}
return &CgroupV2{p}, nil return &CgroupV2{p}, nil
} }
func (b *Builder) ensurePrefixV2Once() error {
b.v2initOnce.Do(func() {
b.v2initErr = b.ensurePrefixV2()
})
return b.v2initErr
}
func (b *Builder) ensurePrefixV2() error {
s := b.controllerNames()
controlMsg := []byte("+" + strings.Join(s, " +"))
// start from base dir
entries := strings.Split(b.Prefix, "/")
current := basePath
for _, e := range entries {
current += e
if err := os.Mkdir(current, dirPerm); err == nil {
if err := writeFile(path.Join(current, cgroupSubtreeControl), controlMsg, filePerm); err != nil {
return err
}
}
}
return nil
}
func (b *Builder) buildV1(name string) (cg Cgroup, err error) { func (b *Builder) buildV1(name string) (cg Cgroup, err error) {
v1 := &CgroupV1{prefix: b.Prefix} v1 := &CgroupV1{prefix: b.Prefix}
@ -235,7 +269,7 @@ func (b *Builder) buildV1(name string) (cg Cgroup, err error) {
} }
var path string var path string
path, err = CreateV1SubCgroupPathName(c.name, b.Prefix, name) path, err = CreateV1ControllerPathName(c.name, b.Prefix, name)
*c.cg = NewV1Controller(path) *c.cg = NewV1Controller(path)
if errors.Is(err, os.ErrExist) { if errors.Is(err, os.ErrExist) {
// do not ignore first time error, which means collapse // do not ignore first time error, which means collapse

View File

@ -3,6 +3,7 @@ package cgroup
import ( import (
"bufio" "bufio"
"os" "os"
"path"
"strconv" "strconv"
"strings" "strings"
) )
@ -14,8 +15,8 @@ type Info struct {
Enabled bool Enabled bool
} }
// GetCgroupInfo read /proc/cgroups and return the result // GetCgroupV1Info read /proc/cgroups and return the result
func GetCgroupInfo() (map[string]Info, error) { func GetCgroupV1Info() (map[string]Info, error) {
f, err := os.Open(procCgroupsPath) f, err := os.Open(procCgroupsPath)
if err != nil { if err != nil {
return nil, err return nil, err
@ -57,9 +58,9 @@ func GetCgroupInfo() (map[string]Info, error) {
return rt, nil return rt, nil
} }
// GetAllSubCgroup reads /proc/cgroups and get all available sub-cgroup as set // GetAvailableControllerV1 reads /proc/cgroups and get all available controller as set
func GetAllSubCgroup() (map[string]bool, error) { func GetAvailableControllerV1() (map[string]bool, error) {
info, err := GetCgroupInfo() info, err := GetCgroupV1Info()
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -73,3 +74,17 @@ func GetAllSubCgroup() (map[string]bool, error) {
} }
return rt, nil return rt, nil
} }
// GetAvailableControllerV2 reads /sys/fs/cgroup/cgroup.controllers to get all controller
func GetAvailableControllerV2() (map[string]bool, error) {
c, err := readFile(path.Join(basePath, cgroupControllers))
if err != nil {
return nil, err
}
m := make(map[string]bool)
f := strings.Fields(string(c))
for _, v := range f {
m[v] = true
}
return m, nil
}

View File

@ -4,9 +4,11 @@ const (
// systemd mounted cgroups // systemd mounted cgroups
basePath = "/sys/fs/cgroup" basePath = "/sys/fs/cgroup"
cgroupProcs = "cgroup.procs" cgroupProcs = "cgroup.procs"
cgroupControl = "cgroup.subtree_control"
procCgroupsPath = "/proc/cgroups" procCgroupsPath = "/proc/cgroups"
cgroupSubtreeControl = "cgroup.subtree_control"
cgroupControllers = "cgroup.controllers"
filePerm = 0644 filePerm = 0644
dirPerm = 0755 dirPerm = 0755
) )

View File

@ -2,7 +2,7 @@
// under systemd defined mount path (i.e.,sys/fs/cgroup) including v1 and // under systemd defined mount path (i.e.,sys/fs/cgroup) including v1 and
// v2 implementation. // v2 implementation.
// //
// Available sub-systems: // Available cgroup controller:
// cpu // cpu
// cpuset // cpuset
// cpuacct // cpuacct

View File

@ -17,14 +17,15 @@ func EnsureDirExists(path string) error {
return os.ErrExist return os.ErrExist
} }
// CreateSubCgroupPath creates path for sub-cgroup with given group and prefix // CreateSubCgroupPath creates path for controller with given group and prefix
func CreateV1SubCgroupPath(controller, prefix string) (string, error) { func CreateV1ControllerPath(controller, prefix string) (string, error) {
base := path.Join(basePath, controller, prefix) base := path.Join(basePath, controller, prefix)
EnsureDirExists(base) EnsureDirExists(base)
return os.MkdirTemp(base, "") return os.MkdirTemp(base, "")
} }
func CreateV1SubCgroupPathName(controller, prefix, name string) (string, error) { // CreateV1ControllerPathName create path for controller with given group, prefix and name
func CreateV1ControllerPathName(controller, prefix, name string) (string, error) {
p := path.Join(basePath, controller, prefix, name) p := path.Join(basePath, controller, prefix, name)
return p, EnsureDirExists(p) return p, EnsureDirExists(p)
} }

View File

@ -9,7 +9,7 @@ import (
var _ Cgroup = &CgroupV1{} var _ Cgroup = &CgroupV1{}
// CgroupV1 is the combination of sub-cgroups // CgroupV1 is the combination of v1 controllers
type CgroupV1 struct { type CgroupV1 struct {
prefix string prefix string
@ -22,7 +22,7 @@ type CgroupV1 struct {
all []*v1controller all []*v1controller
} }
// AddProc writes cgroup.procs to all sub-cgroup // AddProc writes cgroup.procs to all controller
func (c *CgroupV1) AddProc(pid int) error { func (c *CgroupV1) AddProc(pid int) error {
for _, s := range c.all { for _, s := range c.all {
if err := s.WriteUint(cgroupProcs, uint64(pid)); err != nil { if err := s.WriteUint(cgroupProcs, uint64(pid)); err != nil {
@ -32,7 +32,7 @@ func (c *CgroupV1) AddProc(pid int) error {
return nil return nil
} }
// Destroy removes dir for sub-cgroup, errors are ignored if remove one failed // Destroy removes dir for controller, errors are ignored if remove one failed
func (c *CgroupV1) Destroy() error { func (c *CgroupV1) Destroy() error {
var err1 error var err1 error
for _, s := range c.all { for _, s := range c.all {

View File

@ -3,7 +3,6 @@ package cgroup
import ( import (
"bufio" "bufio"
"bytes" "bytes"
"errors"
"os" "os"
"path" "path"
"strconv" "strconv"
@ -49,36 +48,28 @@ func (c *CgroupV2) MemoryUsage() (uint64, error) {
return c.ReadUint("memory.current") return c.ReadUint("memory.current")
} }
// not exist, use rusage.max_rss instead // MemoryMaxUsage not exist, use rusage.max_rss instead
func (c *CgroupV2) MemoryMaxUsage() (uint64, error) { func (c *CgroupV2) MemoryMaxUsage() (uint64, error) {
return 0, os.ErrNotExist return 0, os.ErrNotExist
} }
// cpu.max quota period // SetCPUBandwidth set cpu.max quota period
func (c *CgroupV2) SetCPUBandwidth(quota, period uint64) error { func (c *CgroupV2) SetCPUBandwidth(quota, period uint64) error {
content := strconv.FormatUint(quota, 10) + " " + strconv.FormatUint(period, 10) content := strconv.FormatUint(quota, 10) + " " + strconv.FormatUint(period, 10)
err := c.WriteFile("cpu.max", []byte(content)) return c.WriteFile("cpu.max", []byte(content))
if errors.Is(err, os.ErrNotExist) {
return nil
}
return err
} }
// cpuset.cpus // SetCPUSet sets cpuset.cpus
func (c *CgroupV2) SetCPUSet(content []byte) error { func (c *CgroupV2) SetCPUSet(content []byte) error {
err := c.WriteFile("cpuset.cpus", content) return c.WriteFile("cpuset.cpus", content)
if errors.Is(err, os.ErrNotExist) {
return nil
}
return err
} }
// memory.max // SetMemoryLimit memory.max
func (c *CgroupV2) SetMemoryLimit(l uint64) error { func (c *CgroupV2) SetMemoryLimit(l uint64) error {
return c.WriteUint("memory.max", l) return c.WriteUint("memory.max", l)
} }
// pids.max // SetProcLimit pids.max
func (c *CgroupV2) SetProcLimit(l uint64) error { func (c *CgroupV2) SetProcLimit(l uint64) error {
return c.WriteUint("pids.max", l) return c.WriteUint("pids.max", l)
} }