mirror of
https://github.com/criyle/go-sandbox.git
synced 2025-11-04 14:49:53 +08:00
add cgroup v1 support for resource stat
This commit is contained in:
parent
144c38f0e8
commit
7c57b248bb
@ -23,6 +23,7 @@ New Features:
|
||||
3. Allow multiple traced programs in different threads
|
||||
4. Allow pipes as input / output files
|
||||
5. Use Linux Namespace to isolate file access (elimilate ptrace)
|
||||
6. Use Linux Control Groups to limit & acct CPU & memory (elimilate wait4.rusage)
|
||||
|
||||
Default file access action:
|
||||
|
||||
@ -66,4 +67,4 @@ It seems unshare net or ipc takes time, maybe limits action by seccomp instead.
|
||||
|
||||
TODO:
|
||||
|
||||
1. Use Linux Control Groups to limit & acct CPU & memory (elimilate wait4 rusage)
|
||||
1. Add ability to pre-fork container deamons
|
||||
|
||||
92
cgroup/cgroup.go
Normal file
92
cgroup/cgroup.go
Normal file
@ -0,0 +1,92 @@
|
||||
// Package cgroup provices basic resource control over cgroups
|
||||
// it measure
|
||||
// cpu: cpuacct.usage (ns)
|
||||
// memory: memory.max_usage_in_bytes
|
||||
// it limits:
|
||||
// memory: memory.limit_in_bytes
|
||||
// # of tasks: pids.max
|
||||
package cgroup
|
||||
|
||||
import "os"
|
||||
|
||||
// additional ideas:
|
||||
// cpu share(not used): cpu.share
|
||||
// reclaim pages from old process: memory.force_empty
|
||||
// (tasks kill are managed out of cgroup as freeze takes some time)
|
||||
// freeze: freezer.state
|
||||
|
||||
// CGroup is the combination of sub-cgroups
|
||||
type CGroup struct {
|
||||
prefix string
|
||||
cpuacct, memory, pids *SubCGroup
|
||||
}
|
||||
|
||||
// NewCGroup creates new cgrouup directories
|
||||
func NewCGroup(prefix string) (*CGroup, error) {
|
||||
cpuacctPath, err := CreateSubCGroupPath("cpuacct", prefix)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
memoryPath, err := CreateSubCGroupPath("memory", prefix)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
pidsPath, err := CreateSubCGroupPath("pids", prefix)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &CGroup{
|
||||
prefix: prefix,
|
||||
cpuacct: NewSubCGroup(cpuacctPath),
|
||||
memory: NewSubCGroup(memoryPath),
|
||||
pids: NewSubCGroup(pidsPath),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// AddProc writes cgroup.procs to all sub-cgroup
|
||||
func (c *CGroup) AddProc(pid int) error {
|
||||
if err := c.cpuacct.WriteUint(cgroupProcs, uint64(pid)); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := c.memory.WriteUint(cgroupProcs, uint64(pid)); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := c.pids.WriteUint(cgroupProcs, uint64(pid)); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Destroy removes dir for sub-cggroup
|
||||
func (c *CGroup) Destroy() error {
|
||||
if err := os.Remove(c.cpuacct.path); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.Remove(c.memory.path); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.Remove(c.pids.path); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// CpuacctUsage read cpuacct.usage in ns
|
||||
func (c *CGroup) CpuacctUsage() (uint64, error) {
|
||||
return c.cpuacct.ReadUint("cpuacct.usage")
|
||||
}
|
||||
|
||||
// MemoryMaxUsageInBytes read memory.max_usage_in_bytes
|
||||
func (c *CGroup) MemoryMaxUsageInBytes() (uint64, error) {
|
||||
return c.memory.ReadUint("memory.max_usage_in_bytes")
|
||||
}
|
||||
|
||||
// SetMemoryLimitInBytes write memory.limit_in_bytes
|
||||
func (c *CGroup) SetMemoryLimitInBytes(i uint64) error {
|
||||
return c.memory.WriteUint("memory.limit_in_bytes", i)
|
||||
}
|
||||
|
||||
// SetPidsMax write pids.max
|
||||
func (c *CGroup) SetPidsMax(i uint64) error {
|
||||
return c.pids.WriteUint("pids.max", i)
|
||||
}
|
||||
7
cgroup/consts.go
Normal file
7
cgroup/consts.go
Normal file
@ -0,0 +1,7 @@
|
||||
package cgroup
|
||||
|
||||
const (
|
||||
// systemd mounted cgroups
|
||||
basePath = "/sys/fs/cgroup"
|
||||
cgroupProcs = "cgroup.procs"
|
||||
)
|
||||
38
cgroup/subcgroup.go
Normal file
38
cgroup/subcgroup.go
Normal file
@ -0,0 +1,38 @@
|
||||
package cgroup
|
||||
|
||||
import (
|
||||
"io/ioutil"
|
||||
"path"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// SubCGroup is the sub-cgroup
|
||||
type SubCGroup struct {
|
||||
path string
|
||||
}
|
||||
|
||||
// NewSubCGroup creates a sug CGroup
|
||||
func NewSubCGroup(p string) *SubCGroup {
|
||||
return &SubCGroup{
|
||||
path: p,
|
||||
}
|
||||
}
|
||||
|
||||
// WriteUint writes uint64 into given file
|
||||
func (c *SubCGroup) WriteUint(filename string, i uint64) error {
|
||||
return ioutil.WriteFile(path.Join(c.path, filename), []byte(strconv.FormatUint(i, 10)), 644)
|
||||
}
|
||||
|
||||
// ReadUint read uint64 into given file
|
||||
func (c *SubCGroup) ReadUint(filename string) (uint64, error) {
|
||||
b, err := ioutil.ReadFile(path.Join(c.path, filename))
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
s, err := strconv.ParseUint(strings.TrimSpace(string(b)), 10, 64)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return s, nil
|
||||
}
|
||||
22
cgroup/utils.go
Normal file
22
cgroup/utils.go
Normal file
@ -0,0 +1,22 @@
|
||||
package cgroup
|
||||
|
||||
import (
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"path"
|
||||
)
|
||||
|
||||
// EnsureDirExists creates dir if not exists
|
||||
func EnsureDirExists(path string) error {
|
||||
if _, err := os.Stat(path); os.IsNotExist(err) {
|
||||
return os.Mkdir(path, os.ModePerm)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// CreateSubCGroupPath creates path for sub-cgroup
|
||||
func CreateSubCGroupPath(group, prefix string) (string, error) {
|
||||
base := path.Join(basePath, group, prefix)
|
||||
EnsureDirExists(base)
|
||||
return ioutil.TempDir(base, "")
|
||||
}
|
||||
@ -5,7 +5,9 @@ import (
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/criyle/go-judger/cgroup"
|
||||
"github.com/criyle/go-judger/runconfig"
|
||||
"github.com/criyle/go-judger/runprogram"
|
||||
"github.com/criyle/go-judger/rununshared"
|
||||
@ -31,11 +33,13 @@ func printUsage() {
|
||||
func main() {
|
||||
var (
|
||||
addReadable, addWritable, addRawReadable, addRawWritable arrayFlags
|
||||
allowProc, unsafe, showDetails, namespace bool
|
||||
allowProc, unsafe, showDetails, namespace, useCGroup bool
|
||||
pType, result string
|
||||
timeLimit, realTimeLimit, memoryLimit, outputLimit, stackLimit uint64
|
||||
inputFileName, outputFileName, errorFileName, workPath string
|
||||
runner Runner
|
||||
cg *cgroup.CGroup
|
||||
err error
|
||||
)
|
||||
|
||||
flag.Usage = printUsage
|
||||
@ -58,6 +62,7 @@ func main() {
|
||||
flag.Var(&addRawReadable, "add-readable-raw", "Add a readable file (don't transform to its real path)")
|
||||
flag.Var(&addRawWritable, "add-writable-raw", "Add a writable file (don't transform to its real path)")
|
||||
flag.BoolVar(&namespace, "ns", false, "Use namespace to restrict file accesses")
|
||||
flag.BoolVar(&useCGroup, "cgroup", false, "Use cgroup to colloct resource usage")
|
||||
flag.Parse()
|
||||
|
||||
args := flag.Args()
|
||||
@ -85,6 +90,26 @@ func main() {
|
||||
addWrite := runconfig.GetExtraSet(addWritable, addRawWritable)
|
||||
h := runconfig.GetConf(pType, workPath, args, addRead, addWrite, allowProc, showDetails)
|
||||
|
||||
if useCGroup {
|
||||
cg, err = cgroup.NewCGroup("run_program")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
defer cg.Destroy()
|
||||
if err = cg.SetMemoryLimitInBytes(memoryLimit << 20); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}
|
||||
|
||||
syncFunc := func(pid int) error {
|
||||
if cg != nil {
|
||||
if err := cg.AddProc(pid); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// open input / output / err files
|
||||
files, err := prepareFiles(inputFileName, outputFileName, errorFileName)
|
||||
if err != nil {
|
||||
@ -135,7 +160,8 @@ func main() {
|
||||
Target: "w",
|
||||
},
|
||||
}),
|
||||
ShowDetails: true,
|
||||
ShowDetails: showDetails,
|
||||
SyncFunc: syncFunc,
|
||||
}
|
||||
} else {
|
||||
runner = &runprogram.RunProgram{
|
||||
@ -154,6 +180,7 @@ func main() {
|
||||
ShowDetails: showDetails,
|
||||
Unsafe: unsafe,
|
||||
Handler: h,
|
||||
SyncFunc: syncFunc,
|
||||
}
|
||||
}
|
||||
|
||||
@ -175,6 +202,20 @@ func main() {
|
||||
rt, err := runner.Start()
|
||||
println("results:", rt, err)
|
||||
|
||||
if useCGroup {
|
||||
cpu, err := cg.CpuacctUsage()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
memory, err := cg.MemoryMaxUsageInBytes()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
println("cgroup: cpu: ", cpu, " memory: ", memory)
|
||||
rt.UserTime = cpu / uint64(time.Millisecond)
|
||||
rt.UserMem = memory >> 10
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
c, ok := err.(specs.TraceCode)
|
||||
if !ok {
|
||||
|
||||
@ -36,6 +36,9 @@ type RunProgram struct {
|
||||
|
||||
// ShowDetails / Unsafe debug flag
|
||||
ShowDetails, Unsafe bool
|
||||
|
||||
// Use by cgroup to add proc
|
||||
SyncFunc func(pid int) error
|
||||
}
|
||||
|
||||
// TraceAction defines action against a syscall check
|
||||
|
||||
@ -26,13 +26,14 @@ func (r *RunProgram) Start() (rt specs.TraceResult, err error) {
|
||||
}
|
||||
|
||||
ch := &forkexec.Runner{
|
||||
Args: r.Args,
|
||||
Env: r.Env,
|
||||
RLimits: r.RLimits.PrepareRLimit(),
|
||||
Files: r.Files,
|
||||
WorkDir: r.WorkDir,
|
||||
Seccomp: bpf,
|
||||
Ptrace: true,
|
||||
Args: r.Args,
|
||||
Env: r.Env,
|
||||
RLimits: r.RLimits.PrepareRLimit(),
|
||||
Files: r.Files,
|
||||
WorkDir: r.WorkDir,
|
||||
Seccomp: bpf,
|
||||
Ptrace: true,
|
||||
SyncFunc: r.SyncFunc,
|
||||
}
|
||||
|
||||
th := &tracerHandler{
|
||||
|
||||
@ -46,6 +46,7 @@ func (r *RunUnshared) Start() (rt specs.TraceResult, err error) {
|
||||
Mounts: r.Mounts,
|
||||
PivotRoot: r.Root,
|
||||
DropCaps: true,
|
||||
SyncFunc: r.SyncFunc,
|
||||
}
|
||||
return r.Trace(ch)
|
||||
}
|
||||
|
||||
@ -35,4 +35,7 @@ type RunUnshared struct {
|
||||
|
||||
// Show Details
|
||||
ShowDetails bool
|
||||
|
||||
// Use by cgroup to add proc
|
||||
SyncFunc func(pid int) error
|
||||
}
|
||||
|
||||
@ -26,7 +26,6 @@ func Trace(handler Handler, runner Runner, limits specs.ResLimit) (result specs.
|
||||
traced = make(map[int]bool) // store all process that have set ptrace options
|
||||
execved = false // store whether the runner process have successfully execvd
|
||||
pid int // store pid of wait4 result
|
||||
initMem uint64 // initial memory usage (likely due to original process)
|
||||
sTime = time.Now().UnixNano() // records start time for trace process
|
||||
fTime int64 // records finish time for execve
|
||||
)
|
||||
@ -85,13 +84,9 @@ func Trace(handler Handler, runner Runner, limits specs.ResLimit) (result specs.
|
||||
|
||||
status := specs.TraceCodeNormal
|
||||
if pid == pgid {
|
||||
if initMem == 0 {
|
||||
initMem = uint64(rusage.Maxrss)
|
||||
handler.Debug("initial memory:", initMem)
|
||||
}
|
||||
// update resource usage and check against limits
|
||||
userTime := uint64(rusage.Utime.Sec*1e3 + rusage.Utime.Usec/1e3) // ms
|
||||
userMem := uint64(rusage.Maxrss) - initMem // kb
|
||||
userMem := uint64(rusage.Maxrss) // kb
|
||||
|
||||
// check tle / mle
|
||||
if userTime > limits.TimeLimit {
|
||||
|
||||
Loading…
Reference in New Issue
Block a user