Add configuration to OutputLimit & CpuSet

This commit is contained in:
criyle 2020-12-06 18:45:56 -08:00
parent 5c61c1fca6
commit e1f8c954e0
14 changed files with 39 additions and 12 deletions

View File

@ -62,6 +62,8 @@ The `executorserver` need root privilege to create `cgroup`. Either creates sub-
- `-auth-token` to add token-based authentication to REST / gRPC
- `-src-prefix` to restrict `src` copyIn path (need to be absolute path)
- `-time-limit-checker-interval` specifies time limit checker interval (default 100ms)
- `-output-limit` specifies size limit of POSIX rlimit of output
- `-cpuset` specifies `cpuset.cpus` cgroup for each container
#### Environment Variables

View File

@ -26,6 +26,8 @@ type Config struct {
// runner limit
TimeLimitCheckerInterval time.Duration `flagUsage:"specifies time limit checker interval" default:"100ms"`
ExtraMemoryLimit *envexec.Size `flagUsage:"specifies extra memory buffer for check memory limit" default:"16k"`
OutputLimit *envexec.Size `flagUsage:"specifies POSIX rlimit for output for each command" default:"256m"`
Cpuset string `flagUsage:"control the usage of cpuset for all containerd process"`
// server config
HTTPAddr string `flagUsage:"specifies the http binding address" default:":5050"`

View File

@ -75,6 +75,7 @@ func main() {
TmpFsParam: conf.TmpFsParam,
NetShare: conf.NetShare,
CgroupPrefix: conf.CgroupPrefix,
Cpuset: conf.Cpuset,
Logger: logger.Sugar(),
})
if err != nil {
@ -102,6 +103,7 @@ func main() {
WorkDir: conf.Dir,
TimeLimitTickInterval: conf.TimeLimitCheckerInterval,
ExtraMemoryLimit: *conf.ExtraMemoryLimit,
OutputLimit: *conf.OutputLimit,
})
work.Start()
logger.Sugar().Infof("Starting worker with parallelism=%d, workdir=%s, timeLimitCheckInterval=%v",

1
env/config.go vendored
View File

@ -15,5 +15,6 @@ type Config struct {
NetShare bool
MountConf string
CgroupPrefix string
Cpuset string
Logger
}

8
env/env_linux.go vendored
View File

@ -75,7 +75,11 @@ func NewBuilder(c Config) (pool.EnvBuilder, error) {
DomainName: domainName,
WorkDir: workDir,
}
cgb, err := cgroup.NewBuilder(c.CgroupPrefix).WithCPUAcct().WithMemory().WithPids().FilterByEnv()
cgb := cgroup.NewBuilder(c.CgroupPrefix).WithCPUAcct().WithMemory().WithPids()
if c.Cpuset != "" {
cgb = cgb.WithCPUSet()
}
cgb, err = cgb.FilterByEnv()
if err != nil {
return nil, err
}
@ -92,7 +96,7 @@ func NewBuilder(c Config) (pool.EnvBuilder, error) {
if cgb != nil {
cgroupPool = pool.NewFakeCgroupPool(cgb)
}
return pool.NewEnvBuilder(b, cgroupPool, workDir), nil
return pool.NewEnvBuilder(b, cgroupPool, workDir, c.Cpuset), nil
}
type credGen struct {

4
go.mod
View File

@ -5,7 +5,7 @@ go 1.14
require (
cloud.google.com/go v0.73.0 // indirect
github.com/creack/pty v1.1.11
github.com/criyle/go-sandbox v0.5.2
github.com/criyle/go-sandbox v0.5.4
github.com/fatih/camelcase v1.0.0 // indirect
github.com/fatih/structs v1.1.0 // indirect
github.com/gin-contrib/pprof v1.3.0
@ -30,7 +30,7 @@ require (
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9
golang.org/x/sys v0.0.0-20201204225414-ed752295db88
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 // indirect
golang.org/x/tools v0.0.0-20201204222352-654352759326 // indirect
golang.org/x/tools v0.0.0-20201206230334-368bee879bfd // indirect
google.golang.org/genproto v0.0.0-20201204160425-06b3db808446 // indirect
google.golang.org/grpc v1.34.0
google.golang.org/grpc/examples v0.0.0-20201204235607-0d6a24f68a5f // indirect

8
go.sum
View File

@ -85,8 +85,8 @@ github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7Do
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
github.com/creack/pty v1.1.11 h1:07n33Z8lZxZ2qwegKbObQohDhXDQxiMMz1NOUGYlesw=
github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
github.com/criyle/go-sandbox v0.5.2 h1:P6I5vE3dHq6peu3PXBHXpahimKJHnbt/vbV3Eby0qz4=
github.com/criyle/go-sandbox v0.5.2/go.mod h1:6iDDyseyHVugTM2ixPleIBuRx3pVuMKh2qTm6IRmvd8=
github.com/criyle/go-sandbox v0.5.4 h1:6CNSbw3zjCyI8PxGhOH0Liz9F05P2UDGTW4uvb3DS+8=
github.com/criyle/go-sandbox v0.5.4/go.mod h1:6iDDyseyHVugTM2ixPleIBuRx3pVuMKh2qTm6IRmvd8=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@ -710,8 +710,8 @@ golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd h1:kJP9fbfkpUoA4y03Nxor8be
golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.0.0-20201202200335-bef1c476418a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.0.0-20201204222352-654352759326 h1:XKLw9EEEfGJFE2K5Ni4nXgtFBIfI+zKPIi2SlRYmIG4=
golang.org/x/tools v0.0.0-20201204222352-654352759326/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.0.0-20201206230334-368bee879bfd h1:EqFvKLTxjH6gEy2baWxX2AgJwZkBIDIcZFYcoYlI9RA=
golang.org/x/tools v0.0.0-20201206230334-368bee879bfd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=

View File

@ -33,6 +33,7 @@ type Cmd struct {
MemoryLimit Size
StackLimit Size
ExtraMemoryLimit Size
OutputLimit Size
ProcLimit uint64
// file contents to copyin before exec

View File

@ -33,6 +33,7 @@ type Limit struct {
Memory Size // Memory limit
Proc uint64 // Process count limit
Stack Size // Stack limit
Output Size // Output limit
}
// Usage defines the peak process resource usage

View File

@ -13,6 +13,10 @@ var (
type wCgroup cgroup.Cgroup
func (c *wCgroup) SetCpuset(s string) error {
return (*cgroup.Cgroup)(c).SetCpusetCpus([]byte(s))
}
func (c *wCgroup) SetMemoryLimit(s envexec.Size) error {
return (*cgroup.Cgroup)(c).SetMemoryLimitInBytes(uint64(s))
}

View File

@ -9,6 +9,7 @@ import (
// Cgroup defines interface to limit and monitor resources consumption of a process
type Cgroup interface {
SetCpuset(string) error
SetMemoryLimit(envexec.Size) error
SetProcLimit(uint64) error

View File

@ -11,14 +11,16 @@ type environmentBuilder struct {
builder EnvironmentBuilder
cgPool CgroupPool
workDir string
cpuset string
}
// NewEnvBuilder creates builder for linux container pools
func NewEnvBuilder(builder EnvironmentBuilder, cgPool CgroupPool, workDir string) EnvBuilder {
func NewEnvBuilder(builder EnvironmentBuilder, cgPool CgroupPool, workDir, cpuset string) EnvBuilder {
return &environmentBuilder{
builder: builder,
cgPool: cgPool,
workDir: workDir,
cpuset: cpuset,
}
}
@ -40,5 +42,6 @@ func (b *environmentBuilder) Build() (Environment, error) {
Environment: m,
cgPool: b.cgPool,
wd: wd[0],
cpuset: b.cpuset,
}, nil
}

View File

@ -12,8 +12,6 @@ import (
"github.com/criyle/go-sandbox/pkg/rlimit"
)
const outputLimit = 256 << 20 // 256M
var _ envexec.Environment = &environ{}
// environ defines interface to access container resources
@ -21,6 +19,7 @@ type environ struct {
container.Environment
cgPool CgroupPool
wd *os.File // container work dir
cpuset string
}
// Destory destories the environment
@ -45,6 +44,9 @@ func (c *environ) Execve(ctx context.Context, param envexec.ExecveParam) (envexe
if err != nil {
return nil, fmt.Errorf("execve: failed to get cgroup %v", err)
}
if c.cpuset != "" {
cg.SetCpuset(c.cpuset)
}
cg.SetMemoryLimit(param.Limit.Memory)
cg.SetProcLimit(param.Limit.Proc)
syncFunc = cg.AddProc
@ -53,7 +55,7 @@ func (c *environ) Execve(ctx context.Context, param envexec.ExecveParam) (envexe
rLimits := rlimit.RLimits{
CPU: uint64(param.Limit.Time.Truncate(time.Second)/time.Second) + 1,
Data: param.Limit.Memory.Byte(),
FileSize: outputLimit,
FileSize: param.Limit.Output.Byte(),
Stack: param.Limit.Stack.Byte(),
}

View File

@ -22,6 +22,7 @@ type Config struct {
WorkDir string
TimeLimitTickInterval time.Duration
ExtraMemoryLimit envexec.Size
OutputLimit envexec.Size
}
// Worker defines interface for executor
@ -41,6 +42,7 @@ type worker struct {
timeLimitTickInterval time.Duration
extraMemoryLimit envexec.Size
outputLimit envexec.Size
startOnce sync.Once
stopOnce sync.Once
@ -64,6 +66,7 @@ func New(conf Config) Worker {
workDir: conf.WorkDir,
timeLimitTickInterval: conf.TimeLimitTickInterval,
extraMemoryLimit: conf.ExtraMemoryLimit,
outputLimit: conf.OutputLimit,
}
}
@ -285,6 +288,7 @@ func (w *worker) prepareCmd(rc Cmd) (*envexec.Cmd, map[string]bool, error) {
MemoryLimit: envexec.Size(rc.MemoryLimit),
StackLimit: envexec.Size(rc.StackLimit),
ExtraMemoryLimit: w.extraMemoryLimit,
OutputLimit: w.outputLimit,
ProcLimit: rc.ProcLimit,
CopyIn: copyIn,
CopyOut: copyOut,