mirror of
https://github.com/criyle/go-judge.git
synced 2025-11-04 14:50:02 +08:00
Add configuration to OutputLimit & CpuSet
This commit is contained in:
parent
5c61c1fca6
commit
e1f8c954e0
@ -62,6 +62,8 @@ The `executorserver` need root privilege to create `cgroup`. Either creates sub-
|
|||||||
- `-auth-token` to add token-based authentication to REST / gRPC
|
- `-auth-token` to add token-based authentication to REST / gRPC
|
||||||
- `-src-prefix` to restrict `src` copyIn path (need to be absolute path)
|
- `-src-prefix` to restrict `src` copyIn path (need to be absolute path)
|
||||||
- `-time-limit-checker-interval` specifies time limit checker interval (default 100ms)
|
- `-time-limit-checker-interval` specifies time limit checker interval (default 100ms)
|
||||||
|
- `-output-limit` specifies size limit of POSIX rlimit of output
|
||||||
|
- `-cpuset` specifies `cpuset.cpus` cgroup for each container
|
||||||
|
|
||||||
#### Environment Variables
|
#### Environment Variables
|
||||||
|
|
||||||
|
|||||||
@ -26,6 +26,8 @@ type Config struct {
|
|||||||
// runner limit
|
// runner limit
|
||||||
TimeLimitCheckerInterval time.Duration `flagUsage:"specifies time limit checker interval" default:"100ms"`
|
TimeLimitCheckerInterval time.Duration `flagUsage:"specifies time limit checker interval" default:"100ms"`
|
||||||
ExtraMemoryLimit *envexec.Size `flagUsage:"specifies extra memory buffer for check memory limit" default:"16k"`
|
ExtraMemoryLimit *envexec.Size `flagUsage:"specifies extra memory buffer for check memory limit" default:"16k"`
|
||||||
|
OutputLimit *envexec.Size `flagUsage:"specifies POSIX rlimit for output for each command" default:"256m"`
|
||||||
|
Cpuset string `flagUsage:"control the usage of cpuset for all containerd process"`
|
||||||
|
|
||||||
// server config
|
// server config
|
||||||
HTTPAddr string `flagUsage:"specifies the http binding address" default:":5050"`
|
HTTPAddr string `flagUsage:"specifies the http binding address" default:":5050"`
|
||||||
|
|||||||
@ -75,6 +75,7 @@ func main() {
|
|||||||
TmpFsParam: conf.TmpFsParam,
|
TmpFsParam: conf.TmpFsParam,
|
||||||
NetShare: conf.NetShare,
|
NetShare: conf.NetShare,
|
||||||
CgroupPrefix: conf.CgroupPrefix,
|
CgroupPrefix: conf.CgroupPrefix,
|
||||||
|
Cpuset: conf.Cpuset,
|
||||||
Logger: logger.Sugar(),
|
Logger: logger.Sugar(),
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@ -102,6 +103,7 @@ func main() {
|
|||||||
WorkDir: conf.Dir,
|
WorkDir: conf.Dir,
|
||||||
TimeLimitTickInterval: conf.TimeLimitCheckerInterval,
|
TimeLimitTickInterval: conf.TimeLimitCheckerInterval,
|
||||||
ExtraMemoryLimit: *conf.ExtraMemoryLimit,
|
ExtraMemoryLimit: *conf.ExtraMemoryLimit,
|
||||||
|
OutputLimit: *conf.OutputLimit,
|
||||||
})
|
})
|
||||||
work.Start()
|
work.Start()
|
||||||
logger.Sugar().Infof("Starting worker with parallelism=%d, workdir=%s, timeLimitCheckInterval=%v",
|
logger.Sugar().Infof("Starting worker with parallelism=%d, workdir=%s, timeLimitCheckInterval=%v",
|
||||||
|
|||||||
1
env/config.go
vendored
1
env/config.go
vendored
@ -15,5 +15,6 @@ type Config struct {
|
|||||||
NetShare bool
|
NetShare bool
|
||||||
MountConf string
|
MountConf string
|
||||||
CgroupPrefix string
|
CgroupPrefix string
|
||||||
|
Cpuset string
|
||||||
Logger
|
Logger
|
||||||
}
|
}
|
||||||
|
|||||||
8
env/env_linux.go
vendored
8
env/env_linux.go
vendored
@ -75,7 +75,11 @@ func NewBuilder(c Config) (pool.EnvBuilder, error) {
|
|||||||
DomainName: domainName,
|
DomainName: domainName,
|
||||||
WorkDir: workDir,
|
WorkDir: workDir,
|
||||||
}
|
}
|
||||||
cgb, err := cgroup.NewBuilder(c.CgroupPrefix).WithCPUAcct().WithMemory().WithPids().FilterByEnv()
|
cgb := cgroup.NewBuilder(c.CgroupPrefix).WithCPUAcct().WithMemory().WithPids()
|
||||||
|
if c.Cpuset != "" {
|
||||||
|
cgb = cgb.WithCPUSet()
|
||||||
|
}
|
||||||
|
cgb, err = cgb.FilterByEnv()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@ -92,7 +96,7 @@ func NewBuilder(c Config) (pool.EnvBuilder, error) {
|
|||||||
if cgb != nil {
|
if cgb != nil {
|
||||||
cgroupPool = pool.NewFakeCgroupPool(cgb)
|
cgroupPool = pool.NewFakeCgroupPool(cgb)
|
||||||
}
|
}
|
||||||
return pool.NewEnvBuilder(b, cgroupPool, workDir), nil
|
return pool.NewEnvBuilder(b, cgroupPool, workDir, c.Cpuset), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
type credGen struct {
|
type credGen struct {
|
||||||
|
|||||||
4
go.mod
4
go.mod
@ -5,7 +5,7 @@ go 1.14
|
|||||||
require (
|
require (
|
||||||
cloud.google.com/go v0.73.0 // indirect
|
cloud.google.com/go v0.73.0 // indirect
|
||||||
github.com/creack/pty v1.1.11
|
github.com/creack/pty v1.1.11
|
||||||
github.com/criyle/go-sandbox v0.5.2
|
github.com/criyle/go-sandbox v0.5.4
|
||||||
github.com/fatih/camelcase v1.0.0 // indirect
|
github.com/fatih/camelcase v1.0.0 // indirect
|
||||||
github.com/fatih/structs v1.1.0 // indirect
|
github.com/fatih/structs v1.1.0 // indirect
|
||||||
github.com/gin-contrib/pprof v1.3.0
|
github.com/gin-contrib/pprof v1.3.0
|
||||||
@ -30,7 +30,7 @@ require (
|
|||||||
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9
|
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9
|
||||||
golang.org/x/sys v0.0.0-20201204225414-ed752295db88
|
golang.org/x/sys v0.0.0-20201204225414-ed752295db88
|
||||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 // indirect
|
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 // indirect
|
||||||
golang.org/x/tools v0.0.0-20201204222352-654352759326 // indirect
|
golang.org/x/tools v0.0.0-20201206230334-368bee879bfd // indirect
|
||||||
google.golang.org/genproto v0.0.0-20201204160425-06b3db808446 // indirect
|
google.golang.org/genproto v0.0.0-20201204160425-06b3db808446 // indirect
|
||||||
google.golang.org/grpc v1.34.0
|
google.golang.org/grpc v1.34.0
|
||||||
google.golang.org/grpc/examples v0.0.0-20201204235607-0d6a24f68a5f // indirect
|
google.golang.org/grpc/examples v0.0.0-20201204235607-0d6a24f68a5f // indirect
|
||||||
|
|||||||
8
go.sum
8
go.sum
@ -85,8 +85,8 @@ github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7Do
|
|||||||
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
|
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
|
||||||
github.com/creack/pty v1.1.11 h1:07n33Z8lZxZ2qwegKbObQohDhXDQxiMMz1NOUGYlesw=
|
github.com/creack/pty v1.1.11 h1:07n33Z8lZxZ2qwegKbObQohDhXDQxiMMz1NOUGYlesw=
|
||||||
github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
|
github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
|
||||||
github.com/criyle/go-sandbox v0.5.2 h1:P6I5vE3dHq6peu3PXBHXpahimKJHnbt/vbV3Eby0qz4=
|
github.com/criyle/go-sandbox v0.5.4 h1:6CNSbw3zjCyI8PxGhOH0Liz9F05P2UDGTW4uvb3DS+8=
|
||||||
github.com/criyle/go-sandbox v0.5.2/go.mod h1:6iDDyseyHVugTM2ixPleIBuRx3pVuMKh2qTm6IRmvd8=
|
github.com/criyle/go-sandbox v0.5.4/go.mod h1:6iDDyseyHVugTM2ixPleIBuRx3pVuMKh2qTm6IRmvd8=
|
||||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||||
@ -710,8 +710,8 @@ golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd h1:kJP9fbfkpUoA4y03Nxor8be
|
|||||||
golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
|
golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
|
||||||
golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
|
golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
|
||||||
golang.org/x/tools v0.0.0-20201202200335-bef1c476418a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
|
golang.org/x/tools v0.0.0-20201202200335-bef1c476418a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
|
||||||
golang.org/x/tools v0.0.0-20201204222352-654352759326 h1:XKLw9EEEfGJFE2K5Ni4nXgtFBIfI+zKPIi2SlRYmIG4=
|
golang.org/x/tools v0.0.0-20201206230334-368bee879bfd h1:EqFvKLTxjH6gEy2baWxX2AgJwZkBIDIcZFYcoYlI9RA=
|
||||||
golang.org/x/tools v0.0.0-20201204222352-654352759326/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
|
golang.org/x/tools v0.0.0-20201206230334-368bee879bfd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
|
||||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||||
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
|
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
|
||||||
|
|||||||
@ -33,6 +33,7 @@ type Cmd struct {
|
|||||||
MemoryLimit Size
|
MemoryLimit Size
|
||||||
StackLimit Size
|
StackLimit Size
|
||||||
ExtraMemoryLimit Size
|
ExtraMemoryLimit Size
|
||||||
|
OutputLimit Size
|
||||||
ProcLimit uint64
|
ProcLimit uint64
|
||||||
|
|
||||||
// file contents to copyin before exec
|
// file contents to copyin before exec
|
||||||
|
|||||||
@ -33,6 +33,7 @@ type Limit struct {
|
|||||||
Memory Size // Memory limit
|
Memory Size // Memory limit
|
||||||
Proc uint64 // Process count limit
|
Proc uint64 // Process count limit
|
||||||
Stack Size // Stack limit
|
Stack Size // Stack limit
|
||||||
|
Output Size // Output limit
|
||||||
}
|
}
|
||||||
|
|
||||||
// Usage defines the peak process resource usage
|
// Usage defines the peak process resource usage
|
||||||
|
|||||||
@ -13,6 +13,10 @@ var (
|
|||||||
|
|
||||||
type wCgroup cgroup.Cgroup
|
type wCgroup cgroup.Cgroup
|
||||||
|
|
||||||
|
func (c *wCgroup) SetCpuset(s string) error {
|
||||||
|
return (*cgroup.Cgroup)(c).SetCpusetCpus([]byte(s))
|
||||||
|
}
|
||||||
|
|
||||||
func (c *wCgroup) SetMemoryLimit(s envexec.Size) error {
|
func (c *wCgroup) SetMemoryLimit(s envexec.Size) error {
|
||||||
return (*cgroup.Cgroup)(c).SetMemoryLimitInBytes(uint64(s))
|
return (*cgroup.Cgroup)(c).SetMemoryLimitInBytes(uint64(s))
|
||||||
}
|
}
|
||||||
|
|||||||
@ -9,6 +9,7 @@ import (
|
|||||||
|
|
||||||
// Cgroup defines interface to limit and monitor resources consumption of a process
|
// Cgroup defines interface to limit and monitor resources consumption of a process
|
||||||
type Cgroup interface {
|
type Cgroup interface {
|
||||||
|
SetCpuset(string) error
|
||||||
SetMemoryLimit(envexec.Size) error
|
SetMemoryLimit(envexec.Size) error
|
||||||
SetProcLimit(uint64) error
|
SetProcLimit(uint64) error
|
||||||
|
|
||||||
|
|||||||
@ -11,14 +11,16 @@ type environmentBuilder struct {
|
|||||||
builder EnvironmentBuilder
|
builder EnvironmentBuilder
|
||||||
cgPool CgroupPool
|
cgPool CgroupPool
|
||||||
workDir string
|
workDir string
|
||||||
|
cpuset string
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewEnvBuilder creates builder for linux container pools
|
// NewEnvBuilder creates builder for linux container pools
|
||||||
func NewEnvBuilder(builder EnvironmentBuilder, cgPool CgroupPool, workDir string) EnvBuilder {
|
func NewEnvBuilder(builder EnvironmentBuilder, cgPool CgroupPool, workDir, cpuset string) EnvBuilder {
|
||||||
return &environmentBuilder{
|
return &environmentBuilder{
|
||||||
builder: builder,
|
builder: builder,
|
||||||
cgPool: cgPool,
|
cgPool: cgPool,
|
||||||
workDir: workDir,
|
workDir: workDir,
|
||||||
|
cpuset: cpuset,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -40,5 +42,6 @@ func (b *environmentBuilder) Build() (Environment, error) {
|
|||||||
Environment: m,
|
Environment: m,
|
||||||
cgPool: b.cgPool,
|
cgPool: b.cgPool,
|
||||||
wd: wd[0],
|
wd: wd[0],
|
||||||
|
cpuset: b.cpuset,
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|||||||
@ -12,8 +12,6 @@ import (
|
|||||||
"github.com/criyle/go-sandbox/pkg/rlimit"
|
"github.com/criyle/go-sandbox/pkg/rlimit"
|
||||||
)
|
)
|
||||||
|
|
||||||
const outputLimit = 256 << 20 // 256M
|
|
||||||
|
|
||||||
var _ envexec.Environment = &environ{}
|
var _ envexec.Environment = &environ{}
|
||||||
|
|
||||||
// environ defines interface to access container resources
|
// environ defines interface to access container resources
|
||||||
@ -21,6 +19,7 @@ type environ struct {
|
|||||||
container.Environment
|
container.Environment
|
||||||
cgPool CgroupPool
|
cgPool CgroupPool
|
||||||
wd *os.File // container work dir
|
wd *os.File // container work dir
|
||||||
|
cpuset string
|
||||||
}
|
}
|
||||||
|
|
||||||
// Destory destories the environment
|
// Destory destories the environment
|
||||||
@ -45,6 +44,9 @@ func (c *environ) Execve(ctx context.Context, param envexec.ExecveParam) (envexe
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("execve: failed to get cgroup %v", err)
|
return nil, fmt.Errorf("execve: failed to get cgroup %v", err)
|
||||||
}
|
}
|
||||||
|
if c.cpuset != "" {
|
||||||
|
cg.SetCpuset(c.cpuset)
|
||||||
|
}
|
||||||
cg.SetMemoryLimit(param.Limit.Memory)
|
cg.SetMemoryLimit(param.Limit.Memory)
|
||||||
cg.SetProcLimit(param.Limit.Proc)
|
cg.SetProcLimit(param.Limit.Proc)
|
||||||
syncFunc = cg.AddProc
|
syncFunc = cg.AddProc
|
||||||
@ -53,7 +55,7 @@ func (c *environ) Execve(ctx context.Context, param envexec.ExecveParam) (envexe
|
|||||||
rLimits := rlimit.RLimits{
|
rLimits := rlimit.RLimits{
|
||||||
CPU: uint64(param.Limit.Time.Truncate(time.Second)/time.Second) + 1,
|
CPU: uint64(param.Limit.Time.Truncate(time.Second)/time.Second) + 1,
|
||||||
Data: param.Limit.Memory.Byte(),
|
Data: param.Limit.Memory.Byte(),
|
||||||
FileSize: outputLimit,
|
FileSize: param.Limit.Output.Byte(),
|
||||||
Stack: param.Limit.Stack.Byte(),
|
Stack: param.Limit.Stack.Byte(),
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@ -22,6 +22,7 @@ type Config struct {
|
|||||||
WorkDir string
|
WorkDir string
|
||||||
TimeLimitTickInterval time.Duration
|
TimeLimitTickInterval time.Duration
|
||||||
ExtraMemoryLimit envexec.Size
|
ExtraMemoryLimit envexec.Size
|
||||||
|
OutputLimit envexec.Size
|
||||||
}
|
}
|
||||||
|
|
||||||
// Worker defines interface for executor
|
// Worker defines interface for executor
|
||||||
@ -41,6 +42,7 @@ type worker struct {
|
|||||||
|
|
||||||
timeLimitTickInterval time.Duration
|
timeLimitTickInterval time.Duration
|
||||||
extraMemoryLimit envexec.Size
|
extraMemoryLimit envexec.Size
|
||||||
|
outputLimit envexec.Size
|
||||||
|
|
||||||
startOnce sync.Once
|
startOnce sync.Once
|
||||||
stopOnce sync.Once
|
stopOnce sync.Once
|
||||||
@ -64,6 +66,7 @@ func New(conf Config) Worker {
|
|||||||
workDir: conf.WorkDir,
|
workDir: conf.WorkDir,
|
||||||
timeLimitTickInterval: conf.TimeLimitTickInterval,
|
timeLimitTickInterval: conf.TimeLimitTickInterval,
|
||||||
extraMemoryLimit: conf.ExtraMemoryLimit,
|
extraMemoryLimit: conf.ExtraMemoryLimit,
|
||||||
|
outputLimit: conf.OutputLimit,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -285,6 +288,7 @@ func (w *worker) prepareCmd(rc Cmd) (*envexec.Cmd, map[string]bool, error) {
|
|||||||
MemoryLimit: envexec.Size(rc.MemoryLimit),
|
MemoryLimit: envexec.Size(rc.MemoryLimit),
|
||||||
StackLimit: envexec.Size(rc.StackLimit),
|
StackLimit: envexec.Size(rc.StackLimit),
|
||||||
ExtraMemoryLimit: w.extraMemoryLimit,
|
ExtraMemoryLimit: w.extraMemoryLimit,
|
||||||
|
OutputLimit: w.outputLimit,
|
||||||
ProcLimit: rc.ProcLimit,
|
ProcLimit: rc.ProcLimit,
|
||||||
CopyIn: copyIn,
|
CopyIn: copyIn,
|
||||||
CopyOut: copyOut,
|
CopyOut: copyOut,
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user