mirror of
https://github.com/criyle/go-judge.git
synced 2025-11-04 14:50:02 +08:00
linuxc: try support faster new clone3(CLONE_INTO_CGROUP) syscall
Some checks are pending
Build / Goreleaser (push) Waiting to run
Build / Upload artifacts-${{ matrix.os }}-${{ matrix.arch }} (amd64_v3, darwin) (push) Blocked by required conditions
Build / Upload artifacts-${{ matrix.os }}-${{ matrix.arch }} (amd64_v3, linux) (push) Blocked by required conditions
Build / Upload artifacts-${{ matrix.os }}-${{ matrix.arch }} (amd64_v3, windows) (push) Blocked by required conditions
Build / Upload artifacts-${{ matrix.os }}-${{ matrix.arch }} (arm64_v8.0, darwin) (push) Blocked by required conditions
Build / Upload artifacts-${{ matrix.os }}-${{ matrix.arch }} (arm64_v8.0, linux) (push) Blocked by required conditions
Build / Upload artifacts-${{ matrix.os }}-${{ matrix.arch }} (arm64_v8.0, windows) (push) Blocked by required conditions
Some checks are pending
Build / Goreleaser (push) Waiting to run
Build / Upload artifacts-${{ matrix.os }}-${{ matrix.arch }} (amd64_v3, darwin) (push) Blocked by required conditions
Build / Upload artifacts-${{ matrix.os }}-${{ matrix.arch }} (amd64_v3, linux) (push) Blocked by required conditions
Build / Upload artifacts-${{ matrix.os }}-${{ matrix.arch }} (amd64_v3, windows) (push) Blocked by required conditions
Build / Upload artifacts-${{ matrix.os }}-${{ matrix.arch }} (arm64_v8.0, darwin) (push) Blocked by required conditions
Build / Upload artifacts-${{ matrix.os }}-${{ matrix.arch }} (arm64_v8.0, linux) (push) Blocked by required conditions
Build / Upload artifacts-${{ matrix.os }}-${{ matrix.arch }} (arm64_v8.0, windows) (push) Blocked by required conditions
criyle/go-sandbox#13
This commit is contained in:
parent
60a18591fc
commit
ffdf3e3925
@ -538,6 +538,8 @@ interface Output {
|
|||||||
|
|
||||||
在 `systemd` 为 `init` 的发行版中运行时,`go-judge` 会使用 `dbus` 通知 `systemd` 来创建一个临时 `scope` 作为 `cgroup` 的根。
|
在 `systemd` 为 `init` 的发行版中运行时,`go-judge` 会使用 `dbus` 通知 `systemd` 来创建一个临时 `scope` 作为 `cgroup` 的根。
|
||||||
|
|
||||||
|
在高于 5.7 的内核中运行时,`go-judge` 会尝试更快的 `clone3(CLONE_INTO_CGROUP)` 方法.
|
||||||
|
|
||||||
#### 内存使用
|
#### 内存使用
|
||||||
|
|
||||||
控制进程通常会使用 `20M` 内存,每个容器进程最大会使用 `20M` 内存,每个请求最大会使用 `2 * 16M` + 总 copy out max 限制 * 2 内存。请注意,缓存文件会存储在宿主机的共享内存中 (`/dev/shm`),请保证其大小足够存储运行时最大可能文件。
|
控制进程通常会使用 `20M` 内存,每个容器进程最大会使用 `20M` 内存,每个请求最大会使用 `2 * 16M` + 总 copy out max 限制 * 2 内存。请注意,缓存文件会存储在宿主机的共享内存中 (`/dev/shm`),请保证其大小足够存储运行时最大可能文件。
|
||||||
|
|||||||
@ -762,6 +762,8 @@ When running in containers, the `go-judge` will migrate all processed into `/api
|
|||||||
|
|
||||||
When running in Linux distributions powered by `systemd`, the `go-judge` will contact `systemd` via `dbus` to create a transient scope as cgroup root.
|
When running in Linux distributions powered by `systemd`, the `go-judge` will contact `systemd` via `dbus` to create a transient scope as cgroup root.
|
||||||
|
|
||||||
|
When running with kernel >= 5.7, the `go-judge` will try faster `clone3(CLONE_INTO_CGROUP)` method.
|
||||||
|
|
||||||
#### Memory Usage
|
#### Memory Usage
|
||||||
|
|
||||||
The controller will consume `20M` memory and each container will consume `20M` + size of tmpfs `2 * 128M`. For each request, it consumes as much as user program limit + extra limit (`16k`) + total copy out max. Notice that the cached file stores in the shared memory (`/dev/shm`) of the host, so please ensure enough size allocated.
|
The controller will consume `20M` memory and each container will consume `20M` + size of tmpfs `2 * 128M`. For each request, it consumes as much as user program limit + extra limit (`16k`) + total copy out max. Notice that the cached file stores in the shared memory (`/dev/shm`) of the host, so please ensure enough size allocated.
|
||||||
|
|||||||
79
env/env_linux.go
vendored
79
env/env_linux.go
vendored
@ -10,10 +10,12 @@ import (
|
|||||||
"github.com/coreos/go-systemd/v22/dbus"
|
"github.com/coreos/go-systemd/v22/dbus"
|
||||||
"github.com/criyle/go-judge/env/linuxcontainer"
|
"github.com/criyle/go-judge/env/linuxcontainer"
|
||||||
"github.com/criyle/go-judge/env/pool"
|
"github.com/criyle/go-judge/env/pool"
|
||||||
|
"github.com/criyle/go-judge/envexec"
|
||||||
"github.com/criyle/go-sandbox/container"
|
"github.com/criyle/go-sandbox/container"
|
||||||
"github.com/criyle/go-sandbox/pkg/cgroup"
|
"github.com/criyle/go-sandbox/pkg/cgroup"
|
||||||
"github.com/criyle/go-sandbox/pkg/forkexec"
|
"github.com/criyle/go-sandbox/pkg/forkexec"
|
||||||
"github.com/criyle/go-sandbox/pkg/mount"
|
"github.com/criyle/go-sandbox/pkg/mount"
|
||||||
|
"github.com/criyle/go-sandbox/runner"
|
||||||
ddbus "github.com/godbus/dbus/v5"
|
ddbus "github.com/godbus/dbus/v5"
|
||||||
"github.com/google/shlex"
|
"github.com/google/shlex"
|
||||||
"golang.org/x/sys/unix"
|
"golang.org/x/sys/unix"
|
||||||
@ -141,26 +143,65 @@ func NewBuilder(c Config) (pool.EnvBuilder, map[string]any, error) {
|
|||||||
if ct != nil {
|
if ct != nil {
|
||||||
cgroupControllers = ct.Names()
|
cgroupControllers = ct.Names()
|
||||||
}
|
}
|
||||||
return linuxcontainer.NewEnvBuilder(linuxcontainer.Config{
|
conf := map[string]any{
|
||||||
Builder: b,
|
"cgroupType": cgroupType,
|
||||||
CgroupPool: cgroupPool,
|
"mount": m,
|
||||||
WorkDir: workDir,
|
"symbolicLink": symbolicLinks,
|
||||||
Cpuset: c.Cpuset,
|
"maskedPaths": maskPaths,
|
||||||
CPURate: c.EnableCPURate,
|
"hostName": hostName,
|
||||||
Seccomp: seccomp,
|
"domainName": domainName,
|
||||||
}), map[string]any{
|
"workDir": workDir,
|
||||||
"cgroupType": cgroupType,
|
"uid": cUID,
|
||||||
"mount": m,
|
"gid": cGID,
|
||||||
"symbolicLink": symbolicLinks,
|
|
||||||
"maskedPaths": maskPaths,
|
|
||||||
"hostName": hostName,
|
|
||||||
"domainName": domainName,
|
|
||||||
"workDir": workDir,
|
|
||||||
"uid": cUID,
|
|
||||||
"gid": cGID,
|
|
||||||
|
|
||||||
"cgroupControllers": cgroupControllers,
|
"cgroupControllers": cgroupControllers,
|
||||||
}, nil
|
}
|
||||||
|
if cgb != nil && cgroupType == cgroup.TypeV2 && major >= 5 && minor >= 7 {
|
||||||
|
c.Info("Running kernel ", major, ".", minor, " >= 5.7 with cgroup V2, trying faster clone3(CLONE_INTO_CGROUP)")
|
||||||
|
if b := func() pool.EnvBuilder {
|
||||||
|
b := linuxcontainer.NewEnvBuilder(linuxcontainer.Config{
|
||||||
|
Builder: b,
|
||||||
|
CgroupPool: cgroupPool,
|
||||||
|
WorkDir: workDir,
|
||||||
|
Cpuset: c.Cpuset,
|
||||||
|
CPURate: c.EnableCPURate,
|
||||||
|
Seccomp: seccomp,
|
||||||
|
CgroupFd: true,
|
||||||
|
})
|
||||||
|
e, err := b.Build()
|
||||||
|
if err != nil {
|
||||||
|
c.Info("Environment build failed: ", err)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
defer e.Destroy()
|
||||||
|
p, err := e.Execve(context.TODO(), envexec.ExecveParam{
|
||||||
|
Args: []string{"/usr/bin/env"},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
c.Info("Environment run failed: ", err)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
<-p.Done()
|
||||||
|
r := p.Result()
|
||||||
|
if r.Status == runner.StatusRunnerError {
|
||||||
|
c.Info("Environment result failed: ", r)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return b
|
||||||
|
}(); b != nil {
|
||||||
|
conf["clone3"] = true
|
||||||
|
return b, conf, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return linuxcontainer.NewEnvBuilder(linuxcontainer.Config{
|
||||||
|
Builder: b,
|
||||||
|
CgroupPool: cgroupPool,
|
||||||
|
WorkDir: workDir,
|
||||||
|
Cpuset: c.Cpuset,
|
||||||
|
CPURate: c.EnableCPURate,
|
||||||
|
Seccomp: seccomp,
|
||||||
|
}), conf, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func newCgroup(c Config) (cgroup.Cgroup, *cgroup.Controllers, error) {
|
func newCgroup(c Config) (cgroup.Cgroup, *cgroup.Controllers, error) {
|
||||||
|
|||||||
5
env/linuxcontainer/cgroup_wrapper_linux.go
vendored
5
env/linuxcontainer/cgroup_wrapper_linux.go
vendored
@ -1,6 +1,7 @@
|
|||||||
package linuxcontainer
|
package linuxcontainer
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"os"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/criyle/go-judge/envexec"
|
"github.com/criyle/go-judge/envexec"
|
||||||
@ -63,3 +64,7 @@ func (c *wCgroup) Reset() error {
|
|||||||
func (c *wCgroup) Destroy() error {
|
func (c *wCgroup) Destroy() error {
|
||||||
return c.cg.Destroy()
|
return c.cg.Destroy()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (c *wCgroup) Open() (*os.File, error) {
|
||||||
|
return c.cg.Open()
|
||||||
|
}
|
||||||
|
|||||||
3
env/linuxcontainer/cgrouppool_linux.go
vendored
3
env/linuxcontainer/cgrouppool_linux.go
vendored
@ -1,6 +1,7 @@
|
|||||||
package linuxcontainer
|
package linuxcontainer
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"os"
|
||||||
"sync"
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@ -22,6 +23,8 @@ type Cgroup interface {
|
|||||||
AddProc(int) error
|
AddProc(int) error
|
||||||
Reset() error
|
Reset() error
|
||||||
Destroy() error
|
Destroy() error
|
||||||
|
|
||||||
|
Open() (*os.File, error)
|
||||||
}
|
}
|
||||||
|
|
||||||
// CgroupPool implements pool of Cgroup
|
// CgroupPool implements pool of Cgroup
|
||||||
|
|||||||
4
env/linuxcontainer/envbuilder_linux.go
vendored
4
env/linuxcontainer/envbuilder_linux.go
vendored
@ -16,6 +16,7 @@ type Config struct {
|
|||||||
Seccomp []syscall.SockFilter
|
Seccomp []syscall.SockFilter
|
||||||
Cpuset string
|
Cpuset string
|
||||||
CPURate bool
|
CPURate bool
|
||||||
|
CgroupFd bool // whether to enable cgroup fd with clone3, kernel >= 5.7
|
||||||
}
|
}
|
||||||
|
|
||||||
type environmentBuilder struct {
|
type environmentBuilder struct {
|
||||||
@ -25,6 +26,7 @@ type environmentBuilder struct {
|
|||||||
seccomp []syscall.SockFilter
|
seccomp []syscall.SockFilter
|
||||||
cpuset string
|
cpuset string
|
||||||
cpuRate bool
|
cpuRate bool
|
||||||
|
cgFd bool
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewEnvBuilder creates builder for linux container pools
|
// NewEnvBuilder creates builder for linux container pools
|
||||||
@ -36,6 +38,7 @@ func NewEnvBuilder(c Config) pool.EnvBuilder {
|
|||||||
seccomp: c.Seccomp,
|
seccomp: c.Seccomp,
|
||||||
cpuset: c.Cpuset,
|
cpuset: c.Cpuset,
|
||||||
cpuRate: c.CPURate,
|
cpuRate: c.CPURate,
|
||||||
|
cgFd: c.CgroupFd,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -61,5 +64,6 @@ func (b *environmentBuilder) Build() (pool.Environment, error) {
|
|||||||
cpuset: b.cpuset,
|
cpuset: b.cpuset,
|
||||||
cpuRate: b.cpuRate,
|
cpuRate: b.cpuRate,
|
||||||
seccomp: b.seccomp,
|
seccomp: b.seccomp,
|
||||||
|
cgFd: b.cgFd,
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|||||||
15
env/linuxcontainer/environment_linux.go
vendored
15
env/linuxcontainer/environment_linux.go
vendored
@ -28,6 +28,7 @@ type environ struct {
|
|||||||
cpuset string
|
cpuset string
|
||||||
seccomp []syscall.SockFilter
|
seccomp []syscall.SockFilter
|
||||||
cpuRate bool
|
cpuRate bool
|
||||||
|
cgFd bool
|
||||||
}
|
}
|
||||||
|
|
||||||
// Destroy destroys the environment
|
// Destroy destroys the environment
|
||||||
@ -45,6 +46,7 @@ func (c *environ) Execve(ctx context.Context, param envexec.ExecveParam) (envexe
|
|||||||
cg Cgroup
|
cg Cgroup
|
||||||
syncFunc func(int) error
|
syncFunc func(int) error
|
||||||
err error
|
err error
|
||||||
|
cgFd uintptr
|
||||||
)
|
)
|
||||||
|
|
||||||
limit := param.Limit
|
limit := param.Limit
|
||||||
@ -56,7 +58,16 @@ func (c *environ) Execve(ctx context.Context, param envexec.ExecveParam) (envexe
|
|||||||
if err := c.setCgroupLimit(cg, limit); err != nil {
|
if err := c.setCgroupLimit(cg, limit); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
syncFunc = cg.AddProc
|
if c.cgFd {
|
||||||
|
f, err := cg.Open()
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("execve: failed to get cgroup fd %v", err)
|
||||||
|
}
|
||||||
|
defer f.Close()
|
||||||
|
cgFd = f.Fd()
|
||||||
|
} else {
|
||||||
|
syncFunc = cg.AddProc
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
rLimits := rlimit.RLimits{
|
rLimits := rlimit.RLimits{
|
||||||
@ -92,6 +103,8 @@ func (c *environ) Execve(ctx context.Context, param envexec.ExecveParam) (envexe
|
|||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
},
|
},
|
||||||
|
SyncAfterExec: syncFunc == nil,
|
||||||
|
CgroupFD: cgFd,
|
||||||
}
|
}
|
||||||
proc := newProcess(func() runner.Result {
|
proc := newProcess(func() runner.Result {
|
||||||
return c.Environment.Execve(ctx, p)
|
return c.Environment.Execve(ctx, p)
|
||||||
|
|||||||
2
go.mod
2
go.mod
@ -5,7 +5,7 @@ go 1.23
|
|||||||
require (
|
require (
|
||||||
github.com/coreos/go-systemd/v22 v22.5.0
|
github.com/coreos/go-systemd/v22 v22.5.0
|
||||||
github.com/creack/pty v1.1.24
|
github.com/creack/pty v1.1.24
|
||||||
github.com/criyle/go-sandbox v0.10.9
|
github.com/criyle/go-sandbox v0.11.0
|
||||||
github.com/elastic/go-seccomp-bpf v1.5.0
|
github.com/elastic/go-seccomp-bpf v1.5.0
|
||||||
github.com/elastic/go-ucfg v0.8.8
|
github.com/elastic/go-ucfg v0.8.8
|
||||||
github.com/gin-contrib/zap v1.1.4
|
github.com/gin-contrib/zap v1.1.4
|
||||||
|
|||||||
4
go.sum
4
go.sum
@ -18,8 +18,8 @@ github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8
|
|||||||
github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
|
github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
|
||||||
github.com/creack/pty v1.1.24 h1:bJrF4RRfyJnbTJqzRLHzcGaZK1NeM5kTC9jGgovnR1s=
|
github.com/creack/pty v1.1.24 h1:bJrF4RRfyJnbTJqzRLHzcGaZK1NeM5kTC9jGgovnR1s=
|
||||||
github.com/creack/pty v1.1.24/go.mod h1:08sCNb52WyoAwi2QDyzUCTgcvVFhUzewun7wtTfvcwE=
|
github.com/creack/pty v1.1.24/go.mod h1:08sCNb52WyoAwi2QDyzUCTgcvVFhUzewun7wtTfvcwE=
|
||||||
github.com/criyle/go-sandbox v0.10.9 h1:IYXVrfdSi8GgXlxBDNBINpH4VqnznxHi3R5J17d69fs=
|
github.com/criyle/go-sandbox v0.11.0 h1:1jkfLigilxQza3zPyF1aIGR5WdDuiMYE1rzVYWlJC9I=
|
||||||
github.com/criyle/go-sandbox v0.10.9/go.mod h1:9IZSv7cxcDkVaPSRufhMPLUg+7M7lTPAt8hjd/iMKFo=
|
github.com/criyle/go-sandbox v0.11.0/go.mod h1:9IZSv7cxcDkVaPSRufhMPLUg+7M7lTPAt8hjd/iMKFo=
|
||||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user