go-judge/env/env_linux.go
criyle 9ae8487a80
Some checks failed
Build / Goreleaser (push) Has been cancelled
Build / Upload artifacts-${{ matrix.os }}-${{ matrix.arch }} (amd64_v3, darwin) (push) Has been cancelled
Build / Upload artifacts-${{ matrix.os }}-${{ matrix.arch }} (amd64_v3, linux) (push) Has been cancelled
Build / Upload artifacts-${{ matrix.os }}-${{ matrix.arch }} (amd64_v3, windows) (push) Has been cancelled
Build / Upload artifacts-${{ matrix.os }}-${{ matrix.arch }} (arm64_v8.0, darwin) (push) Has been cancelled
Build / Upload artifacts-${{ matrix.os }}-${{ matrix.arch }} (arm64_v8.0, linux) (push) Has been cancelled
Build / Upload artifacts-${{ matrix.os }}-${{ matrix.arch }} (arm64_v8.0, windows) (push) Has been cancelled
linuxc: fix cgroup fd & add clean up to containers when exit
criyle/go-sandbox#13
2025-02-20 23:13:57 -05:00

364 lines
9.4 KiB
Go

package env
import (
"context"
"fmt"
"os"
"sync/atomic"
"syscall"
"github.com/coreos/go-systemd/v22/dbus"
"github.com/criyle/go-judge/env/linuxcontainer"
"github.com/criyle/go-judge/env/pool"
"github.com/criyle/go-judge/envexec"
"github.com/criyle/go-sandbox/container"
"github.com/criyle/go-sandbox/pkg/cgroup"
"github.com/criyle/go-sandbox/pkg/forkexec"
"github.com/criyle/go-sandbox/pkg/mount"
"github.com/criyle/go-sandbox/runner"
ddbus "github.com/godbus/dbus/v5"
"github.com/google/shlex"
"golang.org/x/sys/unix"
)
const (
containerName = "executor_server"
defaultWorkDir = "/w"
containerCredStart = 10000
containerCred = 1000
)
// NewBuilder build a environment builder
func NewBuilder(c Config) (pool.EnvBuilder, map[string]any, error) {
var (
mountBuilder *mount.Builder
symbolicLinks []container.SymbolicLink
maskPaths []string
unshareCgroup bool = true
)
mc, err := readMountConfig(c.MountConf)
if err != nil {
if !os.IsNotExist(err) {
return nil, nil, err
}
c.Info("Mount.yaml(", c.MountConf, ") does not exists, use the default container mount")
mountBuilder = getDefaultMount(c.TmpFsParam)
} else {
mountBuilder, err = parseMountConfig(mc)
if err != nil {
return nil, nil, err
}
}
if mc != nil && len(mc.SymLinks) > 0 {
symbolicLinks = make([]container.SymbolicLink, 0, len(mc.SymLinks))
for _, l := range mc.SymLinks {
symbolicLinks = append(symbolicLinks, container.SymbolicLink{LinkPath: l.LinkPath, Target: l.Target})
}
} else {
symbolicLinks = defaultSymLinks
}
if mc != nil && len(mc.MaskPaths) > 0 {
maskPaths = mc.MaskPaths
} else {
maskPaths = defaultMaskPaths
}
m := mountBuilder.FilterNotExist().Mounts
c.Info("Created container mount at:", mountBuilder)
seccomp, err := readSeccompConf(c.SeccompConf)
if err != nil {
return nil, nil, fmt.Errorf("failed to load seccomp config: %v", err)
}
if seccomp != nil {
c.Info("Load seccomp filter: ", c.SeccompConf)
}
unshareFlags := uintptr(forkexec.UnshareFlags)
if c.NetShare {
unshareFlags ^= syscall.CLONE_NEWNET
}
major, minor := kernelVersion()
unshareFlags ^= unix.CLONE_NEWCGROUP
if major < 4 || (major == 4 && minor < 6) {
unshareCgroup = false
c.Info("Kernel version (", major, ".", minor, ") < 4.6, don't unshare cgroup")
}
// use setuid container only if running in root privilege
var credGen container.CredGenerator
if os.Getuid() == 0 && c.ContainerCredStart > 0 {
credGen = newCredGen(uint32(c.ContainerCredStart))
}
hostName := containerName
domainName := containerName
workDir := defaultWorkDir
cUID := containerCred
cGID := containerCred
var initCmd []string
if mc != nil {
hostName = mc.HostName
domainName = mc.DomainName
workDir = mc.WorkDir
cUID = mc.UID
cGID = mc.GID
if mc.InitCmd != "" {
initCmd, err = shlex.Split(mc.InitCmd)
if err != nil {
return nil, nil, fmt.Errorf("failed to parse initCmd: %s %w", mc.InitCmd, err)
}
c.Info("Initialize container with command: ", mc.InitCmd)
}
}
c.Info("Creating container builder: hostName=", hostName, ", domainName=", domainName, ", workDir=", workDir)
b := &container.Builder{
TmpRoot: "go-judge",
Mounts: m,
SymbolicLinks: symbolicLinks,
MaskPaths: maskPaths,
CredGenerator: credGen,
Stderr: os.Stderr,
CloneFlags: unshareFlags,
ExecFile: c.ContainerInitPath,
HostName: hostName,
DomainName: domainName,
InitCommand: initCmd,
WorkDir: workDir,
ContainerUID: cUID,
ContainerGID: cGID,
UnshareCgroupBeforeExec: unshareCgroup,
}
cgb, ct, err := newCgroup(c)
if err != nil {
return nil, nil, err
}
var cgroupPool linuxcontainer.CgroupPool
if cgb != nil {
cgroupPool = linuxcontainer.NewFakeCgroupPool(cgb, c.CPUCfsPeriod)
}
cgroupType := int(cgroup.DetectedCgroupType)
if cgb == nil {
cgroupType = 0
}
cgroupControllers := []string{}
if ct != nil {
cgroupControllers = ct.Names()
}
conf := map[string]any{
"cgroupType": cgroupType,
"mount": m,
"symbolicLink": symbolicLinks,
"maskedPaths": maskPaths,
"hostName": hostName,
"domainName": domainName,
"workDir": workDir,
"uid": cUID,
"gid": cGID,
"cgroupControllers": cgroupControllers,
}
if cgb != nil && cgroupType == cgroup.TypeV2 && major >= 5 && minor >= 7 {
c.Info("Running kernel ", major, ".", minor, " >= 5.7 with cgroup V2, trying faster clone3(CLONE_INTO_CGROUP)")
if b := func() pool.EnvBuilder {
b := linuxcontainer.NewEnvBuilder(linuxcontainer.Config{
Builder: b,
CgroupPool: cgroupPool,
WorkDir: workDir,
Cpuset: c.Cpuset,
CPURate: c.EnableCPURate,
Seccomp: seccomp,
CgroupFd: true,
})
e, err := b.Build()
if err != nil {
c.Info("Environment build failed: ", err)
return nil
}
defer e.Destroy()
p, err := e.Execve(context.TODO(), envexec.ExecveParam{
Args: []string{"/usr/bin/env"},
Limit: envexec.Limit{
Memory: 256 << 20,
Proc: 1,
},
})
if err != nil {
c.Info("Environment run failed: ", err)
return nil
}
<-p.Done()
r := p.Result()
if r.Status == runner.StatusRunnerError {
c.Info("Environment result failed: ", r)
return nil
}
return b
}(); b != nil {
conf["clone3"] = true
return b, conf, nil
}
}
return linuxcontainer.NewEnvBuilder(linuxcontainer.Config{
Builder: b,
CgroupPool: cgroupPool,
WorkDir: workDir,
Cpuset: c.Cpuset,
CPURate: c.EnableCPURate,
Seccomp: seccomp,
}), conf, nil
}
func newCgroup(c Config) (cgroup.Cgroup, *cgroup.Controllers, error) {
prefix := c.CgroupPrefix
t := cgroup.DetectedCgroupType
ct, err := cgroup.GetAvailableController()
if err != nil {
c.Error("Failed to get available controllers", err)
return nil, nil, err
}
if t == cgroup.TypeV2 {
// Check if running on a systemd enabled system
c.Info("Running with cgroup v2, connecting systemd dbus to create cgroup")
var conn *dbus.Conn
if os.Getuid() == 0 {
conn, err = dbus.NewSystemConnectionContext(context.TODO())
} else {
conn, err = dbus.NewUserConnectionContext(context.TODO())
}
if err != nil {
c.Info("Connecting to systemd dbus failed:", err)
c.Info("Assuming running in container, enable cgroup v2 nesting support and take control of the whole cgroupfs")
prefix = ""
} else {
defer conn.Close()
scopeName := c.CgroupPrefix + ".scope"
c.Info("Connected to systemd bus, attempting to create transient unit: ", scopeName)
properties := []dbus.Property{
dbus.PropDescription("go judge - a high performance sandbox service base on container technologies"),
dbus.PropWants(scopeName),
dbus.PropPids(uint32(os.Getpid())),
newSystemdProperty("Delegate", true),
}
ch := make(chan string, 1)
if _, err := conn.StartTransientUnitContext(context.TODO(), scopeName, "replace", properties, ch); err != nil {
return nil, nil, fmt.Errorf("failed to start transient unit: %w", err)
}
s := <-ch
if s != "done" {
return nil, nil, fmt.Errorf("starting transient unit returns error: %w", err)
}
scopeName, err := cgroup.GetCurrentCgroupPrefix()
if err != nil {
return nil, nil, err
}
c.Info("Current cgroup is ", scopeName)
prefix = scopeName
ct, err = cgroup.GetAvailableControllerWithPrefix(prefix)
if err != nil {
return nil, nil, err
}
}
}
cgb, err := cgroup.New(prefix, ct)
if err != nil {
if os.Getuid() == 0 {
c.Error("Failed to create cgroup ", prefix, " ", err)
return nil, nil, err
}
c.Warn("Not running in root and have no permission on cgroup, falling back to rlimit / rusage mode")
return nil, nil, nil
}
// Create api and migrate current process into it
c.Info("Creating nesting api cgroup ", cgb)
if _, err = cgb.Nest("api"); err != nil {
// Only allow to fall back to rlimit mode when not running with root
if os.Getuid() != 0 {
c.Warn("Creating api cgroup with error: ", err)
c.Warn("As running in non-root mode, falling back back to rlimit / rusage mode")
cgb.Destroy()
return nil, nil, nil
}
}
c.Info("Creating containers cgroup")
cg, err := cgb.New("containers")
if err != nil {
c.Warn("Creating containers cgroup with error: ", err)
c.Warn("Falling back to rlimit / rusage mode")
cgb = nil
}
if !ct.Memory {
c.Warn("Memory cgroup is not enabled, failling back to rlimit / rusage mode")
}
if !ct.Pids {
c.Warn("Pid cgroup is not enabled, proc limit does not have effect")
}
return cg, ct, nil
}
func newSystemdProperty(name string, units any) dbus.Property {
return dbus.Property{
Name: name,
Value: ddbus.MakeVariant(units),
}
}
type credGen struct {
cur atomic.Uint32
}
func newCredGen(start uint32) *credGen {
rt := &credGen{}
rt.cur.Store(start)
return rt
}
func (c *credGen) Get() syscall.Credential {
n := c.cur.Add(1)
return syscall.Credential{
Uid: n,
Gid: n,
}
}
func kernelVersion() (major int, minor int) {
var uname syscall.Utsname
if err := syscall.Uname(&uname); err != nil {
return
}
rl := uname.Release
var values [2]int
vi := 0
value := 0
for _, c := range rl {
if '0' <= c && c <= '9' {
value = (value * 10) + int(c-'0')
} else {
// Note that we're assuming N.N.N here. If we see anything else we are likely to
// mis-parse it.
values[vi] = value
vi++
if vi >= len(values) {
break
}
value = 0
}
}
switch vi {
case 0:
return 0, 0
case 1:
return values[0], 0
case 2:
return values[0], values[1]
}
return
}