package env import ( "context" "fmt" "os" "sync/atomic" "syscall" "github.com/coreos/go-systemd/v22/dbus" "github.com/criyle/go-judge/env/linuxcontainer" "github.com/criyle/go-judge/env/pool" "github.com/criyle/go-judge/envexec" "github.com/criyle/go-sandbox/container" "github.com/criyle/go-sandbox/pkg/cgroup" "github.com/criyle/go-sandbox/pkg/forkexec" "github.com/criyle/go-sandbox/pkg/mount" "github.com/criyle/go-sandbox/runner" ddbus "github.com/godbus/dbus/v5" "github.com/google/shlex" "golang.org/x/sys/unix" ) const ( containerName = "executor_server" defaultWorkDir = "/w" containerCredStart = 10000 containerCred = 1000 ) // NewBuilder build a environment builder func NewBuilder(c Config) (pool.EnvBuilder, map[string]any, error) { var ( mountBuilder *mount.Builder symbolicLinks []container.SymbolicLink maskPaths []string ) mc, err := readMountConfig(c.MountConf) if err != nil { if !os.IsNotExist(err) { return nil, nil, err } c.Info("Mount.yaml(", c.MountConf, ") does not exists, use the default container mount") mountBuilder = getDefaultMount(c.TmpFsParam) } else { mountBuilder, err = parseMountConfig(mc) if err != nil { return nil, nil, err } } if mc != nil && len(mc.SymLinks) > 0 { symbolicLinks = make([]container.SymbolicLink, 0, len(mc.SymLinks)) for _, l := range mc.SymLinks { symbolicLinks = append(symbolicLinks, container.SymbolicLink{LinkPath: l.LinkPath, Target: l.Target}) } } else { symbolicLinks = defaultSymLinks } if mc != nil && len(mc.MaskPaths) > 0 { maskPaths = mc.MaskPaths } else { maskPaths = defaultMaskPaths } m := mountBuilder.FilterNotExist().Mounts c.Info("Created container mount at:", mountBuilder) seccomp, err := readSeccompConf(c.SeccompConf) if err != nil { return nil, nil, fmt.Errorf("failed to load seccomp config: %v", err) } if seccomp != nil { c.Info("Load seccomp filter: ", c.SeccompConf) } unshareFlags := uintptr(forkexec.UnshareFlags) if c.NetShare { unshareFlags ^= syscall.CLONE_NEWNET } major, minor := kernelVersion() if major < 4 || (major == 4 && minor < 6) { unshareFlags ^= unix.CLONE_NEWCGROUP c.Info("Kernel version (", major, ".", minor, ") < 4.6, don't unshare cgroup") } // use setuid container only if running in root privilege var credGen container.CredGenerator if os.Getuid() == 0 && c.ContainerCredStart > 0 { credGen = newCredGen(uint32(c.ContainerCredStart)) } hostName := containerName domainName := containerName workDir := defaultWorkDir cUID := containerCred cGID := containerCred var initCmd []string if mc != nil { hostName = mc.HostName domainName = mc.DomainName workDir = mc.WorkDir cUID = mc.UID cGID = mc.GID if mc.InitCmd != "" { initCmd, err = shlex.Split(mc.InitCmd) if err != nil { return nil, nil, fmt.Errorf("failed to parse initCmd: %s %w", mc.InitCmd, err) } c.Info("Initialize container with command: ", mc.InitCmd) } } c.Info("Creating container builder: hostName=", hostName, ", domainName=", domainName, ", workDir=", workDir) b := &container.Builder{ TmpRoot: "go-judge", Mounts: m, SymbolicLinks: symbolicLinks, MaskPaths: maskPaths, CredGenerator: credGen, Stderr: os.Stderr, CloneFlags: unshareFlags, ExecFile: c.ContainerInitPath, HostName: hostName, DomainName: domainName, InitCommand: initCmd, WorkDir: workDir, ContainerUID: cUID, ContainerGID: cGID, } cgb, ct, err := newCgroup(c) if err != nil { return nil, nil, err } var cgroupPool linuxcontainer.CgroupPool if cgb != nil { cgroupPool = linuxcontainer.NewFakeCgroupPool(cgb, c.CPUCfsPeriod) } cgroupType := int(cgroup.DetectedCgroupType) if cgb == nil { cgroupType = 0 } cgroupControllers := []string{} if ct != nil { cgroupControllers = ct.Names() } conf := map[string]any{ "cgroupType": cgroupType, "mount": m, "symbolicLink": symbolicLinks, "maskedPaths": maskPaths, "hostName": hostName, "domainName": domainName, "workDir": workDir, "uid": cUID, "gid": cGID, "cgroupControllers": cgroupControllers, } if cgb != nil && cgroupType == cgroup.TypeV2 && major >= 5 && minor >= 7 { c.Info("Running kernel ", major, ".", minor, " >= 5.7 with cgroup V2, trying faster clone3(CLONE_INTO_CGROUP)") if b := func() pool.EnvBuilder { b := linuxcontainer.NewEnvBuilder(linuxcontainer.Config{ Builder: b, CgroupPool: cgroupPool, WorkDir: workDir, Cpuset: c.Cpuset, CPURate: c.EnableCPURate, Seccomp: seccomp, CgroupFd: true, }) e, err := b.Build() if err != nil { c.Info("Environment build failed: ", err) return nil } defer e.Destroy() p, err := e.Execve(context.TODO(), envexec.ExecveParam{ Args: []string{"/usr/bin/env"}, }) if err != nil { c.Info("Environment run failed: ", err) return nil } <-p.Done() r := p.Result() if r.Status == runner.StatusRunnerError { c.Info("Environment result failed: ", r) return nil } return b }(); b != nil { conf["clone3"] = true return b, conf, nil } } return linuxcontainer.NewEnvBuilder(linuxcontainer.Config{ Builder: b, CgroupPool: cgroupPool, WorkDir: workDir, Cpuset: c.Cpuset, CPURate: c.EnableCPURate, Seccomp: seccomp, }), conf, nil } func newCgroup(c Config) (cgroup.Cgroup, *cgroup.Controllers, error) { prefix := c.CgroupPrefix t := cgroup.DetectedCgroupType ct, err := cgroup.GetAvailableController() if err != nil { c.Error("Failed to get available controllers", err) return nil, nil, err } if t == cgroup.TypeV2 { // Check if running on a systemd enabled system c.Info("Running with cgroup v2, connecting systemd dbus to create cgroup") var conn *dbus.Conn if os.Getuid() == 0 { conn, err = dbus.NewSystemConnectionContext(context.TODO()) } else { conn, err = dbus.NewUserConnectionContext(context.TODO()) } if err != nil { c.Info("Connecting to systemd dbus failed:", err) c.Info("Assuming running in container, enable cgroup v2 nesting support and take control of the whole cgroupfs") prefix = "" } else { defer conn.Close() scopeName := c.CgroupPrefix + ".scope" c.Info("Connected to systemd bus, attempting to create transient unit: ", scopeName) properties := []dbus.Property{ dbus.PropDescription("go judge - a high performance sandbox service base on container technologies"), dbus.PropWants(scopeName), dbus.PropPids(uint32(os.Getpid())), newSystemdProperty("Delegate", true), } ch := make(chan string, 1) if _, err := conn.StartTransientUnitContext(context.TODO(), scopeName, "replace", properties, ch); err != nil { return nil, nil, fmt.Errorf("failed to start transient unit: %w", err) } s := <-ch if s != "done" { return nil, nil, fmt.Errorf("starting transient unit returns error: %w", err) } scopeName, err := cgroup.GetCurrentCgroupPrefix() if err != nil { return nil, nil, err } c.Info("Current cgroup is ", scopeName) prefix = scopeName ct, err = cgroup.GetAvailableControllerWithPrefix(prefix) if err != nil { return nil, nil, err } } } cgb, err := cgroup.New(prefix, ct) if err != nil { if os.Getuid() == 0 { c.Error("Failed to create cgroup ", prefix, " ", err) return nil, nil, err } c.Warn("Not running in root and have no permission on cgroup, falling back to rlimit / rusage mode") return nil, nil, nil } // Create api and migrate current process into it c.Info("Creating nesting api cgroup ", cgb) if _, err = cgb.Nest("api"); err != nil { // Only allow to fall back to rlimit mode when not running with root if os.Getuid() != 0 { c.Warn("Creating api cgroup with error: ", err) c.Warn("As running in non-root mode, falling back back to rlimit / rusage mode") cgb.Destroy() return nil, nil, nil } } c.Info("Creating containers cgroup") cg, err := cgb.New("containers") if err != nil { c.Warn("Creating containers cgroup with error: ", err) c.Warn("Falling back to rlimit / rusage mode") cgb = nil } if !ct.Memory { c.Warn("Memory cgroup is not enabled, failling back to rlimit / rusage mode") } if !ct.Pids { c.Warn("Pid cgroup is not enabled, proc limit does not have effect") } return cg, ct, nil } func newSystemdProperty(name string, units any) dbus.Property { return dbus.Property{ Name: name, Value: ddbus.MakeVariant(units), } } type credGen struct { cur atomic.Uint32 } func newCredGen(start uint32) *credGen { rt := &credGen{} rt.cur.Store(start) return rt } func (c *credGen) Get() syscall.Credential { n := c.cur.Add(1) return syscall.Credential{ Uid: n, Gid: n, } } func kernelVersion() (major int, minor int) { var uname syscall.Utsname if err := syscall.Uname(&uname); err != nil { return } rl := uname.Release var values [2]int vi := 0 value := 0 for _, c := range rl { if '0' <= c && c <= '9' { value = (value * 10) + int(c-'0') } else { // Note that we're assuming N.N.N here. If we see anything else we are likely to // mis-parse it. values[vi] = value vi++ if vi >= len(values) { break } value = 0 } } switch vi { case 0: return 0, 0 case 1: return values[0], 0 case 2: return values[0], values[1] } return }