From d32acd7591c7ac72c377f25780297023fcbf6087 Mon Sep 17 00:00:00 2001 From: criyle Date: Thu, 20 Feb 2025 20:29:20 +0000 Subject: [PATCH] cgroup&container: add ability to utilize cgroup fd with clone3 #13 --- cmd/runprog/main_linux.go | 19 ++++++++++++++++++- container/container_exec_linux.go | 12 +++++++++++- container/host_exec_linux.go | 6 ++++++ container/protocol_linux.go | 1 + pkg/cgroup/cgroup_linux.go | 3 +++ pkg/cgroup/v1_linux.go | 4 ++++ pkg/cgroup/v2_linux.go | 4 ++++ 7 files changed, 47 insertions(+), 2 deletions(-) diff --git a/cmd/runprog/main_linux.go b/cmd/runprog/main_linux.go index 7a1b3a8..da154cd 100644 --- a/cmd/runprog/main_linux.go +++ b/cmd/runprog/main_linux.go @@ -35,6 +35,7 @@ var ( timeLimit, realTimeLimit, memoryLimit, outputLimit, stackLimit uint64 inputFileName, outputFileName, errorFileName, workPath, runt string + useCGroupFd bool pType, result string args []string ) @@ -65,6 +66,7 @@ func main() { flag.Var(&addRawReadable, "add-readable-raw", "Add a readable file (don't transform to its real path)") flag.Var(&addRawWritable, "add-writable-raw", "Add a writable file (don't transform to its real path)") flag.BoolVar(&useCGroup, "cgroup", false, "Use cgroup to colloct resource usage") + flag.BoolVar(&useCGroupFd, "cgroupfd", false, "Use cgroup FD to clone3 (cgroup v2 & kernel > 5.7)") flag.BoolVar(&memfile, "memfd", false, "Use memfd as exec file") flag.StringVar(&runt, "runner", "ptrace", "Runner for the program (ptrace, ns, container)") flag.BoolVar(&cred, "cred", false, "Generate credential for containers (uid=10000)") @@ -145,6 +147,8 @@ func start() (*runner.Result, error) { var ( r runner.Runner cg cgroup.Cgroup + cgDir *os.File + cgroupFd uintptr err error execFile uintptr rt runner.Result @@ -206,6 +210,18 @@ func start() (*runner.Result, error) { if err = cg.SetMemoryLimit(memoryLimit << 20); err != nil { return nil, err } + debug("cgroup:", cg) + if useCGroupFd { + debug("use cgroup fd") + if t != cgroup.TypeV2 { + return nil, fmt.Errorf("use cgroup fd cannot be enabled without cgroup v2") + } + if cgDir, err = cg.Open(); err != nil { + return nil, err + } + defer cgDir.Close() + cgroupFd = cgDir.Fd() + } } syncFunc := func(pid int) error { @@ -332,7 +348,8 @@ func start() (*runner.Result, error) { RLimits: rlims.PrepareRLimit(), Seccomp: filter, SyncFunc: syncFunc, - SyncAfterExec: cg == nil, + CgroupFD: cgroupFd, + SyncAfterExec: cg == nil || cgDir != nil, }, } } else if runt == "ns" { diff --git a/container/container_exec_linux.go b/container/container_exec_linux.go index 869da45..f8127a3 100644 --- a/container/container_exec_linux.go +++ b/container/container_exec_linux.go @@ -14,6 +14,7 @@ func (c *containerServer) handleExecve(cmd *execCmd, msg unixsocket.Msg) error { var ( files []uintptr execFile uintptr + cgroupFd uintptr cred *syscall.Credential ) if cmd == nil { @@ -35,6 +36,14 @@ func (c *containerServer) handleExecve(cmd *execCmd, msg unixsocket.Msg) error { execFile = files[0] files = files[1:] } + // if cgroupFd, then the cgroupFd follows + if cmd.FdCgroup { + if len(files) == 0 { + return c.sendErrorReply("handle: expected cgroup fd") + } + cgroupFd = files[0] + files = files[1:] + } var env []string env = append(env, c.defaultEnv...) @@ -99,8 +108,9 @@ func (c *containerServer) handleExecve(cmd *execCmd, msg unixsocket.Msg) error { Credential: cred, CTTY: cmd.CTTY, Seccomp: seccomp, + CgroupFd: cgroupFd, - UnshareCgroupAfterSync: c.UnshareCgroup, + UnshareCgroupAfterSync: c.UnshareCgroup && !cmd.SyncAfter, } // starts the runner, error is handled same as wait4 to make communication equal pid, err := r.Start() diff --git a/container/host_exec_linux.go b/container/host_exec_linux.go index f61d662..32de90f 100644 --- a/container/host_exec_linux.go +++ b/container/host_exec_linux.go @@ -25,6 +25,9 @@ type ExecveParam struct { // ExecFile specifies file descriptor for executable file using fexecve ExecFile uintptr + // CgroupFD specifies file descriptor for cgroup V2 + CgroupFD uintptr + // RLimits specifies POSIX Resource limit through setrlimit RLimits []rlimit.RLimit @@ -54,6 +57,9 @@ func (c *container) Execve(ctx context.Context, param ExecveParam) runner.Result if param.ExecFile > 0 { files = append(files, int(param.ExecFile)) } + if param.CgroupFD > 0 { + files = append(files, int(param.CgroupFD)) + } files = append(files, uintptrSliceToInt(param.Files)...) msg := unixsocket.Msg{ Fds: files, diff --git a/container/protocol_linux.go b/container/protocol_linux.go index bec79e7..a7ecce1 100644 --- a/container/protocol_linux.go +++ b/container/protocol_linux.go @@ -41,6 +41,7 @@ type execCmd struct { RLimits []rlimit.RLimit // execve posix rlimit Seccomp seccomp.Filter // seccomp filter FdExec bool // if use fexecve (fd[0] as exec) + FdCgroup bool // if use cgroupFd CTTY bool // if set CTTY SyncAfter bool // if sync function calls after execve returns } diff --git a/pkg/cgroup/cgroup_linux.go b/pkg/cgroup/cgroup_linux.go index 11c5df1..f751ed6 100644 --- a/pkg/cgroup/cgroup_linux.go +++ b/pkg/cgroup/cgroup_linux.go @@ -56,6 +56,9 @@ type Cgroup interface { // Random creates a sub-cgroup based on the existing one but the name is randomly generated Random(string) (Cgroup, error) + + // Open opens the cgroup directory on V2 (used for clone3) + Open() (*os.File, error) } // DetectedCgroupType defines the current cgroup type of the system diff --git a/pkg/cgroup/v1_linux.go b/pkg/cgroup/v1_linux.go index 8364f98..04b14e2 100644 --- a/pkg/cgroup/v1_linux.go +++ b/pkg/cgroup/v1_linux.go @@ -26,6 +26,10 @@ type V1 struct { existing bool } +func (c *V1) Open() (*os.File, error) { + return nil, ErrNotInitialized +} + func (c *V1) String() string { names := make([]string, 0, numberOfControllers) for _, v := range []struct { diff --git a/pkg/cgroup/v2_linux.go b/pkg/cgroup/v2_linux.go index d4a07c8..5785dc1 100644 --- a/pkg/cgroup/v2_linux.go +++ b/pkg/cgroup/v2_linux.go @@ -21,6 +21,10 @@ type V2 struct { var _ Cgroup = &V2{} +func (c *V2) Open() (*os.File, error) { + return os.OpenFile(c.path, 0, dirPerm) +} + func (c *V2) String() string { ct, _ := getAvailableControllerV2path(path.Join(c.path, cgroupControllers)) return "v2(" + c.path + ")" + ct.String()