diff --git a/README.md b/README.md index fed0864..29519ec 100644 --- a/README.md +++ b/README.md @@ -34,7 +34,8 @@ Default file access action: Packages: - seccomp: provides utility function that wrappers libseccomp -- forkexec: fork-exec provides ptrace and seccomp before exec +- mount: provides utility function that wrappers mount syscall +- forkexec: fork-exec provides unshare, ptrace, seccomp before exec - tracer: ptrace tracer and provides syscall trap filter context - runprogram: wrapper to call forkexec and trecer - runconfig: defines arch & language specified trace condition diff --git a/forkexec/consts.go b/forkexec/consts.go new file mode 100644 index 0000000..4f3b6fa --- /dev/null +++ b/forkexec/consts.go @@ -0,0 +1,23 @@ +package forkexec + +import "syscall" + +// defines missing consts from syscall package +const ( + SECCOMP_SET_MODE_STRICT = 0 + SECCOMP_SET_MODE_FILTER = 1 + SECCOMP_FILTER_FLAG_TSYNC = 1 + + // CLONE_NEWCGOUP is not included + UnshareFlags = syscall.CLONE_NEWIPC | syscall.CLONE_NEWNET | syscall.CLONE_NEWNS | + syscall.CLONE_NEWPID | syscall.CLONE_NEWUSER | syscall.CLONE_NEWUTS +) + +// used by unshare remount / to private +var ( + none = [...]byte{'n', 'o', 'n', 'e', 0} + slash = [...]byte{'/', 0} + + // tmp dir made by pivot_root + OldRoot = "old_root" +) diff --git a/forkexec/fork.go b/forkexec/fork.go index 06322f4..69ac3ff 100644 --- a/forkexec/fork.go +++ b/forkexec/fork.go @@ -24,45 +24,34 @@ func afterForkInChild() //go:norace func (r *Runner) Start() (int, error) { var ( - err1 syscall.Errno - workdir *byte - nextfd int + err1 syscall.Errno ) - // make exec args0 - argv0, err := syscall.BytePtrFromString(r.Args[0]) - if err != nil { - return 0, err - } - // make exec args - argv, err := syscall.SlicePtrFromStrings(r.Args) - if err != nil { - return 0, err - } - // make env - envv, err := syscall.SlicePtrFromStrings(r.Env) + argv0, argv, envv, err := prepareExec(r.Args, r.Env) if err != nil { return 0, err } - // make work dir - if r.WorkDir != "" { - workdir, err = syscall.BytePtrFromString(r.WorkDir) - if err != nil { - return 0, err - } + // prepare work dir + workdir, err := syscallStringFromString(r.WorkDir) + if err != nil { + return 0, err + } + + // prepare pivot_root param + pivotRoot, oldRoot, err := preparePivotRoot(r.PivotRoot) + if err != nil { + return 0, err + } + + // prepare mount param + mountParams, dirsToMake, err := prepareMounts(r.Mounts) + if err != nil { + return 0, nil } // similar to exec_linux, avoid side effect by shuffling around - fd := make([]int, len(r.Files)) - nextfd = len(r.Files) - for i, ufd := range r.Files { - if nextfd < int(ufd) { - nextfd = int(ufd) - } - fd[i] = int(ufd) - } - nextfd++ + fd, nextfd := prepareFds(r.Files) // Acquire the fork lock so that no other threads // create new fds that are not yet close-on-exec @@ -73,7 +62,8 @@ func (r *Runner) Start() (int, error) { // No more allocation or calls of non-assembly functions. beforeFork() - pid, _, err1 := syscall.RawSyscall6(syscall.SYS_CLONE, uintptr(syscall.SIGCHLD), 0, 0, 0, 0, 0) + // UnshareFlags (new namespaces) is activated by clone syscall + pid, _, err1 := syscall.RawSyscall6(syscall.SYS_CLONE, uintptr(syscall.SIGCHLD)|(r.UnshareFlags&UnshareFlags), 0, 0, 0, 0, 0) if err1 != 0 || pid != 0 { // restore all signals afterFork() @@ -89,30 +79,13 @@ func (r *Runner) Start() (int, error) { afterForkInChild() // Notice: cannot call any GO functions beyond this point - // Set the pgid, so that the wait operation can apply to only certain - // subgroup of processes - _, _, err1 = syscall.RawSyscall(syscall.SYS_SETPGID, 0, 0, 0) + // Get pid of child + pid, _, err1 = syscall.RawSyscall(syscall.SYS_GETPID, 0, 0, 0) if err1 != 0 { goto childerror } - // Set limit - for _, rlim := range r.RLimits { - // Prlimit instead of setrlimit to avoid 32-bit limitation (linux > 3.2) - _, _, err1 = syscall.RawSyscall6(syscall.SYS_PRLIMIT64, 0, uintptr(rlim.Res), uintptr(unsafe.Pointer(&rlim.Rlim)), 0, 0, 0) - if err1 != 0 { - goto childerror - } - } - - // Chdir if needed - if workdir != nil { - _, _, err1 = syscall.RawSyscall(syscall.SYS_CHDIR, uintptr(unsafe.Pointer(workdir)), 0, 0) - if err1 != 0 { - goto childerror - } - } - + // Pass 1 & pass 2 assigns fds for child process // Pass 1: fd[i] < i => nextfd for i := 0; i < len(fd); i++ { if fd[i] >= 0 && fd[i] < int(i) { @@ -147,54 +120,142 @@ func (r *Runner) Start() (int, error) { } } - // Enable Ptrace - _, _, err1 = syscall.RawSyscall(syscall.SYS_PTRACE, uintptr(syscall.PTRACE_TRACEME), 0, 0) + // Set the pgid, so that the wait operation can apply to only certain + // subgroup of processes + _, _, err1 = syscall.RawSyscall(syscall.SYS_SETPGID, 0, 0, 0) if err1 != 0 { goto childerror } - // Load seccomp, stop and wait for tracer - if r.BPF != nil { - // Check if support - // SECCOMP_SET_MODE_STRICT = 0, args = 1 for invalid operation - _, _, err1 = syscall.RawSyscall(unix.SYS_SECCOMP, 0, 1, 0) - if err1 != syscall.EINVAL { - goto childerror - } - - // Load the filter manually - // No new privs - _, _, err1 = syscall.RawSyscall6(syscall.SYS_PRCTL, unix.PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0, 0) - if err1 != 0 { - goto childerror - } - - // If execve is seccomp trapped, then tracee stop is necessary - // otherwise execve will fail due to ENOSYS - // Do getpid and kill to send SYS_KILL to self - // need to do before seccomp as these might be traced - // Get pid of child - pid, _, err1 = syscall.RawSyscall(syscall.SYS_GETPID, 0, 0, 0) - if err1 != 0 { - goto childerror - } - - // Stop to wait for tracer - _, _, err1 = syscall.RawSyscall(syscall.SYS_KILL, pid, uintptr(syscall.SIGSTOP), 0) - if err1 != 0 { - goto childerror - } - - // Load seccomp filter - // SECCOMP_SET_MODE_FILTER = 1 - // SECCOMP_FILTER_FLAG_TSYNC = 1 - _, _, err1 = syscall.RawSyscall(unix.SYS_SECCOMP, 1, 1, uintptr(unsafe.Pointer(r.BPF))) + // If mount point is unshared, mark root as private to avoid propagate + // outside to the original mount namespace + if r.UnshareFlags&syscall.CLONE_NEWNS == syscall.CLONE_NEWNS { + _, _, err1 = syscall.RawSyscall6(syscall.SYS_MOUNT, uintptr(unsafe.Pointer(&none[0])), + uintptr(unsafe.Pointer(&slash[0])), 0, syscall.MS_REC|syscall.MS_PRIVATE, 0, 0) if err1 != 0 { goto childerror } } - // at this point, tracer is successfully attached for seccomp trap filter + // chdir to new root before performing mounts + if pivotRoot != nil { + _, _, err1 = syscall.RawSyscall(syscall.SYS_CHDIR, uintptr(unsafe.Pointer(pivotRoot)), 0, 0) + if err1 != 0 { + goto childerror + } + } + + // performing mounts + for i, m := range mountParams { + // mkdirs(target) + for _, p := range dirsToMake[i] { + _, _, err1 = syscall.RawSyscall(syscall.SYS_MKDIR, uintptr(unsafe.Pointer(p)), 0755, 0) + if err1 != 0 { + goto childerror + } + } + // mount(source, target, fsType, flags, data) + _, _, err1 = syscall.RawSyscall6(syscall.SYS_MOUNT, uintptr(unsafe.Pointer(m.Source)), + uintptr(unsafe.Pointer(m.Target)), uintptr(unsafe.Pointer(m.FsType)), uintptr(m.Flags), + uintptr(unsafe.Pointer(m.Data)), 0) + if err1 != 0 { + goto childerror + } + } + + // pivit_root + if pivotRoot != nil { + // mkdir(root/old_root) + _, _, err1 = syscall.RawSyscall(syscall.SYS_MKDIR, uintptr(unsafe.Pointer(oldRoot)), 0755, 0) + if err1 != 0 { + goto childerror + } + + // pivot_root(root, root/old_root) + _, _, err1 = syscall.RawSyscall(syscall.SYS_MKDIR, uintptr(unsafe.Pointer(pivotRoot)), uintptr(unsafe.Pointer(oldRoot)), 0) + if err1 != 0 { + goto childerror + } + + // umount(root/old, MNT_DETACH) + _, _, err1 = syscall.RawSyscall(syscall.SYS_UMOUNT2, uintptr(unsafe.Pointer(oldRoot)), syscall.MNT_DETACH, 0) + if err1 != 0 { + goto childerror + } + + // rmdir(root/old_root) + _, _, err1 = syscall.RawSyscall(syscall.SYS_RMDIR, uintptr(unsafe.Pointer(oldRoot)), 0755, 0) + if err1 != 0 { + goto childerror + } + } + + // chdir for child + if workdir != nil { + _, _, err1 = syscall.RawSyscall(syscall.SYS_CHDIR, uintptr(unsafe.Pointer(workdir)), 0, 0) + if err1 != 0 { + goto childerror + } + } + + // Set limit + for _, rlim := range r.RLimits { + // Prlimit instead of setrlimit to avoid 32-bit limitation (linux > 3.2) + _, _, err1 = syscall.RawSyscall6(syscall.SYS_PRLIMIT64, 0, uintptr(rlim.Res), uintptr(unsafe.Pointer(&rlim.Rlim)), 0, 0, 0) + if err1 != 0 { + goto childerror + } + } + + // No new privs + if r.NoNewPrivs || r.Seccomp != nil { + _, _, err1 = syscall.RawSyscall6(syscall.SYS_PRCTL, unix.PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0, 0) + if err1 != 0 { + goto childerror + } + } + + // Enable Ptrace + if r.Ptrace { + _, _, err1 = syscall.RawSyscall(syscall.SYS_PTRACE, uintptr(syscall.PTRACE_TRACEME), 0, 0) + if err1 != 0 { + goto childerror + } + } + + // if both seccomp and ptrace is defined, then seccomp filter should have + // traced execve, thus child need parent attached to it first + if r.StopBeforeSeccomp || (r.Seccomp != nil && r.Ptrace) { + // Stop to wait for ptrace tracer + _, _, err1 = syscall.RawSyscall(syscall.SYS_KILL, pid, uintptr(syscall.SIGSTOP), 0) + if err1 != 0 { + goto childerror + } + } + + // Load seccomp, stop and wait for tracer + if r.Seccomp != nil { + // If execve is seccomp trapped, then tracee stop is necessary + // otherwise execve will fail due to ENOSYS + // Do getpid and kill to send SYS_KILL to self + // need to do before seccomp as these might be traced + + // Load seccomp filter + _, _, err1 = syscall.RawSyscall(unix.SYS_SECCOMP, SECCOMP_SET_MODE_FILTER, SECCOMP_FILTER_FLAG_TSYNC, uintptr(unsafe.Pointer(r.Seccomp))) + if err1 != 0 { + goto childerror + } + } + + // stop before execve syscall + if r.StopBeforeExec { + _, _, err1 = syscall.RawSyscall(syscall.SYS_KILL, pid, uintptr(syscall.SIGSTOP), 0) + if err1 != 0 { + goto childerror + } + } + + // at this point, runner is successfully attached for seccomp trap filter // or execve traped without seccomp filter // time to exec _, _, err1 = syscall.RawSyscall(syscall.SYS_EXECVE, diff --git a/forkexec/fork_util.go b/forkexec/fork_util.go new file mode 100644 index 0000000..7e7bbfa --- /dev/null +++ b/forkexec/fork_util.go @@ -0,0 +1,98 @@ +package forkexec + +import ( + "path" + "syscall" + + "github.com/criyle/go-judger/mount" +) + +// prepareExec prepares execve parameters +func prepareExec(Args, Env []string) (*byte, []*byte, []*byte, error) { + // make exec args0 + argv0, err := syscall.BytePtrFromString(Args[0]) + if err != nil { + return nil, nil, nil, err + } + // make exec args + argv, err := syscall.SlicePtrFromStrings(Args) + if err != nil { + return nil, nil, nil, err + } + // make env + envv, err := syscall.SlicePtrFromStrings(Env) + if err != nil { + return nil, nil, nil, err + } + return argv0, argv, envv, nil +} + +// prepareFds prapares fd array +func prepareFds(files []uintptr) ([]int, int) { + fd := make([]int, len(files)) + nextfd := len(files) + for i, ufd := range files { + if nextfd < int(ufd) { + nextfd = int(ufd) + } + fd[i] = int(ufd) + } + nextfd++ + return fd, nextfd +} + +// syscallStringFromString prepares *byte if string is not empty, other wise nil +func syscallStringFromString(str string) (*byte, error) { + if str != "" { + return syscall.BytePtrFromString(str) + } + return nil, nil +} + +// preparePivotRoot prepares pivot root parameters +func preparePivotRoot(r string) (*byte, *byte, error) { + if r == "" { + return nil, nil, nil + } + or := path.Join(r, OldRoot) + root, err := syscall.BytePtrFromString(r) + if err != nil { + return nil, nil, err + } + oldRoot, err := syscall.BytePtrFromString(or) + if err != nil { + return nil, nil, err + } + return root, oldRoot, nil +} + +// prepareMounts prepare mkdir and mount syscall params +func prepareMounts(ms []*mount.Mount) ([]*mount.SyscallParams, [][]*byte, error) { + params, err := mount.ToSyscalls(ms) + if err != nil { + return nil, nil, err + } + // evaluate paths that need to be created + pathsToCreate := make([][]*byte, 0, len(ms)) + for _, m := range ms { + prefix := pathPrefix(m.Target) + paths, err := syscall.SlicePtrFromStrings(prefix) + if err != nil { + return nil, nil, err + } + pathsToCreate = append(pathsToCreate, paths) + } + return params, pathsToCreate, nil +} + +// pathPrefix get all components from path +func pathPrefix(path string) []string { + ret := make([]string, 0) + for i := 1; i < len(path); i++ { + if path[i] == '/' { + ret = append(ret, path[:i]) + } + } + ret = append(ret, path) + return ret +} diff --git a/forkexec/rlimit.go b/forkexec/rlimit.go new file mode 100644 index 0000000..a97f631 --- /dev/null +++ b/forkexec/rlimit.go @@ -0,0 +1,11 @@ +package forkexec + +import "syscall" + +// RLimit is the resource limits defined by Linux setrlimit +type RLimit struct { + // Res is the resource type (e.g. syscall.RLIMIT_CPU) + Res int + // Rlim is the limit applied to that resource + Rlim syscall.Rlimit +} diff --git a/forkexec/runner.go b/forkexec/runner.go index e73c2cf..47813e2 100644 --- a/forkexec/runner.go +++ b/forkexec/runner.go @@ -2,12 +2,17 @@ // executable and ptraced package forkexec -import "syscall" +import ( + "syscall" + + "github.com/criyle/go-judger/mount" +) // Runner is the RunProgramConfig including the exec path, argv // and resource limits. It creates tracee for ptrace-based tracer. +// It can also create unshared process in another namespace type Runner struct { - // argv and env for the child process + // argv and env for execve syscall for the child process Args []string Env []string @@ -17,17 +22,53 @@ type Runner struct { // file disriptors for new process, from 0 to len - 1 Files []uintptr - // work path set by setcwd (current working directory for child) + // work path set by chdir(dir) (current working directory for child) + // if pivot_root is defined, this will execute after changed to new root WorkDir string - // BPF syscall filter applied to child - BPF *syscall.SockFprog -} + // seccomp syscall filter applied to child + Seccomp *syscall.SockFprog -// RLimit is the resource limits defined by Linux setrlimit -type RLimit struct { - // Res is the resource type (e.g. syscall.RLIMIT_CPU) - Res int - // Rlim is the limit applied to that resource - Rlim syscall.Rlimit + // ptrace controls child process to call ptrace(PTRACE_TRACEME) + // runtime.LockOSThread is required for tracer to call ptrace syscalls + Ptrace bool + + // no_new_privs calls ptctl(PR_SET_NO_NEW_PRIVS) to 0 to disable calls to + // setuid processes. It is automatically enabled when seccomp filter is provided + NoNewPrivs bool + + // stop before seccomp calls kill(getpid(), SIGSTOP) to wait for tracer to continue + // right before the calls to seccomp. It is automatically enabled when seccomp + // filter and ptrace are provided since kill might not be avaliable after + // seccomp and execve might be traced by ptrace + StopBeforeSeccomp bool + + // stop before exec calls kill(getpid(), SIGSTOP) to wait for tracer to continue + // right before the final calls to execve. It acts as a synchronize + // for parent process + StopBeforeExec bool + + // unshare flag to create linux namespace, effective when clone child + // since unshare syscall does not join the new pid group + UnshareFlags uintptr + + // mounts defines the mount syscalls after unshare mount namespace + // need CAP_ADMIN inside the namespace (e.g. unshare user namespace) + // if pivot root is provided, relative target will based on PivotRoot directory + Mounts []*mount.Mount + + // pivot_root defines the new root after unshare mount namespace + // root need to be a mount point (e.g. defined in Mounts) and it should + // be a absolute path + // It will call: + // mkdir(root/old_root) + // pivot_root(root, root/old_root) + // umount(root/old, MNT_DETACH) + // rmdir(root/old_root) + PivotRoot string + + // drop_caps calls cap_set(self, 0) to drop all capabilities + // from effective, permitted, inheritable capability sets before execve + // it should avoid calls to set + DropCaps bool } diff --git a/go.sum b/go.sum index 5bd7052..f4f8ecc 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,4 @@ +github.com/seccomp/libseccomp-golang v0.9.1 h1:NJjM5DNFOs0s3kYE1WUOr6G8V97sdt46rlXTMfXGWBo= github.com/seccomp/libseccomp-golang v0.9.1/go.mod h1:GbW5+tmTXfcxTToHLXlScSlAvWlF4P2Ca7zGrPiEpWo= golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb h1:fgwFCsaw9buMuxNd6+DQfAuSFqbNiQZpcgJQAgJsK6k= golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= diff --git a/mount/mount.go b/mount/mount.go new file mode 100644 index 0000000..92574ac --- /dev/null +++ b/mount/mount.go @@ -0,0 +1,81 @@ +package mount + +import ( + "os" + "syscall" +) + +// Mount defines syscall for mount points +type Mount struct { + Source, Target, FsType, Data string + Flags uintptr +} + +// SyscallParams defines the raw syscall arguments to mount +type SyscallParams struct { + Source, Target, FsType, Data *byte + Flags uintptr +} + +// ToSyscall convert Mount to SyscallPrams +func (m *Mount) ToSyscall() (*SyscallParams, error) { + var data *byte + source, err := syscall.BytePtrFromString(m.Source) + if err != nil { + return nil, err + } + target, err := syscall.BytePtrFromString(m.Target) + if err != nil { + return nil, err + } + fsType, err := syscall.BytePtrFromString(m.FsType) + if err != nil { + return nil, err + } + if m.Data != "" { + data, err = syscall.BytePtrFromString(m.Data) + if err != nil { + return nil, err + } + } + return &SyscallParams{ + Source: source, + Target: target, + FsType: fsType, + Flags: m.Flags, + Data: data, + }, nil +} + +// Mount calls mount syscall +func (m *Mount) Mount() error { + err := os.MkdirAll(m.Target, 0755) + if err != nil { + return err + } + return syscall.Mount(m.Source, m.Target, m.FsType, m.Flags, m.Data) +} + +// ToSyscalls converts arrays of Mounts into SyscallParams +func ToSyscalls(ms []*Mount) ([]*SyscallParams, error) { + ret := make([]*SyscallParams, 0, len(ms)) + for _, m := range ms { + sp, err := m.ToSyscall() + if err != nil { + return nil, err + } + ret = append(ret, sp) + } + return ret, nil +} + +// Mounts calls multiple mount syscalls +func Mounts(ms []*Mount) error { + for _, m := range ms { + err := m.Mount() + if err != nil { + return err + } + } + return nil +} diff --git a/runprogram/runprogram_run.go b/runprogram/runprogram_run.go index 57f6e30..04389e6 100644 --- a/runprogram/runprogram_run.go +++ b/runprogram/runprogram_run.go @@ -30,7 +30,8 @@ func (r *RunProgram) Start() (rt tracer.TraceResult, err error) { RLimits: r.RLimits.prepareRLimit(), Files: r.Files, WorkDir: r.WorkDir, - BPF: bpf, + Seccomp: bpf, + Ptrace: true, } th := &tracerHandler{ diff --git a/tracer/tracer_track.go b/tracer/tracer_track.go index a82e4f1..835072b 100644 --- a/tracer/tracer_track.go +++ b/tracer/tracer_track.go @@ -35,6 +35,7 @@ func Trace(handler Handler, runner Runner, limits ResLimit) (result TraceResult, pgid, err := runner.Start() handler.Debug("tracer started: ", pgid, err) if err != nil { + handler.Debug("start tracee failed: ", err) result.TraceStatus = TraceCodeRE return result, err }