Compare commits

...

110 Commits

Author SHA1 Message Date
tobiichi3227
0baa3107cb
forkexec: ensure bind mount preserve the mount flag from source (#17) 2025-10-11 16:11:06 -04:00
criyle
63db3b54e6 mount: ensure bind mount preserve the mount flag from source 2025-08-23 20:21:02 +00:00
criyle
5f66dc11f5 fix(config): fix syscall check on arm64 2025-08-13 22:22:53 +00:00
criyle
2d0d63be76 build(deps): update upstream seccomp library to fix ptracer
elastic/go-seccomp-bpf#40
fixes #8
2025-06-21 16:54:37 -04:00
criyle
7aa1c5a28b refactor: normalize error message & replace path with filepath 2025-05-25 22:02:23 -04:00
criyle
c63c27d3c2 test(pkg): add some unit tests 2025-05-25 20:24:59 -04:00
DNEGEL3125
638345b6eb
test(fileset): add unit tests for FileSet (#15)
* test(FileSet): add unit test for IsInSetSmart

* test(FileSet): add unit test for Add

* test(fileset): add unit test for AddRange
2025-05-22 13:21:42 -04:00
criyle
1e890d6475 container: skip waitAll 2025-04-03 23:21:14 +00:00
criyle
ea0b084794 runprog: ensure alway re-evaluate
related #14
2025-03-14 02:45:32 -04:00
criyle
98f24cf60a runprog: re-evaluate status after retrieve cgroup measurement
related #14
2025-03-14 02:41:55 -04:00
criyle
c316900c2c build: go mod tidy 2025-03-12 15:25:22 -04:00
criyle
f9fbaff75a build: go 1.24 2025-03-12 15:23:29 -04:00
criyle
2ba894ffd1 vfork: fix asm on other architectures
doc: update

#13
2025-02-22 21:30:33 +00:00
criyle
10235abbf2 container: organize test cases 2025-02-22 21:07:11 +00:00
criyle
396ac9723e forkexec: support vfork
(#13)
2025-02-22 20:04:15 +00:00
criyle
f9deb2dc1a container: add unshare cgroup before exec
#13
2025-02-20 23:09:50 -05:00
criyle
f0dd44f466 container: fix cgroup fd
forkexec: disabled VFORK for now as it is not working properly
#13
2025-02-20 22:52:26 -05:00
criyle
659099a2c0 forkexec: use vfork if no sync needed
#13
2025-02-20 22:35:09 +00:00
criyle
f8361a08a0 forkexec: only sync when syncFunc exists
#13
2025-02-20 21:45:03 +00:00
criyle
d32acd7591 cgroup&container: add ability to utilize cgroup fd with clone3
#13
2025-02-20 20:29:20 +00:00
criyle
f15b953065 container: syncAfterExec to sync after execve rather than before 2025-02-20 19:48:39 +00:00
criyle
3d8333e952 forkexec: initial support for clone3 2025-02-20 03:56:49 -05:00
criyle
f6f057edf9 cgroup: use rmdir to delete cgroup 2025-02-20 04:19:59 +00:00
criyle
51b53cf60b runner: add procPeak as measurement of peak process count 2025-02-20 03:12:28 +00:00
criyle
cf936ae31f build(deps): update dependencies and go 1.23 2025-02-20 02:04:22 +00:00
criyle
0b6b557947 cgroup: add pids.peak for cgroup v2 kernel >= 6.1 2025-02-20 01:51:59 +00:00
criyle
4154f44d83 forkexec: add more benchmark for std fork 2025-02-19 00:03:36 -05:00
Yang Gao
7ffa0ff148
Merge pull request #12 from Zqzqsb/fix-vmReadStr 2025-02-11 03:14:52 -05:00
ZQ
25c32240cf test(ptracer): add helper func to find readable memory 2025-02-11 13:45:35 +08:00
ZQ
9df6a51947 fix(ptracer): delete min func 2025-02-11 13:14:49 +08:00
ZQ
8b7494f796 fix(ptracer): fix vmReadStr logic error 2025-02-11 12:11:28 +08:00
ZQ
99241ac91a test(ptracer): add testfile and verify vmReadStr problem 2025-02-11 12:09:10 +08:00
criyle
3fb1ccd890 pkg/cgroup: add enabled controler check for v2 2025-01-26 00:38:08 -05:00
criyle
144d408a6d build(deps): update dependencies 2025-01-25 23:41:30 -05:00
Yang Gao
fa1ba2f855
Merge pull request #10 from BoYanZh/patch-1
fix(pkg/cgroup): nil pointer dereference
2025-01-10 13:26:51 -05:00
BoYanZh
6c726c9b94
fix(pkg/cgroup): nil pointer dereference
when `build` is `V2.New`, `cg` will be `nil` if it is returned from `if !os.IsExist(err) {`
2025-01-10 03:42:03 -05:00
criyle
81eae7a6d5 build(deps): update dependencies 2024-12-13 18:28:00 +00:00
criyle
5bfbd6c8fe forkexec: fix unshare cgroup with ptracer 2024-06-14 10:26:07 +00:00
criyle
6d21194b72 mount: allow read write mount for /proc
criyle/go-judge#106

Co-authored-by: iBug <git@ibugone.com>
2024-05-23 10:21:46 +00:00
criyle
60a64d0fb1 container: add option to run init command after container initialization 2024-04-16 10:47:48 +00:00
criyle
598c606a99 deps: upgrade to rand/v2 2024-04-05 04:24:10 +00:00
criyle
f6274fd54c cgroup: rename types to remove stutters 2024-02-05 08:53:37 +00:00
criyle
ed93876f8c *: fix typos 2024-02-03 16:19:58 +00:00
criyle
231f3cb58f cgroup: refactor to support creating child cgroup directly 2023-11-09 10:21:07 +00:00
dependabot[bot]
5a772dcd4f
Bump golang.org/x/net from 0.15.0 to 0.17.0 (#6) 2023-10-14 01:57:11 +00:00
criyle
bd846192aa deps: upgrade to go1.21 2023-09-27 13:43:12 +00:00
criyle
69e069ba8f doc: update kernel version doc and dependency versions 2023-08-10 13:09:27 +00:00
Yang Gao
7aeb9869c5
Merge pull request #3 from AlphaNecron/max_mem_cgv2
use memory.peak for cgroup v2 to replicate max_usage_in_bytes in v1 (for kernel >= 5.19)
2023-08-10 20:34:28 +08:00
Nguyen Thanh Quang
656b493a3f
use memory.peak for cgroup v2 as an alternative to max_usage_in_bytes 2023-08-04 23:12:23 +07:00
criyle
25d0f22364 *: bump dependency version & better align 2023-04-12 01:19:47 -07:00
criyle
cce22e7ac8 *: bump dependency version with go1.20 2023-02-19 01:06:21 -08:00
criyle
7af58bbeb9 forkexec: build for go1.19 2022-10-01 18:49:58 -07:00
criyle
69f0dffbeb container: fix file exists for bind mount 2022-08-27 00:29:34 -07:00
criyle
84888375c4 container: load default environment variable from /.env 2022-08-25 20:33:48 -07:00
criyle
f308637481 forkexec: fix child error report 2022-08-13 01:32:16 -07:00
criyle
47ad6f0016 container: look up executable by PATH 2022-08-13 01:15:37 -07:00
criyle
7e7babb354 container: ignore signals for init 2022-08-09 21:25:27 -07:00
criyle
5eeaadffdf container: add support for temp root dir 2022-07-15 23:46:42 -07:00
criyle
701fed8d1c forkexec: fix deadlock in go scheduler when syncWithChild
- also fix broken pipe when using fd as executable

resolves #2
2022-04-22 15:42:56 -07:00
criyle
ca60fcad98 *: fix macOS build for go1.18 2022-04-03 02:45:53 -07:00
criyle
8fc50bc5b7 *: bump to go1.18 2022-03-20 23:36:30 -07:00
criyle
2f85a420f9 cgroup: fix v1 set cpu bandwidth 2022-03-01 00:00:59 -08:00
criyle
9baa4110cd *: bump dependecy version with go1.17.6 2022-01-08 18:48:07 -08:00
criyle
c6519b09ec cgroup: add support of cgroup v2 nesting 2021-12-25 23:04:01 +00:00
criyle
5e5b00688a cgroup: implement cgroup v2 2021-12-24 10:42:42 +00:00
criyle
2889743b71 cgroup: initialize support for v2 implementation 2021-12-23 20:38:31 -08:00
criyle
015f569ebe container: add maskPaths ability 2021-11-28 23:02:14 -08:00
criyle
76bbdf9ede forkexec: fix unit tests 2021-11-19 15:01:34 -08:00
criyle
b5d300a0e8 rlimit: restrict open file count 2021-11-12 00:23:34 -08:00
criyle
eb0f2cbc89 container: create symlinks for rootfs 2021-11-11 23:56:53 -08:00
criyle
383f40cabe *: bump dependency versions 2021-10-30 10:58:06 -07:00
criyle
f965052804 mount: remove noatime from proc for compatibility 2021-09-20 18:33:48 -07:00
criyle
fded6ba8c7 *: bump dependency versions & go1.17 build tag 2021-09-19 14:10:44 -07:00
criyle
06ed42eb42 mount: add recursive flag to bind mounts 2021-09-19 13:35:11 -07:00
criyle
eec1a5a9e3 *: bump dependency versions with go1.17.1 2021-09-12 13:39:26 -07:00
criyle
d94a0d6acd *: bump to go1.17 2021-08-18 20:21:28 -07:00
criyle
5f3a6ff032 container: add CAP_SYS_RESOURCE 2021-08-09 23:52:43 -07:00
criyle
4e55516ff3 forkexec: report error location 2021-08-04 22:41:36 -07:00
criyle
26c5f63dcb runner: async -> sync 2021-06-05 16:16:05 -07:00
criyle
963e027b0a container: allocate buffers in advance 2021-04-25 17:58:00 -07:00
criyle
c1d9a5572f container: fix fd leak 2021-03-30 21:36:46 -07:00
criyle
3b55a156bc container: host async wait 2021-03-29 22:49:40 -07:00
criyle
91dae1dfc6 container: change send & recv async 2021-03-29 01:16:45 -07:00
criyle
204d7949c6 unixsocket: non-blocking message & value passing 2021-03-27 19:16:41 -07:00
criyle
ebab07ec35 deprecate io/ioutil 2021-02-24 00:00:51 -08:00
criyle
4ee5c39f1e bump to go 1.16 2021-02-21 19:33:03 -08:00
criyle
9e8c4593c6 Update rlimit & remove discard reader 2021-02-06 12:21:22 -08:00
criyle
464c5ac932 Backwards compatibility for older kernels >= 3.10 2021-01-21 00:20:57 -08:00
criyle
29aefc839b misc benchmark 2021-01-16 18:19:25 -08:00
criyle
d51cbe0bbd bump versions 2021-01-09 11:21:32 -08:00
criyle
a17bcd105c small improvement 2021-01-08 23:50:40 -08:00
criyle
dbede6a2e4 go mod tidy 2020-12-23 12:19:23 -08:00
criyle
7603109e33 Refactor seccomp ptrace & improve container 2020-12-23 12:12:43 -08:00
criyle
3c77bf81d1 Fix cgroup potential null pointer 2020-12-17 22:55:48 -08:00
criyle
a9733a4b7d Fix container.reset 2020-12-17 22:27:18 -08:00
criyle
5343e365c3 Add cpu cgroup support 2020-12-12 21:28:41 -08:00
criyle
4ae1d6d760 Differentiate error from execve / wait4 2020-12-12 12:24:27 -08:00
criyle
7ebe4a999d Add ability to use seccomp filter for container 2020-12-12 12:16:28 -08:00
criyle
6df08e55b7 Add ability to config container uid & gid 2020-12-12 11:53:09 -08:00
criyle
928a0a4631 Fix typo for cpuacct 2020-12-06 18:33:16 -08:00
criyle
6d71f9dc4b Add support to cpuset cgroup 2020-12-06 17:48:10 -08:00
criyle
7ce98ffa42 Add ability to parse size from string 2020-12-05 13:50:32 -08:00
criyle
64df5951ed Fix work dir 2020-11-22 21:31:15 -08:00
criyle
c832edba58 Refactor container mounts 2020-11-21 23:50:06 -08:00
criyle
085f7a63c2 Fix unit tests & minor performance improvement 2020-09-21 00:17:10 -07:00
criyle
5e6fcd1d86 Split fork child & add unit tests 2020-09-06 00:11:52 -07:00
criyle
744db3ff88 Do not close read pipe after limit 2020-07-14 00:12:12 -07:00
criyle
687b9bf6f9 Add tty support ... 2020-06-02 17:21:42 -04:00
criyle
d664e1eae8 set pgid on MacOS 2020-05-15 19:01:10 -04:00
criyle
b6036eb830 Remove libseccomp dependency 2020-05-15 17:13:13 -04:00
114 changed files with 5885 additions and 1948 deletions

2
.gitignore vendored
View File

@ -8,3 +8,5 @@ env*.sh
# Test Files
/runprog
/test
.vscode

View File

@ -6,13 +6,16 @@ Original goal was to replica [uoj-judger/run_program](https://github.com/vfleaki
The idea of rootfs and interval CPU usage checking comes from [syzoj/judge-v3](https://github.com/syzoj/judge-v3) and the pooled pre-forked container comes from [vijos/jd4](https://github.com/vijos/jd4).
If you are looking for sandbox implementation via REST / gRPC API, please check [go-judge](https://github.com/criyle/go-judge).
Notice: Only works on Linux since ptrace, unshare, cgroup are available only on Linux
## Build & Install
- install latest go compiler from [golang/download](https://golang.org/dl/)
- install libseccomp library: (for Ubuntu) `apt install libseccomp-dev`
- build & install: `go install github.com/criyle/go-sandbox/...`
- download repository: `git clone githuc.com/criyle/go-sandbox`
- build: `go build ./cmd/runprog`
- or install directly: `go install github.com/criyle/go-sandbox/cmd/runprog@latest`
## Technologies
@ -39,10 +42,16 @@ Default file access syscall check:
### linux namespace + cgroup
1. Unshare & bind mount rootfs based on hostfs (elimilated ptrace)
2. Use Linux Control Groups to limit & acct CPU & memory (elimilate wait4.rusage)
1. Unshare & bind mount rootfs based on hostfs (eliminated ptrace)
2. Use Linux Control Groups to limit & acct CPU & memory (eliminated wait4.rusage)
3. Container tech with execveat memfd, sethostname, setdomainname
### prefork containers
Utilize the linux namespace + cgroup but create container in advance to reduce the duplicated effort of creating mount points. See Pre-forked container protocol and environment for design details.
On kernel >= 5.7 with cgroup v2, the new `clone3(CLONE_INTO_CGROUP)` with `vfork` is available to reduce the resource consumption of create new address spaces as well.
## Design
### Result Status
@ -154,7 +163,7 @@ type Environment interface {
- seccomp: provides seccomp type definition
- libseccomp: provides utility function that wrappers libseccomp
- forkexec: fork-exec provides mount, unshare, ptrace, seccomp, capset before exec
- memfd: read regular file and creates a seaed memfd for its contents
- memfd: read regular file and creates a sealed memfd for its contents
- unixsocket: send / recv oob msg from a unix socket
- cgroup: creates cgroup directories and collects resource usage / limits
- mount: provides utility function that wrappers mount syscall
@ -163,7 +172,7 @@ type Environment interface {
## Packages
- config: defines arch & language specified trace condition for ptrace runner from UOJ
- cmd/runprog/config: defines arch & language specified trace condition for ptrace runner from UOJ
- container: creates pre-forked container to run programs inside
- runner: interface to run program
- ptrace: wrapper to call forkexec and ptracer
@ -179,6 +188,22 @@ type Environment interface {
- config/config.go: all configs toward running specs (similar to UOJ)
## Kernel Versions
- 6.1: `pids.peak` in cgroup v2
- 5.19: `memory.peak` in cgroup v2
- 5.7: `clone3` with `CLONE_INTO_CGROUP`
- 5.3: `clone3`
- 4.15: cgroup v2 (also need support in the Linux distribution)
- 4.14: SECCOMP_RET_KILL_PROCESS
- 4.6: CLONE_NEWCGROUP
- 3.19: execveat()
- 3.17: seccomp, memfd_create
- 3.10: CentOS 7
- 3.8: CLONE_NEWUSER without CAP_SYS_ADMIN, CAP_SETUID, CAP_SETGID
- 3.5: prctl(PR_SET_NO_NEW_PRIVS)
- 2.6.36: prlimit64
## Benchmarks
### ForkExec
@ -226,3 +251,41 @@ BenchmarkCgroup-4 50283 245094 ns/op
PASS
ok github.com/criyle/go-sandbox/pkg/cgroup 14.744s
```
### Socket
Blocking:
```bash
$ go test -bench . -benchtime 10s
goos: linux
goarch: amd64
pkg: github.com/criyle/go-sandbox/pkg/unixsocket
cpu: Intel(R) Core(TM) i7-7700K CPU @ 4.20GHz
BenchmarkBaseline-8 12170148 1048 ns/op
BenchmarkGoroutine-8 2658846 4910 ns/op
BenchmarkChannel-8 8454133 1431 ns/op
BenchmarkChannelBuffed-8 8767264 1357 ns/op
BenchmarkChannelBuffed4-8 9670935 1230 ns/op
BenchmarkEmptyGoroutine-8 34927512 342.8 ns/op
PASS
ok github.com/criyle/go-sandbox/pkg/unixsocket 83.669s
```
Non-block:
```bash
$ go test -bench . -benchtime 10s
goos: linux
goarch: amd64
pkg: github.com/criyle/go-sandbox/pkg/unixsocket
cpu: Intel(R) Core(TM) i7-7700K CPU @ 4.20GHz
BenchmarkBaseline-8 11609772 1001 ns/op
BenchmarkGoroutine-8 2470767 4788 ns/op
BenchmarkChannel-8 8488646 1427 ns/op
BenchmarkChannelBuffed-8 8876050 1345 ns/op
BenchmarkChannelBuffed4-8 9813187 1212 ns/op
BenchmarkEmptyGoroutine-8 34852828 342.2 ns/op
PASS
ok github.com/criyle/go-sandbox/pkg/unixsocket 81.679s
```

View File

@ -32,11 +32,12 @@ var (
"fstat",
"lseek",
"dup",
"dup2",
"dup3",
"ioctl",
"fcntl",
"fadvise64",
"pread64",
"pwrite64",
// memory action
"mmap",
@ -63,13 +64,10 @@ var (
"exit_group",
// others
"arch_prctl",
"gettimeofday",
"getrlimit",
"getrusage",
"times",
"time",
"clock_gettime",
"restart_syscall",
@ -82,21 +80,15 @@ var (
"execveat",
// file open
"open",
"openat",
// file delete
"unlink",
"unlinkat",
// soft link
"readlink",
"readlinkat",
// permission check
"lstat",
"stat",
"access",
"faccessat",
}
@ -180,7 +172,7 @@ var (
"sched_getaffinity", "sched_yield",
"uname", "sysinfo",
"prlimit64", "getrandom",
"fchmodat",
"fchmodat", "rseq",
},
ExtraBan: []string{"socket", "connect", "geteuid", "getuid"},
},

View File

@ -0,0 +1,26 @@
package config
// This file includes configs for the run program settings
var (
archReadableFiles = []string{
"/lib/x86_64-linux-gnu/",
"/usr/lib/x86_64-linux-gnu/",
}
archSyscallAllows = []string{
"dup2",
"time",
"arch_prctl",
}
archSyscallTraces = []string{
"open",
"unlink",
"readlink",
"lstat",
"stat",
"access",
"newfstatat",
}
)

View File

@ -17,10 +17,19 @@ var (
"uname",
"set_tls",
"arm_fadvise64_64",
"dup2",
}
archSyscallTraces = []string{
"lstat64", // 32-bit
"stat64", // 32-bit
"open",
"unlink",
"readlink",
"lstat",
"stat",
"access",
"fstatat",
"fstatat64",
}
)

View File

@ -8,9 +8,9 @@ var (
"/usr/lib/aarch64-linux-gnu/",
}
archSyscallAllows = []string{
"newfstatat",
}
archSyscallAllows = []string{}
archSyscallTraces = []string{}
archSyscallTraces = []string{
"fstatat",
}
)

View File

@ -4,6 +4,8 @@ import (
"flag"
"fmt"
"os"
"github.com/criyle/go-sandbox/runner"
)
const (
@ -15,3 +17,45 @@ func printUsage() {
flag.PrintDefaults()
os.Exit(2)
}
// Status defines uoj/run_program constants
type Status int
// UOJ run_program constants
const (
StatusNormal Status = iota // 0
StatusInvalid // 1
StatusRE // 2
StatusMLE // 3
StatusTLE // 4
StatusOLE // 5
StatusBan // 6
StatusFatal // 7
)
func getStatus(s runner.Status) int {
switch s {
case runner.StatusNormal:
return int(StatusNormal)
case runner.StatusInvalid:
return int(StatusInvalid)
case runner.StatusTimeLimitExceeded:
return int(StatusTLE)
case runner.StatusMemoryLimitExceeded:
return int(StatusMLE)
case runner.StatusOutputLimitExceeded:
return int(StatusOLE)
case runner.StatusDisallowedSyscall:
return int(StatusBan)
case runner.StatusSignalled, runner.StatusNonzeroExitStatus:
return int(StatusRE)
default:
return int(StatusFatal)
}
}
func debug(v ...interface{}) {
if showDetails {
fmt.Fprintln(os.Stderr, v...)
}
}

View File

@ -4,8 +4,6 @@ package main
import (
"flag"
"fmt"
"io/ioutil"
"log"
"os"
"syscall"
"time"
@ -13,13 +11,15 @@ import (
"github.com/criyle/go-sandbox/pkg/forkexec"
"github.com/criyle/go-sandbox/pkg/rlimit"
"github.com/criyle/go-sandbox/runner"
"golang.org/x/sys/unix"
)
var (
timeLimit, realTimeLimit, memoryLimit, outputLimit, stackLimit uint64
inputFileName, outputFileName, errorFileName, workPath string
profilePath string
profilePath, result string
showDetails bool
args []string
)
@ -36,6 +36,8 @@ func main() {
flag.StringVar(&errorFileName, "err", "", "Set error file name")
flag.StringVar(&workPath, "work-path", "", "Set the work path of the program")
flag.StringVar(&profilePath, "p", "", "sandbox profile")
flag.BoolVar(&showDetails, "show-trace-details", false, "Show trace details")
flag.StringVar(&result, "res", "stdout", "Set the file name for output the result")
flag.Parse()
args = flag.Args()
@ -53,8 +55,53 @@ func main() {
workPath, _ = os.Getwd()
}
ret, err := start()
log.Println(ret, err)
var (
f *os.File
err error
)
if result == "stdout" {
f = os.Stdout
} else if result == "stderr" {
f = os.Stderr
} else {
f, err = os.Create(result)
if err != nil {
debug("Failed to open result file:", err)
return
}
defer f.Close()
}
rt, err := start()
debug(rt, err)
if e, ok := err.(syscall.Errno); ok {
debug("errno", int(e))
}
if rt == nil {
rt = &runner.Result{
Status: runner.StatusRunnerError,
}
}
if err == nil && rt.Status != runner.StatusNormal {
err = rt.Status
}
debug("setupTime: ", rt.SetUpTime)
debug("runningTime: ", rt.RunningTime)
if err != nil {
debug(err)
c, ok := err.(runner.Status)
if !ok {
c = runner.StatusRunnerError
}
// Handle fatal error from trace
fmt.Fprintf(f, "%d %d %d %d\n", getStatus(c), int(rt.Time/time.Millisecond), uint64(rt.Memory)>>10, rt.ExitStatus)
if c == runner.StatusRunnerError {
os.Exit(1)
}
} else {
fmt.Fprintf(f, "%d %d %d %d\n", 0, int(rt.Time/time.Millisecond), uint64(rt.Memory)>>10, rt.ExitStatus)
}
}
func start() (*runner.Result, error) {
@ -68,9 +115,9 @@ func start() (*runner.Result, error) {
var profile string
if profilePath != "" {
c, err := ioutil.ReadFile(profilePath)
c, err := os.ReadFile(profilePath)
if err != nil {
return nil, fmt.Errorf("profile: %v", err)
return nil, fmt.Errorf("profile: %w", err)
}
profile = string(c)
}
@ -94,8 +141,8 @@ func start() (*runner.Result, error) {
Stack: stackLimit << 20,
}
log.Println(rlims)
log.Println(args)
debug(rlims)
debug(args)
r := forkexec.Runner{
Args: args,
@ -113,13 +160,20 @@ func start() (*runner.Result, error) {
if err != nil {
return nil, err
}
defer func() {
killAll(pid)
collectZombie(pid)
}()
var (
wstatus syscall.WaitStatus
rusage syscall.Rusage
)
for {
_, err = syscall.Wait4(pid, &wstatus, 0, &rusage)
for err == syscall.EINTR {
_, err = syscall.Wait4(pid, &wstatus, 0, &rusage)
if err == syscall.EINTR {
continue
}
fTime = time.Now()
if err != nil {
@ -128,25 +182,10 @@ func start() (*runner.Result, error) {
result := runner.Result{
Status: runner.StatusNormal,
Time: time.Duration(rusage.Utime.Nano()),
Memory: runner.Size(rusage.Maxrss),
Memory: runner.Size(rusage.Maxrss), // seems MacOS uses bytes instead of kb
SetUpTime: mTime.Sub(sTime),
RunningTime: fTime.Sub(mTime),
}
switch {
case wstatus.Exited():
if status := wstatus.ExitStatus(); status != 0 {
result.Status = runner.StatusNonzeroExitStatus
return &result, nil
}
case wstatus.Signaled():
result.Status = runner.StatusSignalled
result.ExitStatus = int(wstatus.Signal())
return &result, nil
default:
}
if uint64(result.Time) > timeLimit*1e9 {
result.Status = runner.StatusTimeLimitExceeded
}
@ -154,5 +193,42 @@ func start() (*runner.Result, error) {
result.Status = runner.StatusMemoryLimitExceeded
}
switch {
case wstatus.Exited():
if status := wstatus.ExitStatus(); status != 0 {
result.Status = runner.StatusNonzeroExitStatus
}
return &result, nil
case wstatus.Signaled():
sig := wstatus.Signal()
switch sig {
case unix.SIGXCPU, unix.SIGKILL:
result.Status = runner.StatusTimeLimitExceeded
case unix.SIGXFSZ:
result.Status = runner.StatusOutputLimitExceeded
case unix.SIGSYS:
result.Status = runner.StatusDisallowedSyscall
default:
result.Status = runner.StatusSignalled
}
result.ExitStatus = int(sig)
return &result, nil
}
}
}
// kill all tracee according to pids
func killAll(pgid int) {
unix.Kill(-pgid, unix.SIGKILL)
}
// collect died child processes
func collectZombie(pgid int) {
var wstatus unix.WaitStatus
for {
if _, err := unix.Wait4(-pgid, &wstatus, unix.WNOHANG, nil); err != unix.EINTR && err != nil {
break
}
}
}

View File

@ -3,16 +3,17 @@ package main
import (
"context"
"errors"
"flag"
"fmt"
"io/ioutil"
"io"
"os"
"os/signal"
"sync/atomic"
"syscall"
"time"
"github.com/criyle/go-sandbox/config"
"github.com/criyle/go-sandbox/cmd/runprog/config"
"github.com/criyle/go-sandbox/container"
"github.com/criyle/go-sandbox/pkg/cgroup"
"github.com/criyle/go-sandbox/pkg/forkexec"
@ -25,14 +26,16 @@ import (
"github.com/criyle/go-sandbox/runner/ptrace"
"github.com/criyle/go-sandbox/runner/ptrace/filehandler"
"github.com/criyle/go-sandbox/runner/unshare"
"golang.org/x/sys/unix"
)
var (
addReadable, addWritable, addRawReadable, addRawWritable arrayFlags
allowProc, unsafe, showDetails, useCGroup, memfile, cred bool
allowProc, unsafe, showDetails, useCGroup, memfile, cred, nucg bool
timeLimit, realTimeLimit, memoryLimit, outputLimit, stackLimit uint64
inputFileName, outputFileName, errorFileName, workPath, runt string
useCGroupFd bool
pType, result string
args []string
)
@ -63,9 +66,11 @@ func main() {
flag.Var(&addRawReadable, "add-readable-raw", "Add a readable file (don't transform to its real path)")
flag.Var(&addRawWritable, "add-writable-raw", "Add a writable file (don't transform to its real path)")
flag.BoolVar(&useCGroup, "cgroup", false, "Use cgroup to colloct resource usage")
flag.BoolVar(&useCGroupFd, "cgroupfd", false, "Use cgroup FD to clone3 (cgroup v2 & kernel > 5.7)")
flag.BoolVar(&memfile, "memfd", false, "Use memfd as exec file")
flag.StringVar(&runt, "runner", "ptrace", "Runner for the program (ptrace, ns, container)")
flag.BoolVar(&cred, "cred", false, "Generate credential for containers (uid=10000)")
flag.BoolVar(&nucg, "nucg", false, "don't unshare cgroup")
flag.Parse()
args = flag.Args()
@ -118,12 +123,14 @@ func main() {
c = runner.StatusRunnerError
}
// Handle fatal error from trace
fmt.Fprintf(f, "%d %d %d %d\n", getStatus(c), int(rt.Time/time.Millisecond), uint64(rt.Memory)>>10, rt.ExitStatus)
fmt.Fprintf(f, "%d %d %d %d\n", getStatus(c),
int(rt.Time.Round(time.Millisecond)/time.Millisecond), uint64(rt.Memory)>>10, rt.ExitStatus)
if c == runner.StatusRunnerError {
os.Exit(1)
}
} else {
fmt.Fprintf(f, "%d %d %d %d\n", 0, int(rt.Time/time.Millisecond), uint64(rt.Memory)>>10, rt.ExitStatus)
fmt.Fprintf(f, "%d %d %d %d\n", 0,
int(rt.Time.Round(time.Millisecond)/time.Millisecond), uint64(rt.Memory)>>10, rt.ExitStatus)
}
}
@ -132,14 +139,16 @@ type containerRunner struct {
container.ExecveParam
}
func (r *containerRunner) Run(c context.Context) <-chan runner.Result {
func (r *containerRunner) Run(c context.Context) runner.Result {
return r.Environment.Execve(c, r.ExecveParam)
}
func start() (*runner.Result, error) {
var (
r runner.Runner
cg *cgroup.Cgroup
cg cgroup.Cgroup
cgDir *os.File
cgroupFd uintptr
err error
execFile uintptr
rt runner.Result
@ -171,46 +180,68 @@ func start() (*runner.Result, error) {
// work dir
WithTmpfs("w", "size=8m,nr_inodes=4k").
// tmp dir
WithTmpfs("tmp", "size=8m,nr_inodes=4k")
WithTmpfs("tmp", "size=8m,nr_inodes=4k").
FilterNotExist()
mt, err := mb.Build(true)
mt, err := mb.FilterNotExist().Build()
if err != nil {
return nil, err
}
if useCGroup {
b, err := cgroup.NewBuilder("runprog").WithCPUAcct().WithMemory().WithPids().FilterByEnv()
t := cgroup.DetectType()
if t == cgroup.TypeV2 {
cgroup.EnableV2Nesting()
}
ct, err := cgroup.GetAvailableController()
if err != nil {
return nil, err
}
b, err := cgroup.New("runprog", ct)
if err != nil {
return nil, err
}
debug(b)
cg, err = b.Build()
cg, err = b.Random("runprog")
if err != nil {
return nil, err
}
defer cg.Destroy()
if err = cg.SetMemoryLimitInBytes(memoryLimit << 20); err != nil {
if err = cg.SetMemoryLimit(memoryLimit << 20); err != nil {
return nil, err
}
debug("cgroup:", cg)
if useCGroupFd {
debug("use cgroup fd")
if t != cgroup.TypeV2 {
return nil, fmt.Errorf("use cgroup fd cannot be enabled without cgroup v2")
}
if cgDir, err = cg.Open(); err != nil {
return nil, err
}
defer cgDir.Close()
cgroupFd = cgDir.Fd()
}
}
syncFunc := func(pid int) error {
var syncFunc func(pid int) error
if cg != nil {
syncFunc = func(pid int) error {
if err := cg.AddProc(pid); err != nil {
return err
}
}
return nil
}
}
if memfile {
fin, err := os.Open(args[0])
if err != nil {
return nil, fmt.Errorf("filed to open args[0]: %v", err)
return nil, fmt.Errorf("failed to open args[0]: %w", err)
}
execf, err := memfd.DupToMemfd("run_program", fin)
if err != nil {
return nil, fmt.Errorf("dup to memfd failed: %v", err)
return nil, fmt.Errorf("dup to memfd failed: %w", err)
}
fin.Close()
defer execf.Close()
@ -221,7 +252,7 @@ func start() (*runner.Result, error) {
// open input / output / err files
files, err := prepareFiles(inputFileName, outputFileName, errorFileName)
if err != nil {
return nil, fmt.Errorf("failed to prepare files: %v", err)
return nil, fmt.Errorf("failed to prepare files: %w", err)
}
defer closeFiles(files)
@ -240,12 +271,32 @@ func start() (*runner.Result, error) {
CPUHard: realTimeLimit,
FileSize: outputLimit << 20,
Stack: stackLimit << 20,
Data: memoryLimit << 20,
OpenFile: 256,
DisableCore: true,
}
debug("rlimit: ", rlims)
actionDefault := seccomp.ActionKill
actionDefault := libseccomp.ActionKill
if showDetails {
actionDefault = seccomp.ActionTrace.WithReturnCode(seccomp.MsgDisallow)
actionDefault = libseccomp.ActionTrace
}
if runt != "ptrace" {
allow = append(allow, trace...)
trace = nil
}
builder := libseccomp.Builder{
Allow: allow,
Trace: trace,
Default: actionDefault,
}
// do not build filter for container unsafe since seccomp is not compatible with aarch64 syscalls
var filter seccomp.Filter
if !unsafe || runt != "container" {
filter, err = builder.Build()
if err != nil {
return nil, fmt.Errorf("failed to create seccomp filter: %w", err)
}
}
limit := runner.Limit{
@ -254,32 +305,39 @@ func start() (*runner.Result, error) {
}
if runt == "container" {
root, err := ioutil.TempDir("", "dm")
if err != nil {
return nil, fmt.Errorf("cannot make temp root for container namespace: %v", err)
}
defer os.RemoveAll(root)
var credG container.CredGenerator
if cred {
credG = newCredGen()
}
var stderr io.Writer
if showDetails {
stderr = os.Stderr
}
cloneFlag := forkexec.UnshareFlags
if nucg {
cloneFlag &= ^unix.CLONE_NEWCGROUP
}
b := container.Builder{
Root: root,
Mounts: mt,
TmpRoot: "dm",
Mounts: mb.Mounts,
Stderr: stderr,
CredGenerator: credG,
CloneFlags: forkexec.UnshareFlags,
CloneFlags: uintptr(cloneFlag),
}
m, err := b.Build()
if err != nil {
return nil, fmt.Errorf("failed to new container: %v", err)
return nil, fmt.Errorf("failed to new container: %w", err)
}
defer m.Destroy()
err = m.Ping()
if err != nil {
return nil, fmt.Errorf("failed to ping container: %v", err)
return nil, fmt.Errorf("failed to ping container: %w", err)
}
if unsafe {
filter = nil
}
r = &containerRunner{
Environment: m,
@ -289,19 +347,14 @@ func start() (*runner.Result, error) {
Files: fds,
ExecFile: execFile,
RLimits: rlims.PrepareRLimit(),
Seccomp: filter,
SyncFunc: syncFunc,
CgroupFD: cgroupFd,
SyncAfterExec: cg == nil || cgDir != nil,
},
}
} else if runt == "ns" {
builder := libseccomp.Builder{
Allow: append(allow, trace...),
Default: actionDefault,
}
filter, err := builder.Build()
if err != nil {
return nil, fmt.Errorf("cannot build seccomp filter %v", err)
}
root, err := ioutil.TempDir("", "ns")
root, err := os.MkdirTemp("", "ns")
if err != nil {
return nil, fmt.Errorf("cannot make temp root for new namespace")
}
@ -323,15 +376,6 @@ func start() (*runner.Result, error) {
DomainName: "run_program",
}
} else if runt == "ptrace" {
builder := libseccomp.Builder{
Allow: allow,
Trace: trace,
Default: actionDefault,
}
filter, err := builder.Build()
if err != nil {
return nil, fmt.Errorf("failed to create seccomp filter %v", err)
}
r = &ptrace.Runner{
Args: args,
Env: []string{pathEnv},
@ -359,7 +403,10 @@ func start() (*runner.Result, error) {
c, cancel := context.WithTimeout(context.Background(), time.Duration(int64(realTimeLimit)*int64(time.Second)))
defer cancel()
s := r.Run(c)
s := make(chan runner.Result, 1)
go func() {
s <- r.Run(c)
}()
rTime := time.Now()
select {
@ -380,68 +427,45 @@ func start() (*runner.Result, error) {
debug("results:", rt, err)
if useCGroup {
cpu, err := cg.CpuacctUsage()
cpu, err := cg.CPUUsage()
if err != nil {
return nil, fmt.Errorf("cgroup cpu: %v", err)
}
memory, err := cg.MemoryMaxUsageInBytes()
if err != nil {
return nil, fmt.Errorf("cgroup memory: %v", err)
}
cache, err := cg.FindMemoryStatProperty("cache")
if err != nil {
return nil, fmt.Errorf("cgroup cache %v", err)
}
debug("cgroup: cpu: ", cpu, " memory: ", memory, "cache: ", cache)
} else {
rt.Time = time.Duration(cpu)
rt.Memory = runner.Size(memory - cache)
}
// max memory usage may not exist in cgroup v2
memory, err := cg.MemoryMaxUsage()
if err != nil && !errors.Is(err, os.ErrNotExist) {
return nil, fmt.Errorf("cgroup memory: %v", err)
} else if err == nil {
rt.Memory = runner.Size(memory)
}
procPeak, err := cg.ProcessPeak()
if err != nil && !errors.Is(err, os.ErrNotExist) {
return nil, fmt.Errorf("cgroup pid: %v", err)
} else if err == nil {
rt.ProcPeak = procPeak
}
debug("cgroup: cpu: ", cpu, " memory: ", memory, " procPeak: ", procPeak)
debug("cgroup:", rt)
}
if rt.Status == runner.StatusTimeLimitExceeded || rt.Status == runner.StatusNormal {
if rt.Time > limit.TimeLimit {
rt.Status = runner.StatusTimeLimitExceeded
} else {
rt.Status = runner.StatusNormal
}
}
if rt.Status == runner.StatusMemoryLimitExceeded || rt.Status == runner.StatusNormal {
if rt.Memory > limit.MemoryLimit {
rt.Status = runner.StatusMemoryLimitExceeded
} else {
rt.Status = runner.StatusNormal
}
}
return &rt, nil
}
func debug(v ...interface{}) {
if showDetails {
fmt.Fprintln(os.Stderr, v...)
}
}
// Status defines uoj/run_program constants
type Status int
// UOJ run_program constants
const (
StatusNormal Status = iota // 0
StatusInvalid // 1
StatusRE // 2
StatusMLE // 3
StatusTLE // 4
StatusOLE // 5
StatusBan // 6
StatusFatal // 7
)
func getStatus(s runner.Status) int {
switch s {
case runner.StatusNormal:
return int(StatusNormal)
case runner.StatusInvalid:
return int(StatusInvalid)
case runner.StatusTimeLimitExceeded:
return int(StatusTLE)
case runner.StatusMemoryLimitExceeded:
return int(StatusMLE)
case runner.StatusOutputLimitExceeded:
return int(StatusOLE)
case runner.StatusDisallowedSyscall:
return int(StatusBan)
case runner.StatusSignalled, runner.StatusNonzeroExitStatus:
return int(StatusRE)
default:
return int(StatusFatal)
}
}
type credGen struct {
cur uint32
}

View File

@ -1,14 +0,0 @@
package config
// This file includes configs for the run program settings
var (
archReadableFiles = []string{
"/lib/x86_64-linux-gnu/",
"/usr/lib/x86_64-linux-gnu/",
}
archSyscallAllows = []string{}
archSyscallTraces = []string{}
)

View File

@ -3,7 +3,9 @@ package container
import (
"context"
"errors"
"io/ioutil"
"os"
"runtime"
"syscall"
"testing"
"github.com/criyle/go-sandbox/runner"
@ -14,102 +16,172 @@ func init() {
}
func BenchmarkContainer(b *testing.B) {
tmpDir, err := ioutil.TempDir("", "")
tmpDir, err := os.MkdirTemp("", "")
if err != nil {
b.Error(err)
return
}
builder := &Builder{
Root: tmpDir,
Stderr: os.Stderr,
}
n := runtime.GOMAXPROCS(0)
ch := make(chan Environment, n)
for i := 0; i < n; i++ {
m, err := builder.Build()
if err != nil {
b.Error(err)
return
}
b.Cleanup(func() {
m.Destroy()
})
ch <- m
}
b.ResetTimer()
for i := 0; i < b.N; i++ {
rt := m.Execve(context.TODO(), ExecveParam{
Args: []string{"/bin/echo"},
b.RunParallel(func(pb *testing.PB) {
m := <-ch
for pb.Next() {
r := m.Execve(context.TODO(), ExecveParam{
Args: []string{"/bin/true"},
Env: []string{"PATH=/bin"},
})
r := <-rt
if r.Status != runner.StatusNormal {
b.Error(r.Status, r.Error)
return
}
}
}
func TestContainerSuccess(t *testing.T) {
m := getEnv(t)
if m == nil {
return
}
rt := m.Execve(context.TODO(), ExecveParam{
Args: []string{"/bin/echo"},
Env: []string{"PATH=/bin"},
})
r := <-rt
if r.Status != runner.StatusNormal {
t.Error(r.Status, r.Error)
return
}
}
func TestContainerNotExists(t *testing.T) {
m := getEnv(t)
if m == nil {
return
type testCase struct {
name string
param ExecveParam
expected runner.Status
}
rt := m.Execve(context.TODO(), ExecveParam{
var err error = errors.New("test error")
var successParam = ExecveParam{
Args: []string{"/bin/true"},
Env: []string{"PATH=/bin"},
}
var tests []testCase = []testCase{
{
name: "Success",
param: successParam,
expected: runner.StatusNormal,
},
{
name: "SuccessWithSync",
param: ExecveParam{
Args: []string{"/bin/true"},
Env: []string{"PATH=/bin"},
SyncFunc: func(p int) error { return nil },
},
expected: runner.StatusNormal,
},
{
name: "NotExists",
param: ExecveParam{
Args: []string{"not_exists"},
Env: []string{"PATH=/bin"},
})
r := <-rt
if r.Status != runner.StatusRunnerError {
t.Error(r.Status, r.Error)
return
}
}
func TestContainerSyncFuncFail(t *testing.T) {
m := getEnv(t)
if m == nil {
return
}
err := errors.New("test error")
rt := m.Execve(context.TODO(), ExecveParam{
Args: []string{"/bin/echo"},
},
expected: runner.StatusRunnerError,
},
{
name: "NotExistsWithSync",
param: ExecveParam{
Args: []string{"not_exists"},
Env: []string{"PATH=/bin"},
SyncFunc: func(p int) error { return nil },
},
expected: runner.StatusRunnerError,
},
{
name: "SyncFuncFail",
param: ExecveParam{
Args: []string{"/bin/true"},
Env: []string{"PATH=/bin"},
SyncFunc: func(pid int) error {
return err
},
})
r := <-rt
if r.Status != runner.StatusRunnerError {
t.Error(r.Status, r.Error)
return
},
expected: runner.StatusRunnerError,
},
{
name: "SyncFuncFailAfterExec",
param: ExecveParam{
Args: []string{"/bin/true"},
Env: []string{"PATH=/bin"},
SyncFunc: func(pid int) error {
return err
},
SyncAfterExec: true,
},
expected: runner.StatusRunnerError,
},
}
type credgen struct{}
func (c credgen) Get() syscall.Credential {
return syscall.Credential{
Uid: 10000,
Gid: 10000,
}
}
func getEnv(t *testing.T) Environment {
tmpDir, err := ioutil.TempDir("", "")
if err != nil {
t.Error(err)
return nil
func TestContainerSetCred(t *testing.T) {
t.Parallel()
if os.Getpid() != 1 {
t.Skip("root required for this test")
}
runTest(t, successParam, runner.StatusNormal, credgen{})
}
func runTest(t *testing.T, param ExecveParam, expected runner.Status, credGen CredGenerator) {
t.Parallel()
m := getEnv(t, credGen)
r := m.Execve(context.TODO(), param)
if r.Status != expected {
t.Fatal(r.Status, r.Error, r)
}
if err := m.Ping(); err != nil {
t.Fatal(err)
}
// can also success once more (no protocol mismatch)
r = m.Execve(context.TODO(), successParam)
if r.Status != runner.StatusNormal {
t.Fatal(r.Status, r.Error, r)
}
}
func TestCases(t *testing.T) {
for _, c := range tests {
t.Run(c.name, func(t *testing.T) {
runTest(t, c.param, c.expected, nil)
})
}
}
func getEnv(t *testing.T, credGen CredGenerator) Environment {
tmpDir, err := os.MkdirTemp("", "")
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() {
os.Remove(tmpDir)
})
builder := &Builder{
Root: tmpDir,
CredGenerator: credGen,
Stderr: os.Stderr,
}
m, err := builder.Build()
if err != nil {
t.Error(err)
return nil
t.Fatal(err)
}
t.Cleanup(func() {
m.Destroy()
})
return m
}

View File

@ -1,25 +0,0 @@
package container
const (
cmdPing = "ping"
cmdCopyIn = "copyin"
cmdOpen = "open"
cmdDelete = "delete"
cmdReset = "reset"
cmdExecve = "execve"
cmdOk = "ok"
cmdKill = "kill"
cmdConf = "conf"
initArg = "init"
currentExec = "/proc/self/exe"
containerUID = 1000
containerGID = 1000
containerName = "go-sandbox"
containerWD = "/w"
containerMaxProc = 1
)

47
container/consts_linux.go Normal file
View File

@ -0,0 +1,47 @@
package container
type cmdType int8
const (
cmdPing cmdType = iota + 1
cmdOpen
cmdDelete
cmdReset
cmdExecve
cmdOk
cmdKill
cmdConf
initArg = "container_init"
containerUID = 1000
containerGID = 1000
containerName = "go-sandbox"
containerWD = "/w"
containerMaxProc = 1
)
var defaultSymLinks = []SymbolicLink{
{LinkPath: "/dev/fd", Target: "/proc/self/fd"},
{LinkPath: "/dev/stdin", Target: "/proc/self/fd/0"},
{LinkPath: "/dev/stdout", Target: "/proc/self/fd/1"},
{LinkPath: "/dev/stderr", Target: "/proc/self/fd/2"},
}
var defaultMaskPaths = []string{
// https://github.com/containerd/containerd/blob/f0a32c66dad1e9de716c9960af806105d691cd78/oci/spec.go#L165-L176
"/proc/acpi",
"/proc/asound",
"/proc/kcore",
"/proc/keys",
"/proc/latency_stats",
"/proc/timer_list",
"/proc/timer_stats",
"/proc/sched_debug",
"/sys/firmware",
"/proc/scsi",
"/usr/lib/wsl",
}

View File

@ -1,22 +1,39 @@
package container
import (
"bufio"
"errors"
"fmt"
"os"
"syscall"
"path/filepath"
"strings"
"github.com/criyle/go-sandbox/pkg/unixsocket"
)
func (c *containerServer) handlePing() error {
return c.sendReply(&reply{}, nil)
return c.sendReply(reply{}, unixsocket.Msg{})
}
func (c *containerServer) handleConf(conf *confCmd) error {
if conf != nil {
c.containerConfig = conf.Conf
if err := initContainer(conf.Conf); err != nil {
return err
}
return c.sendReply(&reply{}, nil)
if c.ContainerUID == 0 {
c.ContainerUID = containerUID
}
if c.ContainerGID == 0 {
c.ContainerGID = containerGID
}
env, err := readDotEnv()
if err != nil {
return err
}
c.defaultEnv = env
}
return c.sendReply(reply{}, unixsocket.Msg{})
}
func (c *containerServer) handleOpen(open []OpenCmd) error {
@ -26,16 +43,20 @@ func (c *containerServer) handleOpen(open []OpenCmd) error {
// open files
fds := make([]int, 0, len(open))
fileToClose := make([]*os.File, 0, len(open)) // let sendMsg close these files
for _, o := range open {
outFile, err := os.OpenFile(o.Path, o.Flag, o.Perm)
if err != nil {
for _, f := range fileToClose {
f.Close()
}
return c.sendErrorReply("open: %v", err)
}
defer outFile.Close()
fileToClose = append(fileToClose, outFile)
fds = append(fds, int(outFile.Fd()))
}
return c.sendReply(&reply{}, &unixsocket.Msg{Fds: fds})
return c.sendReplyFiles(reply{}, unixsocket.Msg{Fds: fds}, fileToClose)
}
func (c *containerServer) handleDelete(delete *deleteCmd) error {
@ -45,42 +66,43 @@ func (c *containerServer) handleDelete(delete *deleteCmd) error {
if err := os.Remove(delete.Path); err != nil {
return c.sendErrorReply("delete: %v", err)
}
return c.sendReply(&reply{}, nil)
return c.sendReply(reply{}, unixsocket.Msg{})
}
func (c *containerServer) handleReset() error {
if err := removeContents("/tmp"); err != nil {
return c.sendErrorReply("reset: /tmp %v", err)
for _, m := range c.Mounts {
if !m.IsTmpFs() {
continue
}
if err := removeContents("/w"); err != nil {
return c.sendErrorReply("reset: /w %v", err)
if err := removeContents(filepath.Join("/", m.Target)); err != nil {
return c.sendErrorReply("reset: %v %v", m.Target, err)
}
return c.sendReply(&reply{}, nil)
}
return c.sendReply(reply{}, unixsocket.Msg{})
}
func (c *containerServer) recvCmd() (*cmd, *unixsocket.Msg, error) {
cm := new(cmd)
msg, err := c.socket.RecvMsg(cm)
// readDotEnv attempts to read /.env file and save as default environment variables
func readDotEnv() ([]string, error) {
f, err := os.Open("/.env")
if err != nil {
return nil, nil, err
if errors.Is(err, os.ErrNotExist) {
return nil, nil
}
return cm, msg, nil
return nil, fmt.Errorf("dotenv: open /.env: %w", err)
}
defer f.Close()
func (c *containerServer) sendReply(rep *reply, msg *unixsocket.Msg) error {
return c.socket.SendMsg(rep, msg)
var ret []string
scanner := bufio.NewScanner(f)
for scanner.Scan() {
line := strings.TrimSpace(scanner.Text())
if len(line) == 0 || strings.HasPrefix(line, "#") {
continue
}
// sendErrorReply sends error reply
func (c *containerServer) sendErrorReply(ft string, v ...interface{}) error {
errRep := &errorReply{
Msg: fmt.Sprintf(ft, v...),
if !strings.Contains(line, "=") {
return nil, fmt.Errorf("dotenv: invalid line: %s", line)
}
// store errno
if len(v) == 1 {
if errno, ok := v[0].(syscall.Errno); ok {
errRep.Errno = &errno
ret = append(ret, line)
}
}
return c.sendReply(&reply{Error: errRep}, nil)
return ret, nil
}

View File

@ -10,16 +10,17 @@ import (
"github.com/criyle/go-sandbox/runner"
)
func (c *containerServer) handleExecve(cmd *execCmd, msg *unixsocket.Msg) error {
func (c *containerServer) handleExecve(cmd *execCmd, msg unixsocket.Msg) error {
var (
files []uintptr
execFile uintptr
cgroupFd uintptr
cred *syscall.Credential
)
if cmd == nil {
return c.sendErrorReply("execve: no parameter provided")
return c.sendErrorReply("handle: no parameter provided")
}
if msg != nil {
if len(msg.Fds) > 0 {
files = intSliceToUintptr(msg.Fds)
// don't leak fds to child
closeOnExecFds(msg.Fds)
@ -30,116 +31,182 @@ func (c *containerServer) handleExecve(cmd *execCmd, msg *unixsocket.Msg) error
// if fexecve, then the first fd must be executable
if cmd.FdExec {
if len(files) == 0 {
return fmt.Errorf("execve: expected fexecve fd")
return c.sendErrorReply("handle: expected fexecve fd")
}
execFile = files[0]
files = files[1:]
}
// if cgroupFd, then the cgroupFd follows
if cmd.FdCgroup {
if len(files) == 0 {
return c.sendErrorReply("handle: expected cgroup fd")
}
cgroupFd = files[0]
files = files[1:]
}
syncFunc := func(pid int) error {
msg := &unixsocket.Msg{
var env []string
env = append(env, c.defaultEnv...)
env = append(env, cmd.Env...)
if len(cmd.Argv) > 0 {
exePath, err := lookPath(cmd.Argv[0], env)
if err != nil {
return c.sendErrorReply("handle: %s: %v", cmd.Argv[0], err)
}
cmd.Argv[0] = exePath
}
syncPid := func(pid int) error {
msg := unixsocket.Msg{
Cred: &syscall.Ucred{
Pid: int32(pid),
Uid: uint32(syscall.Getuid()),
Gid: uint32(syscall.Getgid()),
},
}
if err := c.sendReply(&reply{}, msg); err != nil {
return fmt.Errorf("syncFunc: sendReply %v", err)
if err := c.sendReply(reply{}, msg); err != nil {
return fmt.Errorf("sync func: send reply: %w", err)
}
cmd, _, err := c.recvCmd()
if err != nil {
return fmt.Errorf("syncFunc: recvCmd %v", err)
return fmt.Errorf("sync func: recv cmd: %w", err)
}
if cmd.Cmd == cmdKill {
return fmt.Errorf("syncFunc: received kill")
return fmt.Errorf("sync func: received kill")
}
return nil
}
var syncFunc func(pid int) error
if !cmd.SyncAfter {
syncFunc = syncPid
}
if c.Cred {
cred = &syscall.Credential{
Uid: containerUID,
Gid: containerGID,
Uid: uint32(c.ContainerUID),
Gid: uint32(c.ContainerGID),
NoSetGroups: true,
}
}
var seccomp *syscall.SockFprog
if cmd.Seccomp != nil {
seccomp = cmd.Seccomp.SockFprog()
}
r := forkexec.Runner{
Args: cmd.Argv,
Env: cmd.Env,
Env: env,
ExecFile: execFile,
RLimits: cmd.RLimits,
Files: files,
WorkDir: "/w",
WorkDir: c.WorkDir,
NoNewPrivs: true,
DropCaps: true,
SyncFunc: syncFunc,
Credential: cred,
CTTY: cmd.CTTY,
Seccomp: seccomp,
CgroupFd: cgroupFd,
UnshareCgroupAfterSync: true,
UnshareCgroupAfterSync: c.UnshareCgroup,
}
// starts the runner, error is handled same as wait4 to make communication equal
pid, err := r.Start()
// done is to signal kill goroutine exits
killDone := make(chan struct{})
// waitDone is to signal kill goroutine to collect zombies
waitDone := make(chan struct{})
// recv kill
go func() {
// signal done
defer close(killDone)
// msg must be kill
c.recvCmd()
// kill all
syscall.Kill(-1, syscall.SIGKILL)
// make sure collect zombie does not consume the exit status
<-waitDone
// collect zombies
for {
if _, err := syscall.Wait4(-1, nil, syscall.WNOHANG, nil); err != nil && err != syscall.EINTR {
break
}
}
}()
// wait pid if no error encountered for execve
var wstatus syscall.WaitStatus
var rusage syscall.Rusage
if err == nil {
_, err = syscall.Wait4(pid, &wstatus, 0, &rusage)
for err == syscall.EINTR {
_, err = syscall.Wait4(pid, &wstatus, 0, &rusage)
}
}
// sync with kill goroutine
close(waitDone)
if err != nil {
c.sendErrorReply("execve: wait4 %v", err)
} else {
s := "<nil>"
if len(cmd.Argv) > 0 {
s = cmd.Argv[0]
}
return c.sendErrorReply("start: %s: %v", s, err)
}
if cmd.SyncAfter {
if err := syncPid(1); err != nil {
syscall.Kill(-1, syscall.SIGKILL)
c.waitPid <- pid
ret := <-c.waitPidResult
err := c.sendReply(convertReply(ret), unixsocket.Msg{})
c.waitAll <- struct{}{}
<-c.waitAllDone
return err
}
}
return c.handleExecveStarted(pid)
}
func (c *containerServer) handleExecveStarted(pid int) error {
// At this point, either recv kill / send result would be happened
// host -> container: kill
// container -> host: result
// container -> host: done
// Let's register a wait event
c.waitPid <- pid
var ret waitPidResult
select {
case <-c.done: // socket error happened
return c.err
case <-c.recvCh: // kill cmd received
syscall.Kill(-1, syscall.SIGKILL)
ret = <-c.waitPidResult
c.waitAll <- struct{}{}
if err := c.sendReply(convertReply(ret), unixsocket.Msg{}); err != nil {
return err
}
case ret = <-c.waitPidResult: // child process returned
syscall.Kill(-1, syscall.SIGKILL)
c.waitAll <- struct{}{}
if err := c.sendReply(convertReply(ret), unixsocket.Msg{}); err != nil {
return err
}
if _, _, err := c.recvCmd(); err != nil { // kill cmd received
return err
}
}
<-c.waitAllDone
return nil
}
func convertReply(ret waitPidResult) reply {
if ret.Err != nil {
return reply{
Error: &errorReply{
Msg: fmt.Sprintf("execve: wait4: %v", ret.Err),
},
}
}
waitStatus := ret.WaitStatus
rusage := ret.Rusage
status := runner.StatusNormal
userTime := time.Duration(rusage.Utime.Nano()) // ns
userMem := runner.Size(rusage.Maxrss << 10) // bytes
switch {
case wstatus.Exited():
exitStatus := wstatus.ExitStatus()
case waitStatus.Exited():
exitStatus := waitStatus.ExitStatus()
if exitStatus != 0 {
status = runner.StatusNonzeroExitStatus
}
c.sendReply(&reply{
return reply{
ExecReply: &execReply{
Status: status,
ExitStatus: exitStatus,
Time: userTime,
Memory: userMem,
},
}, nil)
}
case wstatus.Signaled():
switch wstatus.Signal() {
case waitStatus.Signaled():
switch waitStatus.Signal() {
// kill signal treats as TLE
case syscall.SIGXCPU, syscall.SIGKILL:
status = runner.StatusTimeLimitExceeded
@ -150,21 +217,20 @@ func (c *containerServer) handleExecve(cmd *execCmd, msg *unixsocket.Msg) error
default:
status = runner.StatusSignalled
}
c.sendReply(&reply{
return reply{
ExecReply: &execReply{
ExitStatus: int(wstatus.Signal()),
ExitStatus: int(waitStatus.Signal()),
Status: status,
Time: userTime,
Memory: userMem,
},
}, nil)
}
default:
c.sendErrorReply("execve: unknown status %v", wstatus)
return reply{
Error: &errorReply{
Msg: fmt.Sprintf("execve: unknown status: %v", waitStatus),
},
}
}
// wait for kill msg and reply done for finish
<-killDone
return c.sendReply(&reply{}, nil)
}

View File

@ -1,9 +1,16 @@
package container
import (
"errors"
"fmt"
"os"
"os/exec"
"os/signal"
"path/filepath"
"runtime"
"strconv"
"sync"
"syscall"
"github.com/criyle/go-sandbox/pkg/unixsocket"
)
@ -11,6 +18,37 @@ import (
type containerServer struct {
socket *socket
containerConfig
defaultEnv []string
done chan struct{}
err error
doneOnce sync.Once
recvCh chan recvCmd
sendCh chan sendReply
waitPid chan int
waitPidResult chan waitPidResult
waitAll chan struct{}
waitAllDone chan struct{}
}
type recvCmd struct {
Cmd cmd
Msg unixsocket.Msg
}
type sendReply struct {
Reply reply
Msg unixsocket.Msg
FileToClose []*os.File
}
type waitPidResult struct {
WaitStatus syscall.WaitStatus
Rusage syscall.Rusage
Err error
}
// Init is called for container init process
@ -20,7 +58,7 @@ type containerServer struct {
func Init() (err error) {
// noop if self is not container init process
// Notice: docker init is also 1, additional check for args[1] == init
if os.Getpid() != 1 || len(os.Args) != 2 || os.Args[1] != initArg {
if os.Getpid() != 1 || len(os.Args) < 2 || os.Args[1] != initArg {
return nil
}
@ -42,34 +80,132 @@ func Init() (err error) {
os.Exit(0)
}()
// ignore any signal that kills the init process
ignoreSignals()
// limit container resource usage
runtime.GOMAXPROCS(containerMaxProc)
// ensure there's no fd leak to child process (e.g. VSCode leaks ptmx fd)
if err := closeOnExecAllFds(); err != nil {
return fmt.Errorf("container_init: failed to close on exec all fds: %w", err)
}
// new_container environment shared the socket at fd 3 (marked close_exec)
const defaultFd = 3
soc, err := unixsocket.NewSocket(defaultFd)
if err != nil {
return fmt.Errorf("container_init: failed to new socket %v", err)
return fmt.Errorf("container_init: failed to create new socket: %w", err)
}
// serve forever
cs := &containerServer{socket: newSocket(soc)}
cs := &containerServer{
socket: newSocket(soc),
done: make(chan struct{}),
sendCh: make(chan sendReply, 1),
recvCh: make(chan recvCmd, 1),
waitPid: make(chan int),
waitAll: make(chan struct{}),
waitPidResult: make(chan waitPidResult, 1),
waitAllDone: make(chan struct{}, 1),
}
go cs.sendLoop()
go cs.recvLoop()
go cs.waitLoop()
return cs.serve()
}
func (c *containerServer) sendLoop() {
for {
select {
case <-c.done:
return
case rep, ok := <-c.sendCh:
if !ok {
return
}
err := c.socket.SendMsg(rep.Reply, rep.Msg)
for _, f := range rep.FileToClose {
f.Close()
}
if err != nil {
c.socketError(err)
return
}
}
}
}
func (c *containerServer) recvLoop() {
for {
var cmd cmd
msg, err := c.socket.RecvMsg(&cmd)
if err != nil {
c.socketError(err)
return
}
c.recvCh <- recvCmd{
Cmd: cmd,
Msg: msg,
}
}
}
func (c *containerServer) socketError(err error) {
c.doneOnce.Do(func() {
c.err = err
close(c.done)
})
}
func (c *containerServer) waitLoop() {
for {
select {
case pid := <-c.waitPid:
var waitStatus syscall.WaitStatus
var rusage syscall.Rusage
_, err := syscall.Wait4(pid, &waitStatus, 0, &rusage)
for err == syscall.EINTR {
_, err = syscall.Wait4(pid, &waitStatus, 0, &rusage)
}
if err != nil {
c.waitPidResult <- waitPidResult{
Err: err,
}
continue
}
c.waitPidResult <- waitPidResult{
WaitStatus: waitStatus,
Rusage: rusage,
}
case <-c.waitAll:
for {
if _, err := syscall.Wait4(-1, nil, syscall.WNOHANG, nil); err != nil && err != syscall.EINTR {
break
}
}
c.waitAllDone <- struct{}{}
}
}
}
func (c *containerServer) serve() error {
for {
cmd, msg, err := c.recvCmd()
if err != nil {
return fmt.Errorf("serve: recvCmd %v", err)
return fmt.Errorf("serve: recvCmd: %w", err)
}
if err := c.handleCmd(cmd, msg); err != nil {
return fmt.Errorf("serve: failed to execute cmd %v", err)
return fmt.Errorf("serve: failed to execute cmd: %w", err)
}
}
}
func (c *containerServer) handleCmd(cmd *cmd, msg *unixsocket.Msg) error {
func (c *containerServer) handleCmd(cmd cmd, msg unixsocket.Msg) error {
switch cmd.Cmd {
case cmdPing:
return c.handlePing()
@ -89,5 +225,154 @@ func (c *containerServer) handleCmd(cmd *cmd, msg *unixsocket.Msg) error {
case cmdExecve:
return c.handleExecve(cmd.ExecCmd, msg)
}
return fmt.Errorf("unknown command: %s", cmd.Cmd)
return fmt.Errorf("unknown command: %v", cmd.Cmd)
}
func initContainer(c containerConfig) error {
if err := initFileSystem(c); err != nil {
return err
}
if err := syscall.Setdomainname([]byte(c.DomainName)); err != nil {
return err
}
if err := syscall.Sethostname([]byte(c.HostName)); err != nil {
return err
}
if err := os.Chdir(c.WorkDir); err != nil {
return err
}
if len(c.InitCommand) > 0 {
cm := exec.Command(c.InitCommand[0], c.InitCommand[1:]...)
if output, err := cm.CombinedOutput(); err != nil {
os.Stderr.Write(output)
return err
}
}
return nil
}
func initFileSystem(c containerConfig) error {
// mount tmpfs as root
const tmpfs = "tmpfs"
if err := syscall.Mount(tmpfs, c.ContainerRoot, tmpfs, 0, ""); err != nil {
return fmt.Errorf("init_fs: mount /: %w", err)
}
// change dir to container root
if err := syscall.Chdir(c.ContainerRoot); err != nil {
return fmt.Errorf("init_fs: chdir: %w", err)
}
// performing mounts
for _, m := range c.Mounts {
if err := m.Mount(); err != nil {
return fmt.Errorf("init_fs: mount %v: %w", m, err)
}
}
// pivot root
const oldRoot = "old_root"
if err := os.Mkdir(oldRoot, 0755); err != nil {
return fmt.Errorf("init_fs: mkdir old_root: %w", err)
}
if err := syscall.PivotRoot(c.ContainerRoot, oldRoot); err != nil {
return fmt.Errorf("init_fs: pivot_root(%s, %s): %w", c.ContainerRoot, oldRoot, err)
}
if err := syscall.Unmount(oldRoot, syscall.MNT_DETACH); err != nil {
return fmt.Errorf("init_fs: unmount old_root: %w", err)
}
if err := os.Remove(oldRoot); err != nil {
return fmt.Errorf("init_fs: unlink old_root: %w", err)
}
// create symlinks
for _, l := range c.SymbolicLinks {
// ensure dir exists
dir := filepath.Dir(l.LinkPath)
if err := os.MkdirAll(dir, 0755); err != nil {
return fmt.Errorf("init_fs: mkdir_all(%s): %w", dir, err)
}
if err := os.Symlink(l.Target, l.LinkPath); err != nil {
return fmt.Errorf("init_fs: symlink: %w", err)
}
}
// mask paths
for _, p := range c.MaskPaths {
if err := maskPath(p); err != nil {
return fmt.Errorf("init_fs: mask path: %w", err)
}
}
// readonly root
const remountFlag = syscall.MS_BIND | syscall.MS_REMOUNT | syscall.MS_RDONLY | syscall.MS_NOATIME | syscall.MS_NOSUID
if err := syscall.Mount(tmpfs, "/", tmpfs, remountFlag, ""); err != nil {
return fmt.Errorf("init_fs: readonly remount /: %w", err)
}
return nil
}
func (c *containerServer) recvCmd() (cmd, unixsocket.Msg, error) {
select {
case <-c.done:
return cmd{}, unixsocket.Msg{}, c.err
case recv := <-c.recvCh:
return recv.Cmd, recv.Msg, nil
}
}
func (c *containerServer) sendReplyFiles(rep reply, msg unixsocket.Msg, fileToClose []*os.File) error {
select {
case <-c.done:
return c.err
case c.sendCh <- sendReply{Reply: rep, Msg: msg, FileToClose: fileToClose}:
return nil
}
}
func (c *containerServer) sendReply(rep reply, msg unixsocket.Msg) error {
return c.sendReplyFiles(rep, msg, nil)
}
// sendErrorReply sends error reply
func (c *containerServer) sendErrorReply(ft string, v ...interface{}) error {
errRep := &errorReply{
Msg: fmt.Sprintf(ft, v...),
}
// store errno
if len(v) == 1 {
if errno, ok := v[0].(syscall.Errno); ok {
errRep.Errno = &errno
}
}
return c.sendReply(reply{Error: errRep}, unixsocket.Msg{})
}
func closeOnExecAllFds() error {
// get all fd from /proc/self/fd
const fdPath = "/proc/self/fd"
fds, err := os.ReadDir(fdPath)
if err != nil {
return err
}
for _, f := range fds {
fd, err := strconv.Atoi(f.Name())
if err != nil {
return err
}
syscall.CloseOnExec(fd)
}
return nil
}
func maskPath(path string) error {
// bind mount /dev/null if it is file
if err := syscall.Mount("/dev/null", path, "", syscall.MS_BIND, ""); err != nil && !errors.Is(err, os.ErrNotExist) {
if errors.Is(err, syscall.ENOTDIR) {
// otherwise, mount tmpfs to mask it
return syscall.Mount("tmpfs", path, "tmpfs", syscall.MS_RDONLY, "")
}
return fmt.Errorf("mask path: %w", err)
}
return nil
}
func ignoreSignals() {
signal.Ignore(signalToIgnore...)
}

View File

@ -1,36 +1,44 @@
// Package container provides pre-forked container environment to
// run programs in isolated Linux namespaces.
//
// Overview
// # Overview
//
// It creates container within unshared container and communicate
// with host process using unix socket with
// oob for fd / pid and commands encoded by gob.
//
// Protocol
// # Protocol
//
// Host to container communication protocol is single threaded and always initiated by
// the host:
//
// - ping (alive check):
// ## ping (alive check)
//
// - send: ping
// - reply: pong
//
// - conf (set configuration):
// - reply pong
// ## conf (set configuration)
//
// - send: conf
// - reply:
//
// ## open (open files in given mode inside container):
//
// - open (open files in given mode inside container):
// - send: []OpenCmd
// - reply: "success", file fds / "error"
//
// - delete (unlink file / rmdir dir inside container):
// ## delete (unlink file / rmdir dir inside container):
//
// - send: path
// - reply: "finished" / "error"
//
// - reset (clean up container for later use (clear workdir / tmp)):
// ## reset (clean up container for later use (clear workdir / tmp)):
//
// - send:
// - reply: "success"
//
// - execve: (execute file inside container):
// ## execve: (execute file inside container):
//
// - send: argv, env, rLimits, fds
// - reply:
// - success: "success", pid

View File

@ -3,7 +3,9 @@ package container
import (
"context"
"fmt"
"io"
"os"
"os/exec"
"sync"
"syscall"
@ -22,11 +24,24 @@ type Builder struct {
// Root is container root mount path, empty uses current work path
Root string
// TmpRoot defines the tmp dir pattern if not nil. Temp directory will be created as container root dir
TmpRoot string
// Mounts defines container mount points, empty uses default mounts
Mounts []mount.SyscallParams
Mounts []mount.Mount
// SymbolicLinks defines symlinks to be created after mount file system
SymbolicLinks []SymbolicLink
// MaskPaths defines paths to be masked to avoid reading information from
// outside of the container
MaskPaths []string
// WorkDir defines container default work directory (default: /w)
WorkDir string
// Stderr defines whether to dup container stderr to stderr for debug
Stderr bool
Stderr io.Writer
// ExecFile defines executable that called Init, otherwise defer current
// executable (/proc/self/exe)
@ -37,6 +52,29 @@ type Builder struct {
// Clone flags defines unshare clone flag to create container
CloneFlags uintptr
// HostName set container hostname (default: go-sandbox)
HostName string
// DomainName set container domainname (default: go-sandbox)
DomainName string
// InitCommand defines command that runs after the initialization of the container
// to do additional setups (for example, loopback network)
InitCommand []string
// ContainerUID & ContainerGID set the container uid / gid mapping
ContainerUID int
ContainerGID int
// UnshareCgroupBeforeExec calls unshare cgroup before execution
UnshareCgroupBeforeExec bool
}
// SymbolicLink defines symlinks to be created after mount
type SymbolicLink struct {
LinkPath string
Target string
}
// CredGenerator generates uid / gid credential used by container
@ -51,86 +89,141 @@ type Environment interface {
Open([]OpenCmd) ([]*os.File, error)
Delete(p string) error
Reset() error
Execve(context.Context, ExecveParam) <-chan runner.Result
Execve(context.Context, ExecveParam) runner.Result
Destroy() error
}
// container manages single pre-forked container environment
type container struct {
pid int // underlying container init pid
process *os.Process // underlying container init pid
socket *socket // host - container communication
mu sync.Mutex // lock to avoid race condition
done chan struct{}
err error
doneOnce sync.Once
recvCh chan recvReply
sendCh chan sendCmd
}
type recvReply struct {
Reply reply
Msg unixsocket.Msg
}
type sendCmd struct {
Cmd cmd
Msg unixsocket.Msg
}
// Build creates new environment with underlying container
func (b *Builder) Build() (Environment, error) {
c, err := b.startContainer()
if err != nil {
return nil, err
}
// avoid non cinit enabled executable running as container init process
if err = c.Ping(); err != nil {
c.Destroy()
return nil, fmt.Errorf("container: init not responding to ping: %w", err)
}
// container mount points
mounts := b.Mounts
if len(mounts) == 0 {
mounts = mount.NewDefaultBuilder().
WithTmpfs("w", ""). // work dir
WithTmpfs("tmp", ""). // tmp
FilterNotExist().Mounts
}
// container symbolic links
links := b.SymbolicLinks
if len(links) == 0 {
links = defaultSymLinks
}
maskPaths := b.MaskPaths
if len(maskPaths) == 0 {
maskPaths = defaultMaskPaths
}
// container root directory on the host
root := b.Root
if b.TmpRoot != "" {
if root, err = os.MkdirTemp(b.Root, b.TmpRoot); err != nil {
return nil, fmt.Errorf("container: failed to make tmp container root at %s: %w", b.Root, err)
}
defer os.Remove(root)
}
if root == "" {
if root, err = os.Getwd(); err != nil {
return nil, fmt.Errorf("container: failed to get work directory: %w", err)
}
}
workDir := containerWD
if b.WorkDir != "" {
workDir = b.WorkDir
}
hostName := containerName
if b.HostName != "" {
hostName = b.HostName
}
domainName := containerName
if b.DomainName != "" {
domainName = b.DomainName
}
// set configuration and check if container creation successful
if err = c.conf(&containerConfig{
WorkDir: workDir,
HostName: hostName,
DomainName: domainName,
ContainerRoot: root,
Mounts: mounts,
SymbolicLinks: links,
MaskPaths: maskPaths,
InitCommand: b.InitCommand,
Cred: b.CredGenerator != nil,
ContainerUID: b.ContainerUID,
ContainerGID: b.ContainerGID,
UnshareCgroup: b.UnshareCgroupBeforeExec,
}); err != nil {
c.Destroy()
return nil, err
}
return c, nil
}
func (b *Builder) startContainer() (*container, error) {
var (
err error
cred syscall.Credential
uidMap, gidMap []syscall.SysProcIDMap
)
// container mount points
mounts := b.Mounts
if len(mounts) == 0 {
if mounts, err = mount.NewDefaultBuilder().
WithTmpfs("w", ""). // work dir
WithTmpfs("tmp", ""). // tmp
Build(true); err != nil {
return nil, fmt.Errorf("container: failed to build rootfs mount %v", err)
}
}
// container root directory on the host
root := b.Root
if root == "" {
if root, err = os.Getwd(); err != nil {
return nil, fmt.Errorf("container: failed to get work directory %v", err)
}
}
// prepare stdin / stdout / stderr
devNull, err := os.OpenFile(os.DevNull, os.O_RDWR, os.ModePerm)
if err != nil {
return nil, fmt.Errorf("container: failed to open devNull %v", err)
}
defer devNull.Close()
files := make([]uintptr, 0, 4)
files = append(files, devNull.Fd(), devNull.Fd())
if b.Stderr {
files = append(files, os.Stderr.Fd())
} else {
files = append(files, devNull.Fd())
}
// prepare container exec file
execFile, err := b.exec()
if err != nil {
return nil, fmt.Errorf("container: prepare exec %v", err)
}
defer execFile.Close()
// prepare host <-> container unix socket
ins, outs, err := newPassCredSocketPair()
if err != nil {
return nil, fmt.Errorf("container: failed to create socket: %v", err)
return nil, fmt.Errorf("container: failed to create socket: %w", err)
}
defer outs.Close()
outf, err := outs.File()
if err != nil {
ins.Close()
return nil, fmt.Errorf("container: failed to dup container socket fd %v", err)
return nil, fmt.Errorf("container: failed to dup container socket fd: %w", err)
}
defer outf.Close()
files = append(files, uintptr(outf.Fd()))
// prepare container running credential
if b.CredGenerator != nil {
cred = b.CredGenerator.Get()
uidMap, gidMap = getIDMapping(&cred)
uidMap, gidMap = b.getIDMapping(&cred)
} else {
uidMap = []syscall.SysProcIDMap{{HostID: os.Geteuid(), Size: 1}}
gidMap = []syscall.SysProcIDMap{{HostID: os.Getegid(), Size: 1}}
}
var cloneFlag uintptr
@ -140,42 +233,86 @@ func (b *Builder) Build() (Environment, error) {
cloneFlag = b.CloneFlags & forkexec.UnshareFlags
}
r := &forkexec.Runner{
Args: []string{os.Args[0], initArg},
exe := "/proc/self/exe"
if b.ExecFile != "" {
exe = b.ExecFile
}
args := []string{exe, initArg}
r := exec.Cmd{
Path: exe,
Args: args,
Env: []string{PathEnv},
ExecFile: execFile.Fd(),
Files: files,
WorkDir: containerWD,
CloneFlags: cloneFlag,
Mounts: mounts,
HostName: containerName,
DomainName: containerName,
PivotRoot: root,
UIDMappings: uidMap,
GIDMappings: gidMap,
Stderr: b.Stderr,
ExtraFiles: []*os.File{outf},
SysProcAttr: &syscall.SysProcAttr{
Cloneflags: cloneFlag,
UidMappings: uidMap,
GidMappings: gidMap,
AmbientCaps: []uintptr{
unix.CAP_SYS_ADMIN,
unix.CAP_SYS_RESOURCE,
},
Pdeathsig: syscall.SIGKILL,
},
}
pid, err := r.Start()
if err != nil {
if err = r.Start(); err != nil {
ins.Close()
return nil, fmt.Errorf("container: failed to start container %v", err)
return nil, fmt.Errorf("container: failed to start container: %w", err)
}
c := &container{
pid: pid,
process: r.Process,
socket: newSocket(ins),
recvCh: make(chan recvReply, 1),
sendCh: make(chan sendCmd, 1),
done: make(chan struct{}),
}
// set configuration and check if container creation successful
if err = c.conf(&containerConfig{
Cred: b.CredGenerator != nil,
}); err != nil {
c.Destroy()
return nil, err
}
go c.sendLoop()
go c.recvLoop()
return c, nil
}
func (c *container) sendLoop() {
for {
select {
case <-c.done:
return
case cmd, ok := <-c.sendCh:
if !ok {
return
}
if err := c.socket.SendMsg(cmd.Cmd, cmd.Msg); err != nil {
c.socketError(err)
return
}
}
}
}
func (c *container) recvLoop() {
for {
var reply reply
msg, err := c.socket.RecvMsg(&reply)
if err != nil {
c.socketError(err)
return
}
c.recvCh <- recvReply{
Reply: reply,
Msg: msg,
}
}
}
func (c *container) socketError(err error) {
c.doneOnce.Do(func() {
c.err = err
close(c.done)
})
}
// Destroy kill the container process (with its children)
// if stderr enabled, collect the output as error
func (c *container) Destroy() error {
@ -187,29 +324,11 @@ func (c *container) Destroy() error {
defer c.mu.Unlock()
// kill process
var wstatus unix.WaitStatus
unix.Kill(c.pid, unix.SIGKILL)
// wait for container process to exit
_, err := unix.Wait4(c.pid, &wstatus, 0, nil)
for err == unix.EINTR {
_, err = unix.Wait4(c.pid, &wstatus, 0, nil)
}
c.process.Kill()
_, err := c.process.Wait()
return err
}
// exec prepares executable
func (b *Builder) exec() (*os.File, error) {
if b.ExecFile != "" {
return os.Open(b.ExecFile)
}
return OpenCurrentExec()
}
// OpenCurrentExec opens current executable (/proc/self/exe)
func OpenCurrentExec() (*os.File, error) {
return os.Open(currentExec)
}
// newPassCredSocketPair creates socket pair and let the first socket to receive credential information
func newPassCredSocketPair() (*unixsocket.Socket, *unixsocket.Socket, error) {
ins, outs, err := unixsocket.NewSocketPair()
@ -224,7 +343,17 @@ func newPassCredSocketPair() (*unixsocket.Socket, *unixsocket.Socket, error) {
return ins, outs, nil
}
func getIDMapping(cred *syscall.Credential) ([]syscall.SysProcIDMap, []syscall.SysProcIDMap) {
func (b *Builder) getIDMapping(cred *syscall.Credential) ([]syscall.SysProcIDMap, []syscall.SysProcIDMap) {
cUID := b.ContainerUID
if cUID == 0 {
cUID = containerUID
}
cGID := b.ContainerGID
if cGID == 0 {
cGID = containerGID
}
uidMap := []syscall.SysProcIDMap{
{
ContainerID: 0,
@ -232,7 +361,7 @@ func getIDMapping(cred *syscall.Credential) ([]syscall.SysProcIDMap, []syscall.S
Size: 1,
},
{
ContainerID: containerUID,
ContainerID: cUID,
HostID: int(cred.Uid),
Size: 1,
},
@ -245,7 +374,7 @@ func getIDMapping(cred *syscall.Credential) ([]syscall.SysProcIDMap, []syscall.S
Size: 1,
},
{
ContainerID: containerGID,
ContainerID: cGID,
HostID: int(cred.Gid),
Size: 1,
},
@ -253,3 +382,33 @@ func getIDMapping(cred *syscall.Credential) ([]syscall.SysProcIDMap, []syscall.S
return uidMap, gidMap
}
func (c *container) recvAckReply(name string) error {
reply, _, err := c.recvReply()
if err != nil {
return fmt.Errorf("%s: recv ack: %w", name, err)
}
if reply.Error != nil {
return fmt.Errorf("%s: container error: %v", name, reply.Error)
}
return nil
}
func (c *container) recvReply() (reply, unixsocket.Msg, error) {
select {
case <-c.done:
return reply{}, unixsocket.Msg{}, c.err
case recv := <-c.recvCh:
return recv.Reply, recv.Msg, nil
}
}
func (c *container) sendCmd(cmd cmd, msg unixsocket.Msg) error {
select {
case <-c.done:
return c.err
case c.sendCh <- sendCmd{Cmd: cmd, Msg: msg}:
return nil
}
}

View File

@ -3,12 +3,13 @@ package container
import (
"fmt"
"os"
"syscall"
"time"
"github.com/criyle/go-sandbox/pkg/unixsocket"
)
// Ping send ping message to container
// Ping send ping message to container, wait for 3 second before timeout
func (c *container) Ping() error {
c.mu.Lock()
defer c.mu.Unlock()
@ -22,8 +23,8 @@ func (c *container) Ping() error {
cmd := cmd{
Cmd: cmdPing,
}
if err := c.sendCmd(&cmd, nil); err != nil {
return fmt.Errorf("ping: %v", err)
if err := c.sendCmd(cmd, unixsocket.Msg{}); err != nil {
return fmt.Errorf("ping: %w", err)
}
// receive no error
return c.recvAckReply("ping")
@ -38,8 +39,8 @@ func (c *container) conf(conf *containerConfig) error {
Cmd: cmdConf,
ConfCmd: &confCmd{Conf: *conf},
}
if err := c.sendCmd(&cmd, nil); err != nil {
return fmt.Errorf("conf: %v", err)
if err := c.sendCmd(cmd, unixsocket.Msg{}); err != nil {
return fmt.Errorf("conf: %w", err)
}
return c.recvAckReply("conf")
}
@ -49,32 +50,36 @@ func (c *container) Open(p []OpenCmd) ([]*os.File, error) {
c.mu.Lock()
defer c.mu.Unlock()
syscall.ForkLock.RLock()
defer syscall.ForkLock.RUnlock()
// send copyin
cmd := cmd{
Cmd: cmdOpen,
OpenCmd: p,
}
if err := c.sendCmd(&cmd, nil); err != nil {
return nil, fmt.Errorf("open: %v", err)
if err := c.sendCmd(cmd, unixsocket.Msg{}); err != nil {
return nil, fmt.Errorf("open: %w", err)
}
reply, msg, err := c.recvReply()
if err != nil {
return nil, fmt.Errorf("open: %v", err)
return nil, fmt.Errorf("open: %w", err)
}
if reply.Error != nil {
return nil, fmt.Errorf("open: %v", reply.Error)
}
if len(msg.Fds) != len(p) {
closeFds(msg.Fds)
return nil, fmt.Errorf("open: unexpected number of fd %v / %v", len(msg.Fds), len(p))
return nil, fmt.Errorf("open: unexpected number of fds: got %d, want %d", len(msg.Fds), len(p))
}
ret := make([]*os.File, 0, len(p))
for i, fd := range msg.Fds {
syscall.CloseOnExec(fd)
f := os.NewFile(uintptr(fd), p[i].Path)
if f == nil {
closeFds(msg.Fds)
return nil, fmt.Errorf("open: failed NewFile %v", fd)
return nil, fmt.Errorf("open: failed to create file for fd: %d", fd)
}
ret = append(ret, f)
}
@ -90,8 +95,8 @@ func (c *container) Delete(p string) error {
Cmd: cmdDelete,
DeleteCmd: &deleteCmd{Path: p},
}
if err := c.sendCmd(&cmd, nil); err != nil {
return fmt.Errorf("delete: %v", err)
if err := c.sendCmd(cmd, unixsocket.Msg{}); err != nil {
return fmt.Errorf("delete: %w", err)
}
return c.recvAckReply("delete")
}
@ -104,32 +109,8 @@ func (c *container) Reset() error {
cmd := cmd{
Cmd: cmdReset,
}
if err := c.sendCmd(&cmd, nil); err != nil {
return fmt.Errorf("reset: %v", err)
if err := c.sendCmd(cmd, unixsocket.Msg{}); err != nil {
return fmt.Errorf("reset: %w", err)
}
return c.recvAckReply("reset")
}
func (c *container) recvAckReply(name string) error {
reply, _, err := c.recvReply()
if err != nil {
return fmt.Errorf("%v: recvAck %v", name, err)
}
if reply.Error != nil {
return fmt.Errorf("%v: container error %v", name, reply.Error)
}
return nil
}
func (c *container) recvReply() (*reply, *unixsocket.Msg, error) {
reply := new(reply)
msg, err := c.socket.RecvMsg(reply)
if err != nil {
return nil, nil, err
}
return reply, msg, nil
}
func (c *container) sendCmd(cmd *cmd, msg *unixsocket.Msg) error {
return c.socket.SendMsg(cmd, msg)
}

View File

@ -6,6 +6,7 @@ import (
"time"
"github.com/criyle/go-sandbox/pkg/rlimit"
"github.com/criyle/go-sandbox/pkg/seccomp"
"github.com/criyle/go-sandbox/pkg/unixsocket"
"github.com/criyle/go-sandbox/runner"
)
@ -24,142 +25,152 @@ type ExecveParam struct {
// ExecFile specifies file descriptor for executable file using fexecve
ExecFile uintptr
// CgroupFD specifies file descriptor for cgroup V2
CgroupFD uintptr
// RLimits specifies POSIX Resource limit through setrlimit
RLimits []rlimit.RLimit
// Seccomp specifies seccomp filter
Seccomp seccomp.Filter
// CTTY specifies whether to set controlling TTY
CTTY bool
// SyncFunc calls with pid just before execve (for attach the process to cgroups)
SyncFunc func(pid int) error
// SyncAfterExec makes syncFunc sync after the start of the execution
// Thus, since pid is not guarantee to be exist (may exit early), it is not passed
SyncAfterExec bool
}
// Execve runs process inside container. It accepts context cancelation as time limit exceeded.
func (c *container) Execve(ctx context.Context, param ExecveParam) <-chan runner.Result {
// Execve runs process inside container. It accepts context cancellation as time limit exceeded.
func (c *container) Execve(ctx context.Context, param ExecveParam) runner.Result {
c.mu.Lock()
defer c.mu.Unlock()
sTime := time.Now()
// make sure goroutine not leaked (blocked) even if result is not consumed
result := make(chan runner.Result, 1)
errResult := func(f string, v ...interface{}) <-chan runner.Result {
result <- runner.Result{
Status: runner.StatusRunnerError,
Error: fmt.Sprintf(f, v...),
}
return result
}
// if execve with fd, put fd at the first parameter
var files []int
if param.ExecFile > 0 {
files = append(files, int(param.ExecFile))
}
if param.CgroupFD > 0 {
files = append(files, int(param.CgroupFD))
}
files = append(files, uintptrSliceToInt(param.Files)...)
msg := &unixsocket.Msg{
msg := unixsocket.Msg{
Fds: files,
}
execCmd := &execCmd{
Argv: param.Args,
Env: param.Env,
RLimits: param.RLimits,
Seccomp: param.Seccomp,
FdExec: param.ExecFile > 0,
CTTY: param.CTTY,
SyncAfter: param.SyncAfterExec,
FdCgroup: param.CgroupFD > 0,
}
cm := cmd{
Cmd: cmdExecve,
ExecCmd: execCmd,
}
if err := c.sendCmd(&cm, msg); err != nil {
c.mu.Unlock()
if err := c.sendCmd(cm, msg); err != nil {
return errResult("execve: sendCmd %v", err)
}
// sync function
reply, msg, err := c.recvReply()
rep, msg, err := c.recvReply()
if err != nil {
c.mu.Unlock()
return errResult("execve: recvReply %v", err)
}
// if sync function did not involved
if reply.Error != nil || msg == nil || msg.Cred == nil {
if rep.Error != nil {
return errResult("execve: %v", rep.Error)
}
// if pid not received
if msg.Cred == nil {
// tell kill function to exit and sync
c.execveSyncKill()
c.mu.Unlock()
return errResult("execve: no pid received or error %v", reply.Error)
// tell err exec function to exit and sync
c.execveSyncKill()
return errResult("execve: no pid received")
}
if param.SyncFunc != nil {
if err := param.SyncFunc(int(msg.Cred.Pid)); err != nil {
// tell sync function to exit and recv error
c.execveSyncKill()
// tell kill function to exit and sync
c.execveSyncKill()
c.mu.Unlock()
return errResult("execve: syncfunc failed %v", err)
}
}
// send to syncFunc ack ok
if err := c.sendCmd(&cmd{Cmd: cmdOk}, nil); err != nil {
c.mu.Unlock()
if err := c.sendCmd(cmd{Cmd: cmdOk}, unixsocket.Msg{}); err != nil {
return errResult("execve: ack failed %v", err)
}
// wait for done
return c.waitForDone(ctx, sTime)
}
func (c *container) waitForDone(ctx context.Context, sTime time.Time) runner.Result {
mTime := time.Now()
select {
case <-c.done: // socket error
return convertReplyResult(reply{}, sTime, mTime, c.err)
waitDone := make(chan struct{})
case <-ctx.Done(): // cancel
c.sendCmd(cmd{Cmd: cmdKill}, unixsocket.Msg{}) // kill
reply, _, err := c.recvReply()
return convertReplyResult(reply, sTime, mTime, err)
// Wait
go func() {
reply2, _, err := c.recvReply()
close(waitDone)
// done signal (should recv after kill)
c.recvReply()
// unlock after last read / write
c.mu.Unlock()
case ret := <-c.recvCh: // result
err := c.sendCmd(cmd{Cmd: cmdKill}, unixsocket.Msg{}) // kill
return convertReplyResult(ret.Reply, sTime, mTime, err)
}
}
func convertReplyResult(reply reply, sTime, mTime time.Time, err error) runner.Result {
// handle potential error
if err != nil {
result <- runner.Result{
return runner.Result{
Status: runner.StatusRunnerError,
Error: err.Error(),
}
return
}
if reply2.Error != nil {
result <- runner.Result{
if reply.Error != nil {
return runner.Result{
Status: runner.StatusRunnerError,
Error: reply2.Error.Error(),
Error: reply.Error.Error(),
}
return
}
if reply2.ExecReply == nil {
result <- runner.Result{
if reply.ExecReply == nil {
return runner.Result{
Status: runner.StatusRunnerError,
Error: "execve: no reply received",
}
return
}
// emit result after all communication finish
result <- runner.Result{
Status: reply2.ExecReply.Status,
ExitStatus: reply2.ExecReply.ExitStatus,
Time: reply2.ExecReply.Time,
Memory: reply2.ExecReply.Memory,
return runner.Result{
Status: reply.ExecReply.Status,
ExitStatus: reply.ExecReply.ExitStatus,
Time: reply.ExecReply.Time,
Memory: reply.ExecReply.Memory,
SetUpTime: mTime.Sub(sTime),
RunningTime: time.Since(mTime),
}
}()
// Kill (if wait is done, a kill message need to be send to collect zombies)
go func() {
select {
case <-ctx.Done():
case <-waitDone:
}
c.sendCmd(&cmd{Cmd: cmdKill}, nil)
}()
return result
}
// execveSyncKill will send kill and recv reply
func (c *container) execveSyncKill() {
c.sendCmd(&cmd{Cmd: cmdKill}, nil)
c.sendCmd(cmd{Cmd: cmdKill}, unixsocket.Msg{})
c.recvReply()
}
func errResult(f string, v ...interface{}) runner.Result {
return runner.Result{
Status: runner.StatusRunnerError,
Error: fmt.Sprintf(f, v...),
}
}

64
container/lookup_linux.go Normal file
View File

@ -0,0 +1,64 @@
package container
import (
"errors"
"io/fs"
"os"
"path/filepath"
"strings"
)
var (
errNotFound = errors.New("executable file not found in $PATH")
errNoPath = errors.New("no PATH environment variable provided for look up")
)
func findExecutable(file string) error {
d, err := os.Stat(file)
if err != nil {
return err
}
if m := d.Mode(); !m.IsDir() && m&0111 != 0 {
return nil
}
return fs.ErrPermission
}
func lookPath(name string, env []string) (string, error) {
// don't look if abs path provided
if filepath.Base(name) != name {
return name, nil
}
// don't look if exist in current dir
if err := findExecutable(name); err == nil {
return name, nil
}
path, err := findPath(env)
if err != nil {
return "", err
}
for _, dir := range path {
if dir == "" {
dir = "."
}
p := filepath.Join(dir, name)
if err := findExecutable(p); err == nil {
return p, nil
}
}
return "", errNotFound
}
func findPath(env []string) ([]string, error) {
// find PATH=
const pathPrefix = "PATH="
for i := len(env) - 1; i >= 0; i-- {
s := env[i]
if strings.HasPrefix(s, pathPrefix) {
return filepath.SplitList(s[len(pathPrefix):]), nil
}
}
return nil, errNoPath
}

View File

@ -5,18 +5,21 @@ import (
"syscall"
"time"
"github.com/criyle/go-sandbox/pkg/mount"
"github.com/criyle/go-sandbox/pkg/rlimit"
"github.com/criyle/go-sandbox/pkg/seccomp"
"github.com/criyle/go-sandbox/runner"
)
// cmd is the control message send into container
type cmd struct {
Cmd string // type of the cmd
OpenCmd []OpenCmd // open argument
DeleteCmd *deleteCmd // delete argument
ExecCmd *execCmd // execve argument
ConfCmd *confCmd // to set configuration
OpenCmd []OpenCmd // open argument
Cmd cmdType // type of the cmd
}
// OpenCmd correspond to a single open syscall
@ -36,7 +39,11 @@ type execCmd struct {
Argv []string // execve argv
Env []string // execve env
RLimits []rlimit.RLimit // execve posix rlimit
Seccomp seccomp.Filter // seccomp filter
FdExec bool // if use fexecve (fd[0] as exec)
FdCgroup bool // if use cgroupFd
CTTY bool // if set CTTY
SyncAfter bool // if sync function calls after execve returns
}
// confCmd stores conf parameter
@ -46,7 +53,21 @@ type confCmd struct {
// ContainerConfig set the container config
type containerConfig struct {
WorkDir string
HostName string
DomainName string
ContainerRoot string
Mounts []mount.Mount
SymbolicLinks []SymbolicLink
MaskPaths []string
InitCommand []string
ContainerUID int
ContainerGID int
Cred bool
UnshareCgroup bool
}
// reply is the reply message send back to controller
@ -57,8 +78,8 @@ type reply struct {
// errorReply stores error returned back from container
type errorReply struct {
Msg string
Errno *syscall.Errno
Msg string
}
// execReply stores execve result

17
container/signal_linux.go Normal file
View File

@ -0,0 +1,17 @@
//go:build linux && !mips64 && !mips64le
package container
import (
"os"
"syscall"
)
var signalToIgnore = []os.Signal{
// signals that cause run-time panic
syscall.SIGBUS, syscall.SIGFPE, syscall.SIGSEGV,
// signals that cause the program to exit
syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM,
// signals that cause the program to exit with a stack dump
syscall.SIGQUIT, syscall.SIGILL, syscall.SIGTRAP, syscall.SIGABRT, syscall.SIGSTKFLT, syscall.SIGSYS,
}

View File

@ -0,0 +1,17 @@
//go:build linux && (mips64 || mips64le)
package container
import (
"os"
"syscall"
)
var signalToIgnore = []os.Signal{
// signals that cause run-time panic
syscall.SIGBUS, syscall.SIGFPE, syscall.SIGSEGV,
// signals that cause the program to exit
syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM,
// signals that cause the program to exit with a stack dump
syscall.SIGQUIT, syscall.SIGILL, syscall.SIGTRAP, syscall.SIGABRT, syscall.SIGSYS,
}

View File

@ -4,65 +4,66 @@ import (
"bytes"
"encoding/gob"
"fmt"
"sync"
"github.com/criyle/go-sandbox/pkg/unixsocket"
)
// 16k buffsize
// 16k buffer size
const bufferSize = 16 << 10
var bufferPool = sync.Pool{
New: func() interface{} {
return make([]byte, bufferSize)
},
}
type socket struct {
*unixsocket.Socket
recvBuff bytes.Buffer
decoder *gob.Decoder
buff []byte
decoder *gob.Decoder
recvBuff bufferRotator
sendBuff bytes.Buffer
encoder *gob.Encoder
sendBuff bytes.Buffer
}
// bufferRotator replace the underlying Buffers to avoid allocation
type bufferRotator struct {
*bytes.Buffer
}
func (b *bufferRotator) Rotate(buffer *bytes.Buffer) {
b.Buffer = buffer
}
func newSocket(s *unixsocket.Socket) *socket {
soc := socket{
Socket: s,
}
soc.buff = make([]byte, bufferSize)
soc.decoder = gob.NewDecoder(&soc.recvBuff)
soc.encoder = gob.NewEncoder(&soc.sendBuff)
return &soc
}
func (s *socket) RecvMsg(e interface{}) (*unixsocket.Msg, error) {
buff := bufferPool.Get().([]byte)
defer bufferPool.Put(buff)
n, msg, err := s.Socket.RecvMsg(buff)
func (s *socket) RecvMsg(e any) (msg unixsocket.Msg, err error) {
n, msg, err := s.Socket.RecvMsg(s.buff)
if err != nil {
return nil, fmt.Errorf("RecvMsg: %v", err)
return msg, fmt.Errorf("recv msg: %w", err)
}
s.recvBuff.Reset()
s.recvBuff.Write(buff[:n])
s.recvBuff.Rotate(bytes.NewBuffer(s.buff[:n]))
if err := s.decoder.Decode(e); err != nil {
return nil, fmt.Errorf("RecvMsg: failed to decode %v", err)
return msg, fmt.Errorf("recv msg: decode: %w", err)
}
return msg, nil
}
func (s *socket) SendMsg(e interface{}, msg *unixsocket.Msg) error {
func (s *socket) SendMsg(e any, msg unixsocket.Msg) error {
s.sendBuff.Reset()
if err := s.encoder.Encode(e); err != nil {
return fmt.Errorf("SendMsg: failed to encode %v", err)
return fmt.Errorf("send msg: encode: %w", err)
}
if err := s.Socket.SendMsg(s.sendBuff.Bytes(), msg); err != nil {
return fmt.Errorf("SendMsg: failed to SendMsg %v", err)
return fmt.Errorf("send msg: %w", err)
}
return nil
}

View File

@ -2,7 +2,7 @@ package container
import (
"os"
"path"
"path/filepath"
"syscall"
)
@ -54,10 +54,10 @@ func removeContents(dir string) error {
}
for _, name := range names {
err = os.RemoveAll(path.Join(dir, name))
err1 := os.RemoveAll(filepath.Join(dir, name))
if err != nil {
err = err1
}
}
return err
}
}
return nil
}

7
go.mod
View File

@ -1,8 +1,9 @@
module github.com/criyle/go-sandbox
go 1.14
go 1.24
require (
github.com/seccomp/libseccomp-golang v0.9.1
golang.org/x/sys v0.0.0-20200513112337-417ce2331b5c
github.com/elastic/go-seccomp-bpf v1.6.0
golang.org/x/net v0.43.0
golang.org/x/sys v0.35.0
)

18
go.sum
View File

@ -1,4 +1,14 @@
github.com/seccomp/libseccomp-golang v0.9.1 h1:NJjM5DNFOs0s3kYE1WUOr6G8V97sdt46rlXTMfXGWBo=
github.com/seccomp/libseccomp-golang v0.9.1/go.mod h1:GbW5+tmTXfcxTToHLXlScSlAvWlF4P2Ca7zGrPiEpWo=
golang.org/x/sys v0.0.0-20200513112337-417ce2331b5c h1:kISX68E8gSkNYAFRFiDU8rl5RIn1sJYKYb/r2vMLDrU=
golang.org/x/sys v0.0.0-20200513112337-417ce2331b5c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/elastic/go-seccomp-bpf v1.6.0 h1:NYduiYxRJ0ZkIyQVwlSskcqPPSg6ynu5pK0/d7SQATs=
github.com/elastic/go-seccomp-bpf v1.6.0/go.mod h1:5tFsTvH4NtWGfpjsOQD53H8HdVQ+zSZFRUDSGevC0Kc=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
golang.org/x/net v0.43.0 h1:lat02VYK2j4aLzMzecihNvTlJNQUq316m2Mr9rnM6YE=
golang.org/x/net v0.43.0/go.mod h1:vhO1fvI4dGsIjh73sWfUVjj3N7CA9WkKJNQm2svM6Jg=
golang.org/x/sys v0.35.0 h1:vz1N37gP5bs89s7He8XuIYXpyY0+QlsKmzipCbUtyxI=
golang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

View File

@ -0,0 +1,88 @@
package cgroup
import (
"os"
"testing"
)
func BenchmarkCgroup(b *testing.B) {
if err := EnableV2Nesting(); err != nil {
b.Fatal(err)
}
ct, err := GetAvailableControllerV2()
if err != nil {
b.Fatal(err)
}
builder, err := New("benchmark", ct)
if err != nil {
b.Fatal(err)
}
defer builder.Destroy()
b.ResetTimer()
for i := 0; i < b.N; i++ {
cg, err := builder.New("test")
if err != nil {
b.Fatal(err)
}
if err := cg.SetCPUSet([]byte("0")); err != nil {
b.Fatal(err)
}
if err := cg.SetMemoryLimit(4096); err != nil {
b.Fatal(err)
}
if err := cg.SetProcLimit(1); err != nil {
b.Fatal(err)
}
if _, err := cg.CPUUsage(); err != nil {
b.Fatal(err)
}
if _, err := cg.MemoryMaxUsage(); err != nil {
b.Fatal(err)
}
cg.Destroy()
}
}
func TestCgroupAll(t *testing.T) {
// ensure root privilege when testing
if os.Getuid() != 0 {
t.Skip("no root privilege")
}
if err := EnableV2Nesting(); err != nil {
t.Fatal(err)
}
ct, err := GetAvailableControllerV2()
if err != nil {
t.Fatal(err)
}
builder, err := New("benchmark", ct)
if err != nil {
t.Fatal(err)
}
defer builder.Destroy()
if err != nil {
t.Fatal(err)
}
cg, err := builder.New("test")
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() {
cg.Destroy()
})
if err := cg.SetCPUSet([]byte("0")); err != nil {
t.Fatal(err)
}
if err := cg.SetMemoryLimit(4096); err != nil {
t.Fatal(err)
}
if err := cg.SetProcLimit(1); err != nil {
t.Fatal(err)
}
if _, err := cg.CPUUsage(); err != nil {
t.Fatal(err)
}
if _, err := cg.MemoryMaxUsage(); err != nil {
t.Fatal(err)
}
}

View File

@ -1,66 +0,0 @@
package cgroup
import "testing"
func BenchmarkCgroup(b *testing.B) {
builder, err := NewBuilder("benchmark").WithCPUAcct().WithMemory().WithPids().FilterByEnv()
if err != nil {
b.Error(err)
return
}
b.ResetTimer()
for i := 0; i < b.N; i++ {
cg, err := builder.Build()
if err != nil {
b.Error(err)
return
}
if err := cg.SetMemoryLimitInBytes(4096); err != nil {
b.Error(err)
return
}
if err := cg.SetPidsMax(1); err != nil {
b.Error(err)
return
}
if _, err := cg.CpuacctUsage(); err != nil {
b.Error(err)
return
}
if _, err := cg.MemoryMaxUsageInBytes(); err != nil {
b.Error(err)
return
}
cg.Destroy()
}
}
func TestCgroup(t *testing.T) {
builder, err := NewBuilder("test").WithCPUAcct().WithMemory().WithPids().FilterByEnv()
if err != nil {
t.Error(err)
return
}
cg, err := builder.Build()
if err != nil {
t.Error(err)
return
}
if err := cg.SetMemoryLimitInBytes(4096); err != nil {
t.Error(err)
return
}
if err := cg.SetPidsMax(1); err != nil {
t.Error(err)
return
}
if _, err := cg.CpuacctUsage(); err != nil {
t.Error(err)
return
}
if _, err := cg.MemoryMaxUsageInBytes(); err != nil {
t.Error(err)
return
}
cg.Destroy()
}

View File

@ -1,68 +0,0 @@
package cgroup
import (
"fmt"
"strings"
)
// Builder builds cgroup directories
// available: cpuacct, memory, pids
type Builder struct {
Prefix string
CPUAcct, Memory, Pids bool
}
// NewBuilder return a dumb builder without any sub-cgroup
func NewBuilder(prefix string) *Builder {
return &Builder{
Prefix: prefix,
}
}
// WithCPUAcct includes cpuacct cgroup
func (b *Builder) WithCPUAcct() *Builder {
b.CPUAcct = true
return b
}
// WithMemory includes memory cgroup
func (b *Builder) WithMemory() *Builder {
b.Memory = true
return b
}
// WithPids includes pids cgroup
func (b *Builder) WithPids() *Builder {
b.Pids = true
return b
}
// FilterByEnv reads /proc/cgroups and filter out non-exists ones
func (b *Builder) FilterByEnv() (*Builder, error) {
m, err := GetAllSubCgroup()
if err != nil {
return b, err
}
b.CPUAcct = b.CPUAcct && m["cpuacct"]
b.Memory = b.Memory && m["memory"]
b.Pids = b.Pids && m["pids"]
return b, nil
}
// String prints the build properties
func (b *Builder) String() string {
s := make([]string, 0, 3)
for _, t := range []struct {
name string
enabled bool
}{
{"cpuacct", b.CPUAcct},
{"memory", b.Memory},
{"pids", b.Pids},
} {
if t.enabled {
s = append(s, t.name)
}
}
return fmt.Sprintf("cgroup builder: [%s]", strings.Join(s, ", "))
}

View File

@ -1,136 +0,0 @@
package cgroup
import (
"bytes"
"fmt"
"os"
)
// Cgroup is the combination of sub-cgroups
type Cgroup struct {
prefix string
cpuacct, memory, pids *SubCgroup
}
// Build creates new cgrouup directories
func (b *Builder) Build() (cg *Cgroup, err error) {
var (
cpuacctPath, memoryPath, pidsPath string
)
// if failed, remove potential created directory
defer func() {
if err != nil {
remove(cpuacctPath)
remove(memoryPath)
remove(pidsPath)
}
}()
if b.CPUAcct {
if cpuacctPath, err = CreateSubCgroupPath("cpuacct", b.Prefix); err != nil {
return
}
}
if b.Memory {
if memoryPath, err = CreateSubCgroupPath("memory", b.Prefix); err != nil {
return
}
}
if b.Pids {
if pidsPath, err = CreateSubCgroupPath("pids", b.Prefix); err != nil {
return
}
}
return &Cgroup{
prefix: b.Prefix,
cpuacct: NewSubCgroup(cpuacctPath),
memory: NewSubCgroup(memoryPath),
pids: NewSubCgroup(pidsPath),
}, nil
}
// AddProc writes cgroup.procs to all sub-cgroup
func (c *Cgroup) AddProc(pid int) error {
if err := c.cpuacct.WriteUint(cgroupProcs, uint64(pid)); err != nil {
return err
}
if err := c.memory.WriteUint(cgroupProcs, uint64(pid)); err != nil {
return err
}
if err := c.pids.WriteUint(cgroupProcs, uint64(pid)); err != nil {
return err
}
return nil
}
// Destroy removes dir for sub-cgroup, errors are ignored if remove one failed
func (c *Cgroup) Destroy() error {
var err1 error
if err := remove(c.cpuacct.path); err != nil {
err1 = err
}
if err := remove(c.memory.path); err != nil {
err1 = err
}
if err := remove(c.pids.path); err != nil {
err1 = err
}
return err1
}
// CpuacctUsage read cpuacct.usage in ns
func (c *Cgroup) CpuacctUsage() (uint64, error) {
return c.cpuacct.ReadUint("cpuacct.usage")
}
// MemoryMaxUsageInBytes read memory.max_usage_in_bytes
func (c *Cgroup) MemoryMaxUsageInBytes() (uint64, error) {
return c.memory.ReadUint("memory.max_usage_in_bytes")
}
// SetMemoryLimitInBytes write memory.limit_in_bytes
func (c *Cgroup) SetMemoryLimitInBytes(i uint64) error {
return c.memory.WriteUint("memory.limit_in_bytes", i)
}
// SetPidsMax write pids.max
func (c *Cgroup) SetPidsMax(i uint64) error {
return c.pids.WriteUint("pids.max", i)
}
// SetCpuacctUsage write cpuacct.usage in ns
func (c *Cgroup) SetCpuacctUsage(i uint64) error {
return c.cpuacct.WriteUint("cpuacct.usage", i)
}
// SetMemoryMaxUsageInBytes write cpuacct.usage in ns
func (c *Cgroup) SetMemoryMaxUsageInBytes(i uint64) error {
return c.memory.WriteUint("memory.max_usage_in_bytes", i)
}
// FindMemoryStatProperty find certain property from memory.stat
func (c *Cgroup) FindMemoryStatProperty(prop string) (uint64, error) {
content, err := c.memory.ReadFile("memory.stat")
if err != nil {
return 0, err
}
r := bytes.NewReader(content)
for {
var p string
var i uint64
_, err = fmt.Fscanln(r, &p, &i)
if err != nil {
return 0, err
}
if p == prop {
return i, nil
}
}
}
func remove(name string) error {
if name != "" {
return os.Remove(name)
}
return nil
}

View File

@ -0,0 +1,196 @@
package cgroup
import (
"bufio"
"fmt"
"os"
"path/filepath"
"strconv"
"strings"
)
const numberOfControllers = 5
// Controllers defines enabled controller of a cgroup
type Controllers struct {
CPU bool
CPUSet bool
CPUAcct bool
Memory bool
Pids bool
}
// Set changes the enabled status of a specific controller
func (c *Controllers) Set(ct string, value bool) {
switch ct {
case CPU:
c.CPU = value
case CPUSet:
c.CPUSet = value
case CPUAcct:
c.CPUAcct = value
case Memory:
c.Memory = value
case Pids:
c.Pids = value
}
}
// Intersect reset the specific controller if it is not enabled in the other
func (c *Controllers) Intersect(o *Controllers) {
c.CPU = c.CPU && o.CPU
c.CPUSet = c.CPUSet && o.CPUSet
c.CPUAcct = c.CPUAcct && o.CPUAcct
c.Memory = c.Memory && o.Memory
c.Pids = c.Pids && o.Pids
}
// Contains returns true if the current controller enabled all controllers in the other controller
func (c *Controllers) Contains(o *Controllers) bool {
return (c.CPU || !o.CPU) && (c.CPUSet || !o.CPUSet) && (c.CPUAcct || !o.CPUAcct) &&
(c.Memory || !o.Memory) && (c.Pids || !o.Pids)
}
// Names returns a list of string of all enabled container names
func (c *Controllers) Names() []string {
names := make([]string, 0, numberOfControllers)
for _, v := range []struct {
e bool
n string
}{
{c.CPU, CPU},
{c.CPUAcct, CPUAcct},
{c.CPUSet, CPUSet},
{c.Memory, Memory},
{c.Pids, Pids},
} {
if v.e {
names = append(names, v.n)
}
}
return names
}
func (c *Controllers) String() string {
return "[" + strings.Join(c.Names(), ", ") + "]"
}
// Info reads the cgroup mount info from /proc/cgroups
type Info struct {
Hierarchy int
NumCgroups int
Enabled bool
}
// GetCgroupV1Info read /proc/cgroups and return the result
func GetCgroupV1Info() (map[string]Info, error) {
f, err := os.Open(procCgroupsPath)
if err != nil {
return nil, err
}
defer f.Close()
rt := make(map[string]Info)
s := bufio.NewScanner(f)
for s.Scan() {
text := s.Text()
if text[0] == '#' {
continue
}
parts := strings.Fields(text)
if len(parts) < 4 {
continue
}
// format: subsys_name hierarchy num_cgroups enabled
name := parts[0]
hierarchy, err := strconv.Atoi(parts[1])
if err != nil {
return nil, err
}
numCgroups, err := strconv.Atoi(parts[2])
if err != nil {
return nil, err
}
enabled := parts[3] != "0"
rt[name] = Info{
Hierarchy: hierarchy,
NumCgroups: numCgroups,
Enabled: enabled,
}
}
if err := s.Err(); err != nil {
return nil, err
}
return rt, nil
}
// GetCurrentCgroupPrefix returns the cgroup prefix of current process
func GetCurrentCgroupPrefix() (string, error) {
c, err := os.ReadFile(procSelfCgroup)
if err != nil {
return "", err
}
firstLine, _, _ := strings.Cut(string(c), "\n")
f := strings.Split(firstLine, ":")
if len(f) < 3 {
return "", fmt.Errorf("invalid " + procSelfCgroup)
}
return f[2][1:], nil
}
// GetAvailableController returns available cgroup controller in the system
func GetAvailableController() (*Controllers, error) {
if DetectedCgroupType == TypeV1 {
return GetAvailableControllerV1()
}
return GetAvailableControllerV2()
}
// GetAvailableControllerWithPrefix returns available cgroup controller within the cgroup prefix
func GetAvailableControllerWithPrefix(prefix string) (*Controllers, error) {
if DetectedCgroupType == TypeV1 {
return GetAvailableControllerV1()
}
return getAvailableControllerV2(prefix)
}
// GetAvailableControllerV1 reads /proc/cgroups and get all available controller as set
func GetAvailableControllerV1() (*Controllers, error) {
info, err := GetCgroupV1Info()
if err != nil {
return nil, err
}
rt := &Controllers{}
for k, v := range info {
if !v.Enabled {
continue
}
rt.Set(k, true)
}
return rt, nil
}
// GetAvailableControllerV2 reads /sys/fs/cgroup/cgroup.controllers to get all controller
func GetAvailableControllerV2() (*Controllers, error) {
return getAvailableControllerV2(".")
}
func getAvailableControllerV2(prefix string) (*Controllers, error) {
return getAvailableControllerV2path(filepath.Join(basePath, prefix, cgroupControllers))
}
func getAvailableControllerV2path(p string) (*Controllers, error) {
c, err := readFile(p)
if err != nil {
return nil, err
}
m := &Controllers{}
f := strings.Fields(string(c))
for _, v := range f {
m.Set(v, true)
}
return m, nil
}

234
pkg/cgroup/cgroup_linux.go Normal file
View File

@ -0,0 +1,234 @@
package cgroup
import (
"errors"
"fmt"
"os"
"path/filepath"
"strings"
)
// Cgroup defines the common interface to control cgroups
// including v1 and v2 implementations.
// TODO: implement systemd integration
type Cgroup interface {
// AddProc add a process into the cgroup
AddProc(pid ...int) error
// Destroy deletes the cgroup
Destroy() error
// Existing returns true if the cgroup was opened rather than created
Existing() bool
//Nest creates a sub-cgroup, moves current process into that cgroup
Nest(name string) (Cgroup, error)
// CPUUsage reads total cpu usage of cgroup
CPUUsage() (uint64, error)
// MemoryUsage reads current total memory usage
MemoryUsage() (uint64, error)
// MemoryMaxUsageInBytes reads max total memory usage. Not exist in cgroup v2 with kernel version < 5.19
MemoryMaxUsage() (uint64, error)
// ProcessPeak reads maximum number of process ever existed in cgroup. Not exist in cgroup v1 or kernel < 6.1
ProcessPeak() (uint64, error)
// SetCPUBandwidth sets the cpu bandwidth. Times in ns
SetCPUBandwidth(quota, period uint64) error
// SetCpusetCpus sets the available cpu to use (cpuset.cpus).
SetCPUSet([]byte) error
// SetMemoryLimit sets memory.limit_in_bytes
SetMemoryLimit(uint64) error
// SetProcLimit sets pids.max
SetProcLimit(uint64) error
// Processes lists all existing process pid from the cgroup
Processes() ([]int, error)
// New creates a sub-cgroup based on the existing one
New(string) (Cgroup, error)
// Random creates a sub-cgroup based on the existing one but the name is randomly generated
Random(string) (Cgroup, error)
// Open opens the cgroup directory on V2 (used for clone3)
Open() (*os.File, error)
}
// DetectedCgroupType defines the current cgroup type of the system
var DetectedCgroupType = DetectType()
// New creates a new cgroup with provided prefix, it opens existing one if existed
func New(prefix string, ct *Controllers) (Cgroup, error) {
if DetectedCgroupType == TypeV1 {
return newV1(prefix, ct)
}
return newV2(prefix, ct)
}
func loopV1Controllers(ct *Controllers, v1 *V1, f func(string, **v1controller) error) error {
for _, c := range []struct {
available bool
name string
cg **v1controller
}{
{ct.CPU, CPU, &v1.cpu},
{ct.CPUSet, CPUSet, &v1.cpuset},
{ct.CPUAcct, CPUAcct, &v1.cpuacct},
{ct.Memory, Memory, &v1.memory},
{ct.Pids, Pids, &v1.pids},
} {
if !c.available {
continue
}
if err := f(c.name, c.cg); err != nil {
return err
}
}
return nil
}
func newV1(prefix string, ct *Controllers) (cg Cgroup, err error) {
v1 := &V1{
prefix: prefix,
}
// if failed, remove potential created directory
defer func() {
if err != nil && !v1.existing {
for _, p := range v1.all {
remove(p.path)
}
}
}()
if err = loopV1Controllers(ct, v1, func(name string, cg **v1controller) error {
path, err := CreateV1ControllerPath(name, prefix)
*cg = newV1Controller(path)
if errors.Is(err, os.ErrExist) {
if len(v1.all) == 0 {
v1.existing = true
}
return nil
}
if err != nil {
return err
}
v1.all = append(v1.all, *cg)
return nil
}); err != nil {
return
}
// init cpu set before use, otherwise it is not functional
if v1.cpuset != nil {
if err = initCpuset(v1.cpuset.path); err != nil {
return
}
}
return v1, err
}
func newV2(prefix string, ct *Controllers) (cg Cgroup, err error) {
v2 := &V2{
path: filepath.Join(basePath, prefix),
control: ct,
}
if _, err := os.Stat(v2.path); err == nil {
v2.existing = true
}
defer func() {
if err != nil && !v2.existing {
remove(v2.path)
}
}()
// ensure controllers were enabled
s := ct.Names()
controlMsg := []byte("+" + strings.Join(s, " +"))
// start from base dir
entries := strings.Split(prefix, "/")
current := ""
for _, e := range entries {
parent := current
current = current + "/" + e
// try mkdir if not exists
if _, err := os.Stat(filepath.Join(basePath, current)); os.IsNotExist(err) {
if err := os.Mkdir(filepath.Join(basePath, current), dirPerm); err != nil {
return nil, err
}
} else if err != nil {
return nil, err
}
// no err means create success, need to enable it in its parent folder
ect, err := getAvailableControllerV2(current)
if err != nil {
return nil, err
}
if ect.Contains(ct) {
continue
}
if err := writeFile(filepath.Join(basePath, parent, cgroupSubtreeControl), controlMsg, filePerm); err != nil {
return nil, err
}
}
return v2, nil
}
// OpenExisting opens a existing cgroup with provided prefix
func OpenExisting(prefix string, ct *Controllers) (Cgroup, error) {
if DetectedCgroupType == TypeV1 {
return openExistingV1(prefix, ct)
}
return openExistingV2(prefix, ct)
}
func openExistingV1(prefix string, ct *Controllers) (cg Cgroup, err error) {
v1 := &V1{
prefix: prefix,
existing: true,
}
if err = loopV1Controllers(ct, v1, func(name string, cg **v1controller) error {
p := filepath.Join(basePath, name, prefix)
*cg = newV1Controller(p)
// os.IsNotExist
if _, err := os.Stat(p); err != nil {
return err
}
v1.all = append(v1.all, *cg)
return nil
}); err != nil {
return
}
// init cpu set before use, otherwise it is not functional
if v1.cpuset != nil {
if err = initCpuset(v1.cpuset.path); err != nil {
return
}
}
return
}
func openExistingV2(prefix string, ct *Controllers) (cg Cgroup, err error) {
ect, err := getAvailableControllerV2(prefix)
if err != nil {
return nil, err
}
if !ect.Contains(ct) {
return nil, fmt.Errorf("open cgroup v2: requesting %v controllers but %v found", ct, ect)
}
return &V2{
path: filepath.Join(basePath, prefix),
control: ect,
existing: true,
}, nil
}

View File

@ -1,8 +0,0 @@
package cgroup
const (
// systemd mounted cgroups
basePath = "/sys/fs/cgroup"
cgroupProcs = "cgroup.procs"
procCgroupsPath = "/proc/cgroups"
)

View File

@ -0,0 +1,42 @@
package cgroup
// Cgroup constants
const (
// systemd mounted cgroups
basePath = "/sys/fs/cgroup"
cgroupProcs = "cgroup.procs"
procCgroupsPath = "/proc/cgroups"
procSelfCgroup = "/proc/self/cgroup"
cgroupSubtreeControl = "cgroup.subtree_control"
cgroupControllers = "cgroup.controllers"
filePerm = 0644
dirPerm = 0755
CPU = "cpu"
CPUAcct = "cpuacct"
CPUSet = "cpuset"
Memory = "memory"
Pids = "pids"
)
// Type defines the version of cgroup
type Type int
// Type enum for cgroup
const (
TypeV1 = iota + 1
TypeV2
)
func (t Type) String() string {
switch t {
case TypeV1:
return "v1"
case TypeV2:
return "v2"
default:
return "invalid"
}
}

View File

@ -1,17 +1,14 @@
// Package cgroup provices builder to create multiple different cgroup-v1 sub groups
// under systemd defined mount path (i.e.,sys/fs/cgroup).
// Package cgroup provides builder to create cgroup
// under systemd defined mount path (i.e.,sys/fs/cgroup) including v1 and
// v2 implementation.
//
// Current available:
// Available cgroup controller:
//
// cpu
// cpuset
// cpuacct
// memory
// pids
//
// Current not available: cpu, cpuset, devices, freezer, net_cls, blkio, perf_event, net_prio, huge_tlb, rdma
//
// Additional ideas:
//
// cpu share(not used): cpu.share
// reclaim pages from old process: memory.force_empty
// (tasks kill are managed out of cgroup as freeze takes some time)
// freeze: freezer.state
// Current not available: devices, freezer, net_cls, blkio, perf_event, net_prio, huge_tlb, rdma
package cgroup

View File

@ -1,69 +0,0 @@
package cgroup
import (
"errors"
"io/ioutil"
"path"
"strconv"
"strings"
"syscall"
)
// SubCgroup is the accessor for single cgroup resource with given path
type SubCgroup struct {
path string
}
// ErrNotInitialized returned when trying to read from not initialized cgroup
var ErrNotInitialized = errors.New("cgroup was not initialized")
// NewSubCgroup creates a cgroup accessor with given path (path needs to be created in advance)
func NewSubCgroup(p string) *SubCgroup {
return &SubCgroup{path: p}
}
// WriteUint writes uint64 into given file
func (c *SubCgroup) WriteUint(filename string, i uint64) error {
if c.path == "" {
return nil
}
return c.WriteFile(filename, []byte(strconv.FormatUint(i, 10)))
}
// ReadUint read uint64 from given file
func (c *SubCgroup) ReadUint(filename string) (uint64, error) {
if c.path == "" {
return 0, ErrNotInitialized
}
b, err := c.ReadFile(filename)
if err != nil {
return 0, err
}
s, err := strconv.ParseUint(strings.TrimSpace(string(b)), 10, 64)
if err != nil {
return 0, err
}
return s, nil
}
// WriteFile writes cgroup file and handles potential EINTR error while writes to
// the slow device (cgroup)
func (c *SubCgroup) WriteFile(name string, content []byte) error {
p := path.Join(c.path, name)
err := ioutil.WriteFile(p, content, 0664)
for err != nil && errors.Is(err, syscall.EINTR) {
err = ioutil.WriteFile(p, content, 0664)
}
return err
}
// ReadFile reads cgroup file and handles potential EINTR error while read to
// the slow device (cgroup)
func (c *SubCgroup) ReadFile(name string) ([]byte, error) {
p := path.Join(c.path, name)
data, err := ioutil.ReadFile(p)
for err != nil && errors.Is(err, syscall.EINTR) {
data, err = ioutil.ReadFile(p)
}
return data, err
}

View File

@ -1,49 +0,0 @@
package cgroup
import (
"bufio"
"io/ioutil"
"os"
"path"
"strings"
)
// EnsureDirExists creates directories if the path not exists
func EnsureDirExists(path string) error {
if _, err := os.Stat(path); os.IsNotExist(err) {
return os.Mkdir(path, os.ModePerm)
}
return nil
}
// CreateSubCgroupPath creates path for sub-cgroup with given group and prefix
func CreateSubCgroupPath(group, prefix string) (string, error) {
base := path.Join(basePath, group, prefix)
EnsureDirExists(base)
return ioutil.TempDir(base, "")
}
// GetAllSubCgroup reads /proc/cgroups and get all available sub-cgroup as set
func GetAllSubCgroup() (map[string]bool, error) {
f, err := os.Open(procCgroupsPath)
if err != nil {
return nil, err
}
defer f.Close()
rt := make(map[string]bool)
s := bufio.NewScanner(f)
for s.Scan() {
text := s.Text()
if text[0] != '#' {
parts := strings.Fields(text)
if len(parts) >= 4 && parts[3] != "0" {
rt[parts[0]] = true
}
}
}
if err := s.Err(); err != nil {
return nil, err
}
return rt, nil
}

186
pkg/cgroup/utils_linux.go Normal file
View File

@ -0,0 +1,186 @@
package cgroup
import (
"errors"
"fmt"
"io/fs"
"math/rand/v2"
"os"
"path/filepath"
"strconv"
"strings"
"syscall"
"golang.org/x/sys/unix"
)
// EnsureDirExists creates directories if the path not exists
func EnsureDirExists(path string) error {
if _, err := os.Stat(path); os.IsNotExist(err) {
return os.MkdirAll(path, dirPerm)
}
return os.ErrExist
}
// CreateV1ControllerPath create path for controller with given group, prefix
func CreateV1ControllerPath(controller, prefix string) (string, error) {
p := filepath.Join(basePath, controller, prefix)
return p, EnsureDirExists(p)
}
const initPath = "init"
// EnableV2Nesting migrates all process in the container to nested /init path
// and enables all available controllers in the root cgroup
func EnableV2Nesting() error {
if DetectType() != TypeV2 {
return nil
}
p, err := readFile(filepath.Join(basePath, cgroupProcs))
if err != nil {
return err
}
procs := strings.Split(string(p), "\n")
if len(procs) == 0 {
return nil
}
// mkdir init
if err := os.Mkdir(filepath.Join(basePath, initPath), dirPerm); err != nil && !errors.Is(err, os.ErrExist) {
return err
}
// move all process into init cgroup
procFile, err := os.OpenFile(filepath.Join(basePath, initPath, cgroupProcs), os.O_RDWR, filePerm)
if err != nil {
return err
}
for _, v := range procs {
if _, err := procFile.WriteString(v); err != nil {
continue
//return err
}
}
procFile.Close()
return nil
}
// ReadProcesses reads cgroup.procs file and return pids individually
func ReadProcesses(path string) ([]int, error) {
content, err := readFile(path)
if err != nil {
return nil, err
}
procs := strings.Split(string(content), "\n")
rt := make([]int, len(procs))
for i, x := range procs {
if len(x) == 0 {
continue
}
rt[i], err = strconv.Atoi(x)
if err != nil {
return nil, err
}
}
return rt, nil
}
// AddProcesses add processes into cgroup.procs file
func AddProcesses(path string, procs []int) error {
f, err := os.OpenFile(path, os.O_RDWR, filePerm)
if err != nil {
return err
}
defer f.Close()
for _, p := range procs {
if _, err := f.WriteString(strconv.Itoa(p)); err != nil {
return err
}
}
return nil
}
// DetectType detects current mounted cgroup type in systemd default path
func DetectType() Type {
// if /sys/fs/cgroup is mounted as CGROUPV2 or TMPFS (V1)
var st unix.Statfs_t
if err := unix.Statfs(basePath, &st); err != nil {
// ignore errors, defaulting to CgroupV1
return TypeV1
}
if st.Type == unix.CGROUP2_SUPER_MAGIC {
return TypeV2
}
return TypeV1
}
func remove(name string) error {
if name != "" {
// os.Remove tried to Unlink, then Rmdir. Since we only delete directories, use
// Rmdir directly
return syscall.Rmdir(name)
}
return nil
}
var errPatternHasSeparator = errors.New("pattern contains path separator")
// prefixAndSuffix splits pattern by the last wildcard "*", if applicable,
// returning prefix as the part before "*" and suffix as the part after "*".
func prefixAndSuffix(pattern string) (prefix, suffix string, err error) {
for i := 0; i < len(pattern); i++ {
if os.IsPathSeparator(pattern[i]) {
return "", "", errPatternHasSeparator
}
}
if pos := strings.LastIndexByte(pattern, '*'); pos != -1 {
prefix, suffix = pattern[:pos], pattern[pos+1:]
} else {
prefix = pattern
}
return prefix, suffix, nil
}
func readFile(p string) ([]byte, error) {
data, err := os.ReadFile(p)
for err != nil && errors.Is(err, syscall.EINTR) {
data, err = os.ReadFile(p)
}
return data, err
}
func writeFile(p string, content []byte, perm fs.FileMode) error {
err := os.WriteFile(p, content, perm)
for err != nil && errors.Is(err, syscall.EINTR) {
err = os.WriteFile(p, content, perm)
}
return err
}
func nextRandom() string {
return strconv.Itoa(int(rand.Int32()))
}
// randomBuild creates a cgroup with random directory, similar to os.MkdirTemp
func randomBuild(pattern string, build func(string) (Cgroup, error)) (Cgroup, error) {
prefix, suffix, err := prefixAndSuffix(pattern)
if err != nil {
return nil, fmt.Errorf("cgroup.builder: random %w", err)
}
try := 0
for {
name := prefix + nextRandom() + suffix
cg, err := build(name)
if err == nil {
return cg, nil
}
if errors.Is(err, os.ErrExist) || (cg != nil && cg.Existing()) {
if try++; try < 10000 {
continue
}
return nil, fmt.Errorf("cgroup.builder: tried 10000 times but failed")
}
return nil, fmt.Errorf("cgroup.builder: random %w", err)
}
}

285
pkg/cgroup/v1_linux.go Normal file
View File

@ -0,0 +1,285 @@
package cgroup
import (
"bytes"
"fmt"
"os"
"path/filepath"
"strings"
)
var _ Cgroup = &V1{}
// V1 is the combination of v1 controllers
type V1 struct {
prefix string
cpu *v1controller
cpuset *v1controller
cpuacct *v1controller
memory *v1controller
pids *v1controller
all []*v1controller
existing bool
}
func (c *V1) Open() (*os.File, error) {
return nil, ErrNotInitialized
}
func (c *V1) String() string {
names := make([]string, 0, numberOfControllers)
for _, v := range []struct {
now *v1controller
name string
}{
{c.cpu, CPU},
{c.cpuset, CPUSet},
{c.cpuacct, CPUAcct},
{c.memory, Memory},
{c.pids, Pids},
} {
if v.now == nil {
continue
}
names = append(names, v.name)
}
return "v1(" + c.prefix + ")[" + strings.Join(names, ", ") + "]"
}
// AddProc writes cgroup.procs to all controller
func (c *V1) AddProc(pids ...int) error {
for _, s := range c.all {
if err := s.AddProc(pids...); err != nil {
return err
}
}
return nil
}
// Processes lists all existing process pid from the cgroup
func (c *V1) Processes() ([]int, error) {
if len(c.all) == 0 {
return nil, os.ErrInvalid
}
return ReadProcesses(filepath.Join(c.all[0].path, cgroupProcs))
}
// New creates a sub-cgroup based on the existing one
func (c *V1) New(name string) (cg Cgroup, err error) {
v1 := &V1{
prefix: filepath.Join(c.prefix, name),
}
defer func() {
if err != nil {
for _, v := range v1.all {
remove(v.path)
}
}
}()
for _, v := range []struct {
now *v1controller
new **v1controller
}{
{c.cpu, &v1.cpu},
{c.cpuset, &v1.cpuset},
{c.cpuacct, &v1.cpuacct},
{c.memory, &v1.memory},
{c.pids, &v1.pids},
} {
if v.now == nil {
continue
}
p := filepath.Join(v.now.path, name)
*v.new = &v1controller{path: p}
err = EnsureDirExists(p)
if os.IsExist(err) {
err = nil
if len(v1.all) == 0 {
v1.existing = true
}
continue
}
if err != nil {
return
}
v1.all = append(v1.all, *v.new)
}
// init cpu set before use, otherwise it is not functional
if v1.cpuset != nil {
if err = initCpuset(v1.cpuset.path); err != nil {
return
}
}
return v1, nil
}
// Random creates a sub-cgroup based on the existing one but the name is randomly generated
func (c *V1) Random(pattern string) (Cgroup, error) {
return randomBuild(pattern, c.New)
}
// Nest creates a sub-cgroup, moves current process into that cgroup
func (c *V1) Nest(name string) (Cgroup, error) {
v1, err := c.New(name)
if err != nil {
return nil, err
}
p, err := c.Processes()
if err != nil {
return nil, err
}
if err := v1.AddProc(p...); err != nil {
return nil, err
}
return v1, nil
}
// Destroy removes dir for controllers recursively, errors are ignored if remove one failed
func (c *V1) Destroy() error {
var err1 error
for _, s := range c.all {
if c.existing {
continue
}
if err := remove(s.path); err != nil {
err1 = err
}
}
return err1
}
// Existing returns true if the cgroup was opened rather than created
func (c *V1) Existing() bool {
return c.existing
}
// SetCPUBandwidth set cpu quota via cfs interface
func (c *V1) SetCPUBandwidth(quota, period uint64) error {
if err := c.SetCPUCfsQuota(quota); err != nil {
return err
}
return c.SetCPUCfsPeriod(period)
}
// SetCPUSet set cpuset.cpus
func (c *V1) SetCPUSet(b []byte) error {
return c.cpuset.WriteFile("cpuset.cpus", b)
}
// CPUUsage read cpuacct.usage in ns
func (c *V1) CPUUsage() (uint64, error) {
return c.cpuacct.ReadUint("cpuacct.usage")
}
// MemoryUsage read memory.usage_in_bytes
func (c *V1) MemoryUsage() (uint64, error) {
return c.memory.ReadUint("memory.usage_in_bytes")
}
// MemoryMaxUsage read memory.max_usage_in_bytes
func (c *V1) MemoryMaxUsage() (uint64, error) {
return c.memory.ReadUint("memory.max_usage_in_bytes")
}
// ProcessPeak implements Cgroup.
func (c *V1) ProcessPeak() (uint64, error) {
return 0, ErrNotInitialized
}
// SetMemoryLimit write memory.limit_in_bytes
func (c *V1) SetMemoryLimit(i uint64) error {
return c.memory.WriteUint("memory.limit_in_bytes", i)
}
// SetProcLimit write pids.max
func (c *V1) SetProcLimit(i uint64) error {
return c.pids.WriteUint("pids.max", i)
}
// SetCpuacctUsage write cpuacct.usage in ns
func (c *V1) SetCpuacctUsage(i uint64) error {
return c.cpuacct.WriteUint("cpuacct.usage", i)
}
// SetMemoryMaxUsageInBytes write cpuacct.usage in ns
func (c *V1) SetMemoryMaxUsageInBytes(i uint64) error {
return c.memory.WriteUint("memory.max_usage_in_bytes", i)
}
// MemoryMemswMaxUsageInBytes read memory.memsw.max_usage_in_bytes
func (c *V1) MemoryMemswMaxUsageInBytes() (uint64, error) {
return c.memory.ReadUint("memory.memsw.max_usage_in_bytes")
}
// SetMemoryMemswLimitInBytes write memory.memsw.limit_in_bytes
func (c *V1) SetMemoryMemswLimitInBytes(i uint64) error {
return c.memory.WriteUint("memory.memsw.limit_in_bytes", i)
}
// SetCPUCfsPeriod set cpu.cfs_period_us in us
func (c *V1) SetCPUCfsPeriod(p uint64) error {
return c.cpu.WriteUint("cpu.cfs_period_us", p)
}
// SetCPUCfsQuota set cpu.cfs_quota_us in us
func (c *V1) SetCPUCfsQuota(p uint64) error {
return c.cpu.WriteUint("cpu.cfs_quota_us", p)
}
// SetCpusetMems set cpuset.mems
func (c *V1) SetCpusetMems(b []byte) error {
return c.cpuset.WriteFile("cpuset.mems", b)
}
// FindMemoryStatProperty find certain property from memory.stat
func (c *V1) FindMemoryStatProperty(prop string) (uint64, error) {
content, err := c.memory.ReadFile("memory.stat")
if err != nil {
return 0, err
}
r := bytes.NewReader(content)
for {
var p string
var i uint64
_, err = fmt.Fscanln(r, &p, &i)
if err != nil {
return 0, err
}
if p == prop {
return i, nil
}
}
}
// initCpuset will copy the config from the parent cpu sets if not exists
func initCpuset(path string) error {
for _, f := range []string{"cpuset.cpus", "cpuset.mems"} {
if err := copyCgroupPropertyFromParent(path, f); err != nil {
return err
}
}
return nil
}
func copyCgroupPropertyFromParent(path, name string) error {
// ensure current one empty
b, err := os.ReadFile(filepath.Join(path, name))
if err != nil {
return err
}
if len(bytes.TrimSpace(b)) > 0 {
return nil
}
// otherwise copy from parent, first to ensure it is empty by recursion
if err := copyCgroupPropertyFromParent(filepath.Dir(path), name); err != nil {
return err
}
b, err = os.ReadFile(filepath.Join(filepath.Dir(path), name))
if err != nil {
return err
}
return os.WriteFile(filepath.Join(path, name), b, filePerm)
}

View File

@ -0,0 +1,69 @@
package cgroup
import (
"errors"
"path/filepath"
"strconv"
"strings"
)
// v1controller is the accessor for single cgroup resource with given path
type v1controller struct {
path string
}
// ErrNotInitialized returned when trying to read from not initialized cgroup
var ErrNotInitialized = errors.New("cgroup was not initialized")
// newV1Controller creates a cgroup accessor with given path (path needs to be created in advance)
func newV1Controller(p string) *v1controller {
return &v1controller{path: p}
}
// WriteUint writes uint64 into given file
func (c *v1controller) WriteUint(filename string, i uint64) error {
if c == nil || c.path == "" {
return nil
}
return c.WriteFile(filename, []byte(strconv.FormatUint(i, 10)))
}
// ReadUint read uint64 from given file
func (c *v1controller) ReadUint(filename string) (uint64, error) {
if c == nil || c.path == "" {
return 0, ErrNotInitialized
}
b, err := c.ReadFile(filename)
if err != nil {
return 0, err
}
s, err := strconv.ParseUint(strings.TrimSpace(string(b)), 10, 64)
if err != nil {
return 0, err
}
return s, nil
}
// WriteFile writes cgroup file and handles potential EINTR error while writes to
// the slow device (cgroup)
func (c *v1controller) WriteFile(name string, content []byte) error {
if c == nil || c.path == "" {
return ErrNotInitialized
}
p := filepath.Join(c.path, name)
return writeFile(p, content, filePerm)
}
// ReadFile reads cgroup file and handles potential EINTR error while read to
// the slow device (cgroup)
func (c *v1controller) ReadFile(name string) ([]byte, error) {
if c == nil || c.path == "" {
return nil, nil
}
p := filepath.Join(c.path, name)
return readFile(p)
}
func (c *v1controller) AddProc(pids ...int) error {
return AddProcesses(filepath.Join(c.path, cgroupProcs), pids)
}

233
pkg/cgroup/v2_linux.go Normal file
View File

@ -0,0 +1,233 @@
package cgroup
import (
"bufio"
"bytes"
"os"
"path/filepath"
"strconv"
"strings"
"sync"
)
// V2 provides cgroup interface for v2
type V2 struct {
path string
control *Controllers
subtreeOnce sync.Once
subtreeErr error
existing bool
}
var _ Cgroup = &V2{}
func (c *V2) Open() (*os.File, error) {
return os.OpenFile(c.path, 0, dirPerm)
}
func (c *V2) String() string {
ct, _ := getAvailableControllerV2path(filepath.Join(c.path, cgroupControllers))
return "v2(" + c.path + ")" + ct.String()
}
// AddProc adds processes into the cgroup
func (c *V2) AddProc(pids ...int) error {
return AddProcesses(filepath.Join(c.path, cgroupProcs), pids)
}
// Processes returns all processes within the cgroup
func (c *V2) Processes() ([]int, error) {
return ReadProcesses(filepath.Join(c.path, cgroupProcs))
}
// New creates a sub-cgroup based on the existing one
func (c *V2) New(name string) (Cgroup, error) {
if err := c.enableSubtreeControl(); err != nil {
return nil, err
}
v2 := &V2{
path: filepath.Join(c.path, name),
control: c.control,
}
if err := os.Mkdir(v2.path, dirPerm); err != nil {
if !os.IsExist(err) {
return nil, err
}
v2.existing = true
}
return v2, nil
}
// Nest creates a sub-cgroup, moves current process into that cgroup
func (c *V2) Nest(name string) (Cgroup, error) {
v2 := &V2{
path: filepath.Join(c.path, name),
control: c.control,
}
if err := os.Mkdir(v2.path, dirPerm); err != nil {
if !os.IsExist(err) {
return nil, err
}
v2.existing = true
}
p, err := c.Processes()
if err != nil {
return nil, err
}
if err := v2.AddProc(p...); err != nil {
return nil, err
}
if err := c.enableSubtreeControl(); err != nil {
return nil, err
}
return v2, nil
}
func (c *V2) enableSubtreeControl() error {
c.subtreeOnce.Do(func() {
ct, err := getAvailableControllerV2path(filepath.Join(c.path, cgroupControllers))
if err != nil {
c.subtreeErr = err
return
}
ect, err := getAvailableControllerV2path(filepath.Join(c.path, cgroupSubtreeControl))
if err != nil {
c.subtreeErr = err
return
}
if ect.Contains(ct) {
return
}
s := ct.Names()
controlMsg := []byte("+" + strings.Join(s, " +"))
c.subtreeErr = writeFile(filepath.Join(c.path, cgroupSubtreeControl), controlMsg, filePerm)
})
return c.subtreeErr
}
// Random creates a sub-cgroup based on the existing one but the name is randomly generated
func (c *V2) Random(pattern string) (Cgroup, error) {
return randomBuild(pattern, c.New)
}
// Destroy destroys the cgroup
func (c *V2) Destroy() error {
if !c.existing {
return remove(c.path)
}
return nil
}
// Existing returns true if the cgroup was opened rather than created
func (c *V2) Existing() bool {
return c.existing
}
// CPUUsage reads cpu.stat usage_usec
func (c *V2) CPUUsage() (uint64, error) {
b, err := c.ReadFile("cpu.stat")
if err != nil {
return 0, err
}
s := bufio.NewScanner(bytes.NewReader(b))
for s.Scan() {
parts := strings.Fields(s.Text())
if len(parts) == 2 && parts[0] == "usage_usec" {
v, err := strconv.Atoi(parts[1])
if err != nil {
return 0, err
}
return uint64(v) * 1000, nil // to ns
}
}
return 0, os.ErrNotExist
}
// MemoryUsage reads memory.current
func (c *V2) MemoryUsage() (uint64, error) {
if !c.control.Memory {
return 0, ErrNotInitialized
}
return c.ReadUint("memory.current")
}
// MemoryMaxUsage reads memory.peak
func (c *V2) MemoryMaxUsage() (uint64, error) {
if !c.control.Memory {
return 0, ErrNotInitialized
}
return c.ReadUint("memory.peak")
}
// ProcessPeak reads pids.peak
func (c *V2) ProcessPeak() (uint64, error) {
if !c.control.Pids {
return 0, ErrNotInitialized
}
return c.ReadUint("pids.peak")
}
// SetCPUBandwidth set cpu.max quota period
func (c *V2) SetCPUBandwidth(quota, period uint64) error {
if !c.control.CPU {
return ErrNotInitialized
}
content := strconv.FormatUint(quota, 10) + " " + strconv.FormatUint(period, 10)
return c.WriteFile("cpu.max", []byte(content))
}
// SetCPUSet sets cpuset.cpus
func (c *V2) SetCPUSet(content []byte) error {
if !c.control.CPUSet {
return ErrNotInitialized
}
return c.WriteFile("cpuset.cpus", content)
}
// SetMemoryLimit memory.max
func (c *V2) SetMemoryLimit(l uint64) error {
if !c.control.Memory {
return ErrNotInitialized
}
return c.WriteUint("memory.max", l)
}
// SetProcLimit pids.max
func (c *V2) SetProcLimit(l uint64) error {
if !c.control.Pids {
return ErrNotInitialized
}
return c.WriteUint("pids.max", l)
}
// WriteUint writes uint64 into given file
func (c *V2) WriteUint(filename string, i uint64) error {
return c.WriteFile(filename, []byte(strconv.FormatUint(i, 10)))
}
// ReadUint read uint64 from given file
func (c *V2) ReadUint(filename string) (uint64, error) {
b, err := c.ReadFile(filename)
if err != nil {
return 0, err
}
s, err := strconv.ParseUint(strings.TrimSpace(string(b)), 10, 64)
if err != nil {
return 0, err
}
return s, nil
}
// WriteFile writes cgroup file and handles potential EINTR error while writes to
// the slow device (cgroup)
func (c *V2) WriteFile(name string, content []byte) error {
p := filepath.Join(c.path, name)
return writeFile(p, content, filePerm)
}
// ReadFile reads cgroup file and handles potential EINTR error while read to
// the slow device (cgroup)
func (c *V2) ReadFile(name string) ([]byte, error) {
p := filepath.Join(c.path, name)
return readFile(p)
}

View File

@ -1,39 +0,0 @@
package darwin
import (
"syscall"
"unsafe"
)
// SandboxInit calls sandbox_init
func SandboxInit(profile *byte, flags uint64, errorBuf **byte) (err error) {
var r1 uintptr
r1, _, err = syscall3(funcPC(libc_sandbox_init_trampoline), uintptr(unsafe.Pointer(profile)), uintptr(flags), uintptr(unsafe.Pointer(errorBuf)))
if r1 != 0 {
err = syscall.EINVAL
} else {
err = nil
}
return
}
// SandboxFreeError calls sandbox_free_error
func SandboxFreeError(errorBuf *byte) {
syscall3(funcPC(libc_sandbox_free_error_trampoline), uintptr(unsafe.Pointer(errorBuf)), 0, 0)
}
func libc_sandbox_init_trampoline()
//go:linkname libc_sandbox_init libc_sandbox_init
//go:cgo_import_dynamic libc_sandbox_init sandbox_init "/usr/lib/libSystem.B.dylib"
func libc_sandbox_free_error_trampoline()
//go:linkname libc_sandbox_free_error libc_sandbox_free_error
//go:cgo_import_dynamic libc_sandbox_free_error sandbox_free_error "/usr/lib/libSystem.B.dylib"
//go:linkname syscall3 syscall.syscall
func syscall3(fn, a1, a2, a3 uintptr) (r1, r2 uintptr, err syscall.Errno)
//go:linkname funcPC syscall.funcPC
func funcPC(f func()) uintptr

View File

@ -1,6 +0,0 @@
#include "textflag.h"
TEXT ·libc_sandbox_init_trampoline(SB),NOSPLIT,$0-0
JMP libc_sandbox_init(SB)
TEXT ·libc_sandbox_free_error_trampoline(SB),NOSPLIT,$0-0
JMP libc_sandbox_free_error(SB)

View File

@ -1,7 +1,6 @@
package forkexec
import (
"io/ioutil"
"os"
"syscall"
"testing"
@ -20,6 +19,43 @@ var (
defaultBind = []string{"/usr", "/lib", "/lib64", "/bin"}
)
func BenchmarkStdFork(b *testing.B) {
f := openNull(b)
defer f.Close()
b.RunParallel(func(pb *testing.PB) {
for pb.Next() {
pid, err := syscall.ForkExec("/bin/echo", nil, &syscall.ProcAttr{
Env: []string{"PATH=/bin"},
Files: []uintptr{f.Fd(), f.Fd(), f.Fd()},
})
if err != nil {
b.Fatal(err)
}
wait4(pid, b)
}
})
}
func BenchmarkStdForkUser(b *testing.B) {
f := openNull(b)
defer f.Close()
b.RunParallel(func(pb *testing.PB) {
for pb.Next() {
pid, err := syscall.ForkExec("/bin/echo", nil, &syscall.ProcAttr{
Env: []string{"PATH=/bin"},
Files: []uintptr{f.Fd(), f.Fd(), f.Fd()},
Sys: &syscall.SysProcAttr{
Cloneflags: syscall.CLONE_NEWUSER,
},
})
if err != nil {
b.Fatal(err)
}
wait4(pid, b)
}
})
}
// BenchmarkSimpleFork is about 0.70ms/op
func BenchmarkSimpleFork(b *testing.B) {
r, f := getRunner(b)
@ -85,7 +121,7 @@ func BenchmarkUnshareNet(b *testing.B) {
// BenchmarkFastUnshareMountPivot is about 104ms/op
func BenchmarkFastUnshareMountPivot(b *testing.B) {
root, err := ioutil.TempDir("", "ns")
root, err := os.MkdirTemp("", "ns")
if err != nil {
b.Errorf("failed to create temp dir")
}
@ -112,7 +148,7 @@ func BenchmarkUnshareAll(b *testing.B) {
// BenchmarkUnshareMountPivot is about 880ms/op
func BenchmarkUnshareMountPivot(b *testing.B) {
root, err := ioutil.TempDir("", "ns")
root, err := os.MkdirTemp("", "ns")
if err != nil {
b.Errorf("failed to create temp dir")
}
@ -139,13 +175,15 @@ func getRunner(b *testing.B) (*Runner, *os.File) {
func benchmarkRun(r *Runner, b *testing.B) {
b.ResetTimer()
for i := 0; i < b.N; i++ {
b.RunParallel(func(pb *testing.PB) {
for pb.Next() {
pid, err := r.Start()
if err != nil {
b.Fail()
b.Fatal(err)
}
wait4(pid, b)
}
})
}
func getMounts(dirs []string) []mount.SyscallParams {
@ -157,7 +195,7 @@ func getMounts(dirs []string) []mount.SyscallParams {
Flags: roBind,
})
}
m, _ := builder.Build(true)
m, _ := builder.FilterNotExist().Build()
return m
}

View File

@ -0,0 +1,17 @@
package forkexec
// cloneArgs holds arguments for clone3 Linux syscall.
// from src/syscall/exec_linux.go:196
type cloneArgs struct {
flags uint64 // Flags bit mask
pidFD uint64 // Where to store PID file descriptor (int *)
childTID uint64 // Where to store child TID, in child's memory (pid_t *)
parentTID uint64 // Where to store child TID, in parent's memory (pid_t *)
exitSignal uint64 // Signal to deliver to parent on child termination
stack uint64 // Pointer to lowest byte of stack
stackSize uint64 // Size of stack
tls uint64 // Location of new TLS
setTID uint64 // Pointer to a pid_t array (since Linux 5.5)
setTIDSize uint64 // Number of elements in set_tid (since Linux 5.5)
cgroup uint64 // File descriptor for target cgroup of child (since Linux 5.7)
}

View File

@ -46,6 +46,11 @@ var (
Permitted: 0,
Inheritable: 0,
}
// 1ms
etxtbsyRetryInterval = unix.Timespec{
Nsec: 1 * 1000 * 1000,
}
)
const (

View File

@ -0,0 +1,102 @@
package forkexec
import (
"fmt"
"syscall"
)
// ErrorLocation defines the location where child process failed to exec
type ErrorLocation int
// ChildError defines the specific error and location where it failed
type ChildError struct {
Err syscall.Errno
Location ErrorLocation
Index int
}
// Location constants
const (
LocClone ErrorLocation = iota + 1
LocCloseWrite
LocUnshareUserRead
LocGetPid
LocKeepCapability
LocSetGroups
LocSetGid
LocSetUid
LocDup3
LocFcntl
LocSetSid
LocIoctl
LocMountRoot
LocMountTmpfs
LocMountChdir
LocMount
LocMountMkdir
LocPivotRoot
LocUmount
LocUnlink
LocMountRootReadonly
LocChdir
LocSetRlimit
LocSetNoNewPrivs
LocDropCapability
LocSetCap
LocPtraceMe
LocStop
LocSeccomp
LocSyncWrite
LocSyncRead
LocExecve
)
var locToString = []string{
"unknown",
"clone",
"close_write",
"unshare_user_read",
"getpid",
"keep_capability",
"setgroups",
"setgid",
"setuid",
"dup3",
"fcntl",
"setsid",
"ioctl",
"mount(root)",
"mount(tmpfs)",
"mount(chdir)",
"mount",
"mount(mkdir)",
"pivot_root",
"umount",
"unlink",
"mount(readonly)",
"chdir",
"setrlimt",
"set_no_new_privs",
"drop_capability",
"set_cap",
"ptrace_me",
"stop",
"seccomp",
"sync_write",
"sync_read",
"execve",
}
func (e ErrorLocation) String() string {
if e >= LocClone && e <= LocExecve {
return locToString[e]
}
return "unknown"
}
func (e ChildError) Error() string {
if e.Index > 0 {
return fmt.Sprintf("%s(%d): %s", e.Location.String(), e.Index, e.Err.Error())
}
return fmt.Sprintf("%s: %s", e.Location.String(), e.Err.Error())
}

View File

@ -1 +0,0 @@
// to use go:linkname

View File

@ -22,7 +22,7 @@ func forkAndExecInChild(r *Runner, argv0 *byte, argv, env []*byte, workdir, prof
beforeFork()
// UnshareFlags (new namespaces) is activated by clone syscall
r1, _, err1 = rawSyscall(funcPC(libc_fork_trampoline), 0, 0, 0)
r1, _, err1 = rawSyscall(libc_fork_trampoline_addr, 0, 0, 0)
if err1 != 0 || r1 != 0 {
// in parent process, immediate return
return
@ -33,18 +33,24 @@ func forkAndExecInChild(r *Runner, argv0 *byte, argv, env []*byte, workdir, prof
// Notice: cannot call any GO functions beyond this point
// Close write end of pipe
if _, _, err1 = rawSyscall(funcPC(libc_close_trampoline), uintptr(p[0]), 0, 0); err1 != 0 {
if _, _, err1 = rawSyscall(libc_close_trampoline_addr, uintptr(p[0]), 0, 0); err1 != 0 {
goto childerror
}
// Set pg id
_, _, err1 = rawSyscall(libc_setpgid_trampoline_addr, 0, 0, 0)
if err1 != 0 {
goto childerror
}
// Pass 1 & pass 2 assigns fds for child process
// Pass 1: fd[i] < i => nextfd
if pipe < nextfd {
_, _, err1 = rawSyscall(funcPC(libc_dup2_trampoline), uintptr(pipe), uintptr(nextfd), 0)
_, _, err1 = rawSyscall(libc_dup2_trampoline_addr, uintptr(pipe), uintptr(nextfd), 0)
if err1 != 0 {
goto childerror
}
rawSyscall(funcPC(libc_fcntl_trampoline), uintptr(nextfd), syscall.F_SETFD, syscall.FD_CLOEXEC)
rawSyscall(libc_fcntl_trampoline_addr, uintptr(nextfd), syscall.F_SETFD, syscall.FD_CLOEXEC)
pipe = nextfd
nextfd++
}
@ -54,11 +60,11 @@ func forkAndExecInChild(r *Runner, argv0 *byte, argv, env []*byte, workdir, prof
if nextfd == pipe {
nextfd++
}
_, _, err1 = rawSyscall(funcPC(libc_dup2_trampoline), uintptr(fd[i]), uintptr(nextfd), 0)
_, _, err1 = rawSyscall(libc_dup2_trampoline_addr, uintptr(fd[i]), uintptr(nextfd), 0)
if err1 != 0 {
goto childerror
}
rawSyscall(funcPC(libc_fcntl_trampoline), uintptr(nextfd), syscall.F_SETFD, syscall.FD_CLOEXEC)
rawSyscall(libc_fcntl_trampoline_addr, uintptr(nextfd), syscall.F_SETFD, syscall.FD_CLOEXEC)
// Set up close on exec
fd[i] = nextfd
nextfd++
@ -67,18 +73,18 @@ func forkAndExecInChild(r *Runner, argv0 *byte, argv, env []*byte, workdir, prof
// Pass 2: fd[i] => i
for i := 0; i < len(fd); i++ {
if fd[i] == -1 {
rawSyscall(funcPC(libc_close_trampoline), uintptr(i), 0, 0)
rawSyscall(libc_close_trampoline_addr, uintptr(i), 0, 0)
continue
}
if fd[i] == int(i) {
// dup2(i, i) will not clear close on exec flag, need to reset the flag
_, _, err1 = rawSyscall(funcPC(libc_fcntl_trampoline), uintptr(fd[i]), syscall.F_SETFD, 0)
_, _, err1 = rawSyscall(libc_fcntl_trampoline_addr, uintptr(fd[i]), syscall.F_SETFD, 0)
if err1 != 0 {
goto childerror
}
continue
}
_, _, err1 = rawSyscall(funcPC(libc_dup2_trampoline), uintptr(fd[i]), uintptr(i), 0)
_, _, err1 = rawSyscall(libc_dup2_trampoline_addr, uintptr(fd[i]), uintptr(i), 0)
if err1 != 0 {
goto childerror
}
@ -86,7 +92,7 @@ func forkAndExecInChild(r *Runner, argv0 *byte, argv, env []*byte, workdir, prof
// chdir for child
if workdir != nil {
_, _, err1 = rawSyscall(funcPC(libc_chdir_trampoline), uintptr(unsafe.Pointer(workdir)), 0, 0)
_, _, err1 = rawSyscall(libc_chdir_trampoline_addr, uintptr(unsafe.Pointer(workdir)), 0, 0)
if err1 != 0 {
goto childerror
}
@ -94,15 +100,18 @@ func forkAndExecInChild(r *Runner, argv0 *byte, argv, env []*byte, workdir, prof
// Set limit
for _, rlim := range r.RLimits {
_, _, err1 = rawSyscall(funcPC(libc_setrlimit_trampoline), uintptr(rlim.Res), uintptr(unsafe.Pointer(&rlim.Rlim)), 0)
_, _, err1 = rawSyscall(libc_setrlimit_trampoline_addr, uintptr(rlim.Res), uintptr(unsafe.Pointer(&rlim.Rlim)), 0)
if err1 != 0 {
if err1 == syscall.EINVAL && (rlim.Res == syscall.RLIMIT_DATA || rlim.Res == syscall.RLIMIT_AS) {
continue
}
goto childerror
}
}
// Load sandbox profile
if profile != nil {
r1, _, err1 = rawSyscall(funcPC(libc_sandbox_init_trampoline), uintptr(unsafe.Pointer(profile)), 0, uintptr(unsafe.Pointer(&errBuf)))
r1, _, err1 = rawSyscall(libc_sandbox_init_trampoline_addr, uintptr(unsafe.Pointer(profile)), 0, uintptr(unsafe.Pointer(&errBuf)))
if err1 != 0 {
goto childerror
}
@ -110,31 +119,31 @@ func forkAndExecInChild(r *Runner, argv0 *byte, argv, env []*byte, workdir, prof
err1 = 253
goto childerror
}
rawSyscall(funcPC(libc_sandbox_free_error_trampoline), uintptr(unsafe.Pointer(errBuf)), 0, 0)
rawSyscall(libc_sandbox_free_error_trampoline_addr, uintptr(unsafe.Pointer(errBuf)), 0, 0)
}
// Sync before exec
err2 = 0
r1, _, err1 = rawSyscall(funcPC(libc_write_trampoline), uintptr(pipe), uintptr(unsafe.Pointer(&err2)), unsafe.Sizeof(err2))
r1, _, err1 = rawSyscall(libc_write_trampoline_addr, uintptr(pipe), uintptr(unsafe.Pointer(&err2)), unsafe.Sizeof(err2))
if r1 == 0 || err1 != 0 {
goto childerror
}
r1, _, err1 = rawSyscall(funcPC(libc_read_trampoline), uintptr(pipe), uintptr(unsafe.Pointer(&err2)), unsafe.Sizeof(err2))
r1, _, err1 = rawSyscall(libc_read_trampoline_addr, uintptr(pipe), uintptr(unsafe.Pointer(&err2)), unsafe.Sizeof(err2))
if r1 == 0 || err1 != 0 {
goto childerror
}
// Time to exec.
_, _, err1 = rawSyscall(funcPC(libc_execve_trampoline),
_, _, err1 = rawSyscall(libc_execve_trampoline_addr,
uintptr(unsafe.Pointer(argv0)),
uintptr(unsafe.Pointer(&argv[0])),
uintptr(unsafe.Pointer(&env[0])))
childerror:
// send error code on pipe
rawSyscall(funcPC(libc_write_trampoline), uintptr(pipe), uintptr(unsafe.Pointer(&err1)), unsafe.Sizeof(err1))
rawSyscall(libc_write_trampoline_addr, uintptr(pipe), uintptr(unsafe.Pointer(&err1)), unsafe.Sizeof(err1))
for {
rawSyscall(funcPC(libc_exit_trampoline), uintptr(err1+err2), 0, 0)
rawSyscall(libc_exit_trampoline_addr, uintptr(err1+err2), 0, 0)
}
}

View File

@ -1,24 +1,48 @@
package forkexec
import (
"runtime"
"syscall"
"unsafe"
"github.com/criyle/go-sandbox/pkg/forkexec/vfork"
"github.com/criyle/go-sandbox/pkg/rlimit"
"golang.org/x/sys/unix"
)
// Reference to src/syscall/exec_linux.go
//
//go:noinline
//go:norace
//go:nocheckptr
func forkAndExecInChild(r *Runner, argv0 *byte, argv, env []*byte, workdir, hostname, domainname, pivotRoot *byte, p [2]int) (r1 uintptr, err1 syscall.Errno) {
var (
clone3 *cloneArgs
pid uintptr
err2 syscall.Errno
unshareUser = r.CloneFlags&unix.CLONE_NEWUSER == unix.CLONE_NEWUSER
i int
rlim rlimit.RLimit
)
pipe := p[1]
// similar to exec_linux, avoid side effect by shuffling around
fd, nextfd := prepareFds(r.Files)
pipe := p[1]
flag := r.CloneFlags & UnshareFlags
if r.SyncFunc == nil && !(r.StopBeforeSeccomp || (r.Seccomp != nil && r.Ptrace)) && flag&syscall.CLONE_NEWUSER != syscall.CLONE_NEWUSER {
flag |= syscall.CLONE_VM | syscall.CLONE_VFORK
}
// use clone3 if cgroupFd specified
if r.CgroupFd > 0 {
clone3 = &cloneArgs{
flags: uint64(flag) | unix.CLONE_INTO_CGROUP,
exitSignal: uint64(syscall.SIGCHLD),
cgroup: uint64(r.CgroupFd),
}
}
flag |= uintptr(syscall.SIGCHLD)
// Acquire the fork lock so that no other threads
// create new fds that are not yet close-on-exec
@ -30,7 +54,16 @@ func forkAndExecInChild(r *Runner, argv0 *byte, argv, env []*byte, workdir, host
beforeFork()
// UnshareFlags (new namespaces) is activated by clone syscall
r1, _, err1 = syscall.RawSyscall6(syscall.SYS_CLONE, uintptr(syscall.SIGCHLD)|(r.CloneFlags&UnshareFlags), 0, 0, 0, 0, 0)
if clone3 != nil {
r1, err1 = vfork.RawVforkSyscall(unix.SYS_CLONE3, uintptr(unsafe.Pointer(clone3)), unsafe.Sizeof(*clone3), 0)
} else {
if runtime.GOARCH == "s390x" {
// On Linux/s390, the first two arguments of clone(2) are swapped.
r1, err1 = vfork.RawVforkSyscall(syscall.SYS_CLONE, 0, flag, 0)
} else {
r1, err1 = vfork.RawVforkSyscall(syscall.SYS_CLONE, flag, 0, 0)
}
}
if err1 != 0 || r1 != 0 {
// in parent process, immediate return
return
@ -42,7 +75,7 @@ func forkAndExecInChild(r *Runner, argv0 *byte, argv, env []*byte, workdir, host
// Close write end of pipe
if _, _, err1 = syscall.RawSyscall(syscall.SYS_CLOSE, uintptr(p[0]), 0, 0); err1 != 0 {
goto childerror
childExitError(pipe, LocCloseWrite, err1)
}
// If usernamespace is unshared, uid map and gid map is required to create folders
@ -53,29 +86,31 @@ func forkAndExecInChild(r *Runner, argv0 *byte, argv, env []*byte, workdir, host
if unshareUser {
r1, _, err1 = syscall.RawSyscall(syscall.SYS_READ, uintptr(pipe), uintptr(unsafe.Pointer(&err2)), unsafe.Sizeof(err2))
if err1 != 0 {
goto childerror
childExitError(pipe, LocUnshareUserRead, err1)
}
if r1 != unsafe.Sizeof(err2) {
err1 = syscall.EINVAL
goto childerror
childExitError(pipe, LocUnshareUserRead, err1)
}
if err2 != 0 {
err1 = err2
goto childerror
childExitError(pipe, LocUnshareUserRead, err1)
}
}
// Get pid of child
pid, _, err1 = syscall.RawSyscall(syscall.SYS_GETPID, 0, 0, 0)
if err1 != 0 {
goto childerror
childExitError(pipe, LocGetPid, err1)
}
// keep capabilities through set_uid / set_gid calls (make sure we can use unshare cgroup), later dropped
if r.Credential != nil || r.UnshareCgroupAfterSync {
_, _, err1 = syscall.RawSyscall(syscall.SYS_PRCTL, syscall.PR_SET_SECUREBITS,
_SECURE_KEEP_CAPS_LOCKED|_SECURE_NO_SETUID_FIXUP|_SECURE_NO_SETUID_FIXUP_LOCKED, 0)
if err1 != 0 {
goto childerror
childExitError(pipe, LocKeepCapability, err1)
}
}
// set the credential for the child process(exec_linux.go)
@ -88,16 +123,16 @@ func forkAndExecInChild(r *Runner, argv0 *byte, argv, env []*byte, workdir, host
if !(r.GIDMappings != nil && !r.GIDMappingsEnableSetgroups && ngroups == 0) && !cred.NoSetGroups {
_, _, err1 = syscall.RawSyscall(unix.SYS_SETGROUPS, ngroups, groups, 0)
if err1 != 0 {
goto childerror
childExitError(pipe, LocSetGroups, err1)
}
}
_, _, err1 = syscall.RawSyscall(unix.SYS_SETGID, uintptr(cred.Gid), 0, 0)
if err1 != 0 {
goto childerror
childExitError(pipe, LocSetGid, err1)
}
_, _, err1 = syscall.RawSyscall(unix.SYS_SETUID, uintptr(cred.Uid), 0, 0)
if err1 != 0 {
goto childerror
childExitError(pipe, LocSetUid, err1)
}
}
@ -106,20 +141,24 @@ func forkAndExecInChild(r *Runner, argv0 *byte, argv, env []*byte, workdir, host
if pipe < nextfd {
_, _, err1 = syscall.RawSyscall(syscall.SYS_DUP3, uintptr(pipe), uintptr(nextfd), syscall.O_CLOEXEC)
if err1 != 0 {
goto childerror
childExitError(pipe, LocDup3, err1)
}
pipe = nextfd
nextfd++
}
if r.ExecFile > 0 && int(r.ExecFile) < nextfd {
// Avoid fd rewrite
for nextfd == pipe {
nextfd++
}
_, _, err1 = syscall.RawSyscall(syscall.SYS_DUP3, r.ExecFile, uintptr(nextfd), syscall.O_CLOEXEC)
if err1 != 0 {
goto childerror
childExitError(pipe, LocDup3, err1)
}
r.ExecFile = uintptr(nextfd)
nextfd++
}
for i := 0; i < len(fd); i++ {
for i = 0; i < len(fd); i++ {
if fd[i] >= 0 && fd[i] < int(i) {
// Avoid fd rewrite
for nextfd == pipe || (r.ExecFile > 0 && nextfd == int(r.ExecFile)) {
@ -127,7 +166,7 @@ func forkAndExecInChild(r *Runner, argv0 *byte, argv, env []*byte, workdir, host
}
_, _, err1 = syscall.RawSyscall(syscall.SYS_DUP3, uintptr(fd[i]), uintptr(nextfd), syscall.O_CLOEXEC)
if err1 != 0 {
goto childerror
childExitError(pipe, LocDup3, err1)
}
// Set up close on exec
fd[i] = nextfd
@ -135,7 +174,7 @@ func forkAndExecInChild(r *Runner, argv0 *byte, argv, env []*byte, workdir, host
}
}
// Pass 2: fd[i] => i
for i := 0; i < len(fd); i++ {
for i = 0; i < len(fd); i++ {
if fd[i] == -1 {
syscall.RawSyscall(syscall.SYS_CLOSE, uintptr(i), 0, 0)
continue
@ -144,30 +183,39 @@ func forkAndExecInChild(r *Runner, argv0 *byte, argv, env []*byte, workdir, host
// dup2(i, i) will not clear close on exec flag, need to reset the flag
_, _, err1 = syscall.RawSyscall(syscall.SYS_FCNTL, uintptr(fd[i]), syscall.F_SETFD, 0)
if err1 != 0 {
goto childerror
childExitError(pipe, LocFcntl, err1)
}
continue
}
_, _, err1 = syscall.RawSyscall(syscall.SYS_DUP3, uintptr(fd[i]), uintptr(i), 0)
if err1 != 0 {
goto childerror
childExitError(pipe, LocDup3, err1)
}
}
// Set the pgid, so that the wait operation can apply to only certain
// subgroup of processes
_, _, err1 = syscall.RawSyscall(syscall.SYS_SETPGID, 0, 0, 0)
// Set the session ID
_, _, err1 = syscall.RawSyscall(syscall.SYS_SETSID, 0, 0, 0)
if err1 != 0 {
goto childerror
childExitError(pipe, LocSetSid, err1)
}
// Set the controlling TTY
if r.CTTY {
_, _, err1 = syscall.RawSyscall(syscall.SYS_IOCTL, uintptr(0), uintptr(syscall.TIOCSCTTY), 1)
if err1 != 0 {
childExitError(pipe, LocIoctl, err1)
}
}
// Mount file system
{
// If mount point is unshared, mark root as private to avoid propagate
// outside to the original mount namespace
if r.CloneFlags&syscall.CLONE_NEWNS == syscall.CLONE_NEWNS {
_, _, err1 = syscall.RawSyscall6(syscall.SYS_MOUNT, uintptr(unsafe.Pointer(&none[0])),
uintptr(unsafe.Pointer(&slash[0])), 0, syscall.MS_REC|syscall.MS_PRIVATE, 0, 0)
if err1 != 0 {
goto childerror
childExitError(pipe, LocMountRoot, err1)
}
}
@ -178,30 +226,30 @@ func forkAndExecInChild(r *Runner, argv0 *byte, argv, env []*byte, workdir, host
uintptr(unsafe.Pointer(pivotRoot)), uintptr(unsafe.Pointer(&tmpfs[0])), 0,
uintptr(unsafe.Pointer(&empty[0])), 0)
if err1 != 0 {
goto childerror
childExitError(pipe, LocMountTmpfs, err1)
}
_, _, err1 = syscall.RawSyscall(syscall.SYS_CHDIR, uintptr(unsafe.Pointer(pivotRoot)), 0, 0)
if err1 != 0 {
goto childerror
childExitError(pipe, LocMountChdir, err1)
}
}
// performing mounts
for _, m := range r.Mounts {
for i, m := range r.Mounts {
// mkdirs(target)
for i, p := range m.Prefixes {
for j, p := range m.Prefixes {
// if target mount point is a file, mknod(target)
if i == len(m.Prefixes)-1 && m.MakeNod {
if j == len(m.Prefixes)-1 && m.MakeNod {
_, _, err1 = syscall.RawSyscall(syscall.SYS_MKNODAT, uintptr(_AT_FDCWD), uintptr(unsafe.Pointer(p)), 0755)
if err1 != 0 && err1 != syscall.EEXIST {
goto childerror
childExitErrorWithIndex(pipe, LocMountMkdir, i, err1)
}
break
}
_, _, err1 = syscall.RawSyscall(syscall.SYS_MKDIRAT, uintptr(_AT_FDCWD), uintptr(unsafe.Pointer(p)), 0755)
if err1 != 0 && err1 != syscall.EEXIST {
goto childerror
childExitErrorWithIndex(pipe, LocMountMkdir, i, err1)
}
}
// mount(source, target, fsType, flags, data)
@ -209,43 +257,52 @@ func forkAndExecInChild(r *Runner, argv0 *byte, argv, env []*byte, workdir, host
uintptr(unsafe.Pointer(m.Target)), uintptr(unsafe.Pointer(m.FsType)), uintptr(m.Flags),
uintptr(unsafe.Pointer(m.Data)), 0)
if err1 != 0 {
goto childerror
childExitErrorWithIndex(pipe, LocMount, i, err1)
}
// bind mount is not respect ro flag so that read-only bind mount needs remount
if m.Flags&bindRo == bindRo {
// Ensure the flag retains for bind mount
const mask = syscall.MS_NOSUID | syscall.MS_NODEV | syscall.MS_NOEXEC | syscall.MS_NOATIME | syscall.MS_NODIRATIME | syscall.MS_RELATIME
var s syscall.Statfs_t
_, _, err1 = syscall.RawSyscall(syscall.SYS_STATFS, uintptr(unsafe.Pointer(m.Source)), uintptr(unsafe.Pointer(&s)), 0)
if err1 != 0 {
childExitErrorWithIndex(pipe, LocMount, i, err1)
}
flag := m.Flags | syscall.MS_REMOUNT | uintptr(s.Flags&mask)
_, _, err1 = syscall.RawSyscall6(syscall.SYS_MOUNT, uintptr(unsafe.Pointer(&empty[0])),
uintptr(unsafe.Pointer(m.Target)), uintptr(unsafe.Pointer(m.FsType)),
uintptr(m.Flags|syscall.MS_REMOUNT), uintptr(unsafe.Pointer(m.Data)), 0)
uintptr(flag), uintptr(unsafe.Pointer(m.Data)), 0)
if err1 != 0 {
goto childerror
childExitErrorWithIndex(pipe, LocMount, i, err1)
}
}
}
// pivit_root
// pivot_root
if pivotRoot != nil {
// mkdir("old_root")
_, _, err1 = syscall.RawSyscall(syscall.SYS_MKDIRAT, uintptr(_AT_FDCWD), uintptr(unsafe.Pointer(&oldRoot[0])), 0755)
if err1 != 0 {
goto childerror
childExitError(pipe, LocPivotRoot, err1)
}
// pivot_root(root, "old_root")
_, _, err1 = syscall.RawSyscall(syscall.SYS_PIVOT_ROOT, uintptr(unsafe.Pointer(pivotRoot)), uintptr(unsafe.Pointer(&oldRoot[0])), 0)
if err1 != 0 {
goto childerror
childExitError(pipe, LocPivotRoot, err1)
}
// umount("old_root", MNT_DETACH)
_, _, err1 = syscall.RawSyscall(syscall.SYS_UMOUNT2, uintptr(unsafe.Pointer(&oldRoot[0])), syscall.MNT_DETACH, 0)
if err1 != 0 {
goto childerror
childExitError(pipe, LocPivotRoot, err1)
}
// rmdir("old_root")
_, _, err1 = syscall.RawSyscall(syscall.SYS_UNLINKAT, uintptr(_AT_FDCWD), uintptr(unsafe.Pointer(&oldRoot[0])), uintptr(unix.AT_REMOVEDIR))
if err1 != 0 {
goto childerror
childExitError(pipe, LocPivotRoot, err1)
}
// mount("tmpfs", "/", "tmpfs", MS_BIND | MS_REMOUNT | MS_RDONLY | MS_NOATIME | MS_NOSUID, nil)
@ -254,19 +311,20 @@ func forkAndExecInChild(r *Runner, argv0 *byte, argv, env []*byte, workdir, host
uintptr(syscall.MS_BIND|syscall.MS_REMOUNT|syscall.MS_RDONLY|syscall.MS_NOATIME|syscall.MS_NOSUID),
uintptr(unsafe.Pointer(&empty[0])), 0)
if err1 != 0 {
goto childerror
childExitError(pipe, LocPivotRoot, err1)
}
}
}
// SetHostName
if hostname != nil {
_, _, err1 = syscall.RawSyscall(syscall.SYS_SETHOSTNAME,
syscall.RawSyscall(syscall.SYS_SETHOSTNAME,
uintptr(unsafe.Pointer(hostname)), uintptr(len(r.HostName)), 0)
}
// SetDomainName
if domainname != nil {
_, _, err1 = syscall.RawSyscall(syscall.SYS_SETDOMAINNAME,
syscall.RawSyscall(syscall.SYS_SETDOMAINNAME,
uintptr(unsafe.Pointer(domainname)), uintptr(len(r.DomainName)), 0)
}
@ -274,16 +332,16 @@ func forkAndExecInChild(r *Runner, argv0 *byte, argv, env []*byte, workdir, host
if workdir != nil {
_, _, err1 = syscall.RawSyscall(syscall.SYS_CHDIR, uintptr(unsafe.Pointer(workdir)), 0, 0)
if err1 != 0 {
goto childerror
childExitError(pipe, LocChdir, err1)
}
}
// Set limit
for _, rlim := range r.RLimits {
for i, rlim = range r.RLimits {
// prlimit instead of setrlimit to avoid 32-bit limitation (linux > 3.2)
_, _, err1 = syscall.RawSyscall6(syscall.SYS_PRLIMIT64, 0, uintptr(rlim.Res), uintptr(unsafe.Pointer(&rlim.Rlim)), 0, 0, 0)
if err1 != 0 {
goto childerror
childExitErrorWithIndex(pipe, LocSetRlimit, i, err1)
}
}
@ -291,7 +349,7 @@ func forkAndExecInChild(r *Runner, argv0 *byte, argv, env []*byte, workdir, host
if r.NoNewPrivs || r.Seccomp != nil {
_, _, err1 = syscall.RawSyscall6(syscall.SYS_PRCTL, unix.PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0, 0)
if err1 != 0 {
goto childerror
childExitError(pipe, LocSetNoNewPrivs, err1)
}
}
@ -301,50 +359,51 @@ func forkAndExecInChild(r *Runner, argv0 *byte, argv, env []*byte, workdir, host
_, _, err1 = syscall.RawSyscall(syscall.SYS_PRCTL, syscall.PR_SET_SECUREBITS,
_SECURE_KEEP_CAPS_LOCKED|_SECURE_NO_SETUID_FIXUP|_SECURE_NO_SETUID_FIXUP_LOCKED|_SECURE_NOROOT|_SECURE_NOROOT_LOCKED, 0)
if err1 != 0 {
goto childerror
childExitError(pipe, LocDropCapability, err1)
}
_, _, err1 = syscall.RawSyscall(syscall.SYS_CAPSET, uintptr(unsafe.Pointer(&dropCapHeader)), uintptr(unsafe.Pointer(&dropCapData)), 0)
if err1 != 0 {
goto childerror
childExitError(pipe, LocSetCap, err1)
}
}
// Enable Ptrace & sync with parent (since ptrace_me is a blocking operation)
if r.Ptrace && r.Seccomp != nil {
err2 = 0
{
if r.SyncFunc != nil {
r1, _, err1 = syscall.RawSyscall(syscall.SYS_WRITE, uintptr(pipe), uintptr(unsafe.Pointer(&err2)), uintptr(unsafe.Sizeof(err2)))
if r1 == 0 || err1 != 0 {
goto childerror
childExitError(pipe, LocSyncWrite, err1)
}
r1, _, err1 = syscall.RawSyscall(syscall.SYS_READ, uintptr(pipe), uintptr(unsafe.Pointer(&err2)), uintptr(unsafe.Sizeof(err2)))
if r1 == 0 || err1 != 0 {
goto childerror
childExitError(pipe, LocSyncRead, err1)
}
}
// unshare cgroup namespace
if r.UnshareCgroupAfterSync {
r1, _, err1 = syscall.RawSyscall(syscall.SYS_UNSHARE, uintptr(unix.CLONE_NEWCGROUP), 0, 0)
if err1 != 0 {
goto childerror
}
// do not error if unshare fails, it is not critical
syscall.RawSyscall(syscall.SYS_UNSHARE, uintptr(unix.CLONE_NEWCGROUP), 0, 0)
if r.DropCaps || r.Credential != nil {
// make sure the children have no privilege at all
_, _, err1 = syscall.RawSyscall(syscall.SYS_PRCTL, syscall.PR_SET_SECUREBITS,
_SECURE_KEEP_CAPS_LOCKED|_SECURE_NO_SETUID_FIXUP|_SECURE_NO_SETUID_FIXUP_LOCKED|_SECURE_NOROOT|_SECURE_NOROOT_LOCKED, 0)
if err1 != 0 {
goto childerror
childExitError(pipe, LocKeepCapability, err1)
}
_, _, err1 = syscall.RawSyscall(syscall.SYS_CAPSET, uintptr(unsafe.Pointer(&dropCapHeader)), uintptr(unsafe.Pointer(&dropCapData)), 0)
if err1 != 0 {
goto childerror
childExitError(pipe, LocSetCap, err1)
}
}
}
}
_, _, err1 = syscall.RawSyscall(syscall.SYS_PTRACE, uintptr(syscall.PTRACE_TRACEME), 0, 0)
if err1 != 0 {
goto childerror
childExitError(pipe, LocPtraceMe, err1)
}
}
@ -355,7 +414,7 @@ func forkAndExecInChild(r *Runner, argv0 *byte, argv, env []*byte, workdir, host
// Stop to wait for ptrace tracer
_, _, err1 = syscall.RawSyscall(syscall.SYS_KILL, pid, uintptr(syscall.SIGSTOP), 0)
if err1 != 0 {
goto childerror
childExitError(pipe, LocStop, err1)
}
}
@ -369,46 +428,49 @@ func forkAndExecInChild(r *Runner, argv0 *byte, argv, env []*byte, workdir, host
// Load seccomp filter
_, _, err1 = syscall.RawSyscall(unix.SYS_SECCOMP, SECCOMP_SET_MODE_FILTER, SECCOMP_FILTER_FLAG_TSYNC, uintptr(unsafe.Pointer(r.Seccomp)))
if err1 != 0 {
goto childerror
childExitError(pipe, LocSeccomp, err1)
}
}
// Before exec, sync with parent through pipe (configured as close_on_exec)
if !r.Ptrace || r.Seccomp == nil {
err2 = 0
{
if r.SyncFunc != nil {
r1, _, err1 = syscall.RawSyscall(syscall.SYS_WRITE, uintptr(pipe), uintptr(unsafe.Pointer(&err2)), uintptr(unsafe.Sizeof(err2)))
if r1 == 0 || err1 != 0 {
goto childerror
childExitError(pipe, LocSyncWrite, err1)
}
r1, _, err1 = syscall.RawSyscall(syscall.SYS_READ, uintptr(pipe), uintptr(unsafe.Pointer(&err2)), uintptr(unsafe.Sizeof(err2)))
if r1 == 0 || err1 != 0 {
goto childerror
childExitError(pipe, LocSyncRead, err1)
}
}
// unshare cgroup namespace
if r.UnshareCgroupAfterSync {
r1, _, err1 = syscall.RawSyscall(syscall.SYS_UNSHARE, uintptr(unix.CLONE_NEWCGROUP), 0, 0)
if err1 != 0 {
goto childerror
}
// do not error if unshare fails, it is not critical
syscall.RawSyscall(syscall.SYS_UNSHARE, uintptr(unix.CLONE_NEWCGROUP), 0, 0)
if r.DropCaps || r.Credential != nil {
// make sure the children have no privilege at all
_, _, err1 = syscall.RawSyscall(syscall.SYS_PRCTL, syscall.PR_SET_SECUREBITS,
_SECURE_KEEP_CAPS_LOCKED|_SECURE_NO_SETUID_FIXUP|_SECURE_NO_SETUID_FIXUP_LOCKED|_SECURE_NOROOT|_SECURE_NOROOT_LOCKED, 0)
if err1 != 0 {
goto childerror
childExitError(pipe, LocKeepCapability, err1)
}
_, _, err1 = syscall.RawSyscall(syscall.SYS_CAPSET, uintptr(unsafe.Pointer(&dropCapHeader)), uintptr(unsafe.Pointer(&dropCapData)), 0)
if err1 != 0 {
goto childerror
childExitError(pipe, LocSetCap, err1)
}
}
if r.Seccomp != nil {
// Load seccomp filter
_, _, err1 = syscall.RawSyscall(unix.SYS_SECCOMP, SECCOMP_SET_MODE_FILTER, SECCOMP_FILTER_FLAG_TSYNC, uintptr(unsafe.Pointer(r.Seccomp)))
if err1 != 0 {
goto childerror
childExitError(pipe, LocSeccomp, err1)
}
}
}
}
@ -418,7 +480,7 @@ func forkAndExecInChild(r *Runner, argv0 *byte, argv, env []*byte, workdir, host
if r.Ptrace && r.Seccomp == nil {
_, _, err1 = syscall.RawSyscall(syscall.SYS_PTRACE, uintptr(syscall.PTRACE_TRACEME), 0, 0)
if err1 != 0 {
goto childerror
childExitError(pipe, LocPtraceMe, err1)
}
}
@ -428,36 +490,63 @@ func forkAndExecInChild(r *Runner, argv0 *byte, argv, env []*byte, workdir, host
// if execfile fd is specified, call fexecve
if r.ExecFile > 0 {
_, _, err1 = syscall.RawSyscall6(unix.SYS_EXECVEAT, r.ExecFile,
uintptr(unsafe.Pointer(&empty[0])),
uintptr(unsafe.Pointer(&argv[0])),
uintptr(unsafe.Pointer(&empty[0])), uintptr(unsafe.Pointer(&argv[0])),
uintptr(unsafe.Pointer(&env[0])), unix.AT_EMPTY_PATH, 0)
} else {
_, _, err1 = syscall.RawSyscall6(unix.SYS_EXECVEAT, uintptr(_AT_FDCWD),
uintptr(unsafe.Pointer(argv0)),
uintptr(unsafe.Pointer(&argv[0])),
uintptr(unsafe.Pointer(&env[0])), 0, 0)
_, _, err1 = syscall.RawSyscall(unix.SYS_EXECVE, uintptr(unsafe.Pointer(argv0)),
uintptr(unsafe.Pointer(&argv[0])), uintptr(unsafe.Pointer(&env[0])))
}
// for slow devices, the file close is not as quickly as enough and it is causing ETXTBSY, retrying on this error
for err1 == syscall.ETXTBSY {
// Fix potential ETXTBSY but with caution (max 50 attempt)
// The ETXTBSY happens when we copy the executable into container, another goroutine
// forks but not execve yet (time consuming for setting up mounting points), the forked
// process is still holding the fd of the copied executable fd. However, we don't
// want to have different logic to lock the container creation
for range [50]struct{}{} {
if err1 != syscall.ETXTBSY {
break
}
// wait instead of busy wait
syscall.RawSyscall(unix.SYS_NANOSLEEP, uintptr(unsafe.Pointer(&etxtbsyRetryInterval)), 0, 0)
if r.ExecFile > 0 {
_, _, err1 = syscall.RawSyscall6(unix.SYS_EXECVEAT, r.ExecFile,
uintptr(unsafe.Pointer(&empty[0])),
uintptr(unsafe.Pointer(&argv[0])),
uintptr(unsafe.Pointer(&empty[0])), uintptr(unsafe.Pointer(&argv[0])),
uintptr(unsafe.Pointer(&env[0])), unix.AT_EMPTY_PATH, 0)
} else {
_, _, err1 = syscall.RawSyscall6(unix.SYS_EXECVEAT, uintptr(_AT_FDCWD),
uintptr(unsafe.Pointer(argv0)),
uintptr(unsafe.Pointer(&argv[0])),
uintptr(unsafe.Pointer(&env[0])), 0, 0)
_, _, err1 = syscall.RawSyscall(unix.SYS_EXECVE, uintptr(unsafe.Pointer(argv0)),
uintptr(unsafe.Pointer(&argv[0])), uintptr(unsafe.Pointer(&env[0])))
}
}
childExitError(pipe, LocExecve, err1)
return
}
//go:nosplit
func childExitError(pipe int, loc ErrorLocation, err syscall.Errno) {
// send error code on pipe
childError := ChildError{
Err: err,
Location: loc,
}
// send error code on pipe
syscall.RawSyscall(unix.SYS_WRITE, uintptr(pipe), uintptr(unsafe.Pointer(&childError)), unsafe.Sizeof(childError))
for {
syscall.RawSyscall(syscall.SYS_EXIT, uintptr(err), 0, 0)
}
}
childerror:
//go:nosplit
func childExitErrorWithIndex(pipe int, loc ErrorLocation, idx int, err syscall.Errno) {
// send error code on pipe
syscall.RawSyscall(unix.SYS_WRITE, uintptr(pipe), uintptr(unsafe.Pointer(&err1)), unsafe.Sizeof(err1))
childError := ChildError{
Err: err,
Location: loc,
Index: idx,
}
// send error code on pipe
syscall.RawSyscall(unix.SYS_WRITE, uintptr(pipe), uintptr(unsafe.Pointer(&childError)), unsafe.Sizeof(childError))
for {
syscall.RawSyscall(syscall.SYS_EXIT, uintptr(err1+err2), 0, 0)
syscall.RawSyscall(syscall.SYS_EXIT, uintptr(err), 0, 0)
}
// cannot reach this point
}

View File

@ -1,7 +1,6 @@
package forkexec
import (
"log"
"syscall"
"unsafe"
@ -84,7 +83,7 @@ func syncWithChild(r *Runner, p [2]int, pid int, err1 syscall.Errno) (int, error
unix.Close(p[0])
return 0, syscall.Errno(err1)
}
r1, _, err1 = syscall3(funcPC(libc_read_trampoline), uintptr(p[0]), uintptr(unsafe.Pointer(&err2)), uintptr(unsafe.Sizeof(err2)))
r1, _, err1 = syscall3(libc_read_trampoline_addr, uintptr(p[0]), uintptr(unsafe.Pointer(&err2)), uintptr(unsafe.Sizeof(err2)))
// child returned error code
if r1 != unsafe.Sizeof(err2) || err2 != 0 || err1 != 0 {
err = handlePipeError(r1, err2)
@ -98,13 +97,13 @@ func syncWithChild(r *Runner, p [2]int, pid int, err1 syscall.Errno) (int, error
}
}
// otherwise, ack child (err1 == 0)
r1, _, err1 = syscall3(funcPC(libc_write_trampoline), uintptr(p[0]), uintptr(unsafe.Pointer(&err1)), uintptr(unsafe.Sizeof(err1)))
r1, _, err1 = syscall3(libc_write_trampoline_addr, uintptr(p[0]), uintptr(unsafe.Pointer(&err1)), uintptr(unsafe.Sizeof(err1)))
if err1 != 0 {
goto fail
}
// if read anything mean child failed after sync (close_on_exec so it should not block)
r1, _, err1 = syscall3(funcPC(libc_read_trampoline), uintptr(p[0]), uintptr(unsafe.Pointer(&err2)), uintptr(unsafe.Sizeof(err2)))
r1, _, err1 = syscall3(libc_read_trampoline_addr, uintptr(p[0]), uintptr(unsafe.Pointer(&err2)), uintptr(unsafe.Sizeof(err2)))
unix.Close(p[0])
if r1 != 0 || err1 != 0 {
err = handlePipeError(r1, err2)
@ -122,7 +121,6 @@ failAfterClose:
// check pipe error
func handlePipeError(r1 uintptr, errno syscall.Errno) error {
log.Println(r1, errno, int(errno))
if r1 == unsafe.Sizeof(errno) {
return syscall.Errno(errno)
}

View File

@ -61,10 +61,11 @@ func (r *Runner) Start() (int, error) {
func syncWithChild(r *Runner, p [2]int, pid int, err1 syscall.Errno) (int, error) {
var (
r1 uintptr
err2 syscall.Errno
err error
unshareUser = r.CloneFlags&unix.CLONE_NEWUSER == unix.CLONE_NEWUSER
childErr ChildError
n int
)
// sync with child
@ -73,7 +74,9 @@ func syncWithChild(r *Runner, p [2]int, pid int, err1 syscall.Errno) (int, error
// clone syscall failed
if err1 != 0 {
unix.Close(p[0])
return 0, syscall.Errno(err1)
childErr.Location = LocClone
childErr.Err = err1
return 0, childErr
}
// synchronize with child for uid / gid map
@ -84,33 +87,37 @@ func syncWithChild(r *Runner, p [2]int, pid int, err1 syscall.Errno) (int, error
syscall.RawSyscall(syscall.SYS_WRITE, uintptr(p[0]), uintptr(unsafe.Pointer(&err2)), uintptr(unsafe.Sizeof(err2)))
}
r1, _, err1 = syscall.RawSyscall(syscall.SYS_READ, uintptr(p[0]), uintptr(unsafe.Pointer(&err2)), uintptr(unsafe.Sizeof(err2)))
// if syncfunc return error, then fail child immediately
// only sync if there is a syncFunc
if r.SyncFunc != nil {
n, err = readChildErr(p[0], &childErr)
// child returned error code
if r1 != unsafe.Sizeof(err2) || err2 != 0 || err1 != 0 {
err = handlePipeError(r1, err2)
if (n != int(unsafe.Sizeof(err2)) && n != int(unsafe.Sizeof(childErr))) || childErr.Err != 0 || err != nil {
childErr.Err = handlePipeError(n, childErr.Err)
goto fail
}
// if syncfunc return error, then fail child immediately
if r.SyncFunc != nil {
if err = r.SyncFunc(int(pid)); err != nil {
goto fail
}
}
// otherwise, ack child (err1 == 0)
syscall.RawSyscall(syscall.SYS_WRITE, uintptr(p[0]), uintptr(unsafe.Pointer(&err1)), uintptr(unsafe.Sizeof(err1)))
}
// if stopped before execve by signal SIGSTOP or PTRACE_ME, then do not wait until execve
if r.Ptrace || r.StopBeforeSeccomp {
if r.StopBeforeSeccomp || (r.Seccomp != nil && r.Ptrace) {
// let's wait it in another goroutine to avoid SIGPIPE
go func() {
readChildErr(p[0], &childErr)
unix.Close(p[0])
}()
return int(pid), nil
}
// if read anything mean child failed after sync (close_on_exec so it should not block)
r1, _, err1 = syscall.RawSyscall(syscall.SYS_READ, uintptr(p[0]), uintptr(unsafe.Pointer(&err2)), uintptr(unsafe.Sizeof(err2)))
n, err = readChildErr(p[0], &childErr)
unix.Close(p[0])
if r1 != 0 || err1 != 0 {
err = handlePipeError(r1, err2)
if n != 0 || err != nil {
childErr.Err = handlePipeError(n, childErr.Err)
goto failAfterClose
}
return int(pid), nil
@ -120,12 +127,35 @@ fail:
failAfterClose:
handleChildFailed(int(pid))
if childErr.Err == 0 {
return 0, err
}
return 0, childErr
}
func readChildErr(fd int, childErr *ChildError) (n int, err error) {
for {
n, err = readlen(fd, (*byte)(unsafe.Pointer(childErr)), int(unsafe.Sizeof(*childErr)))
if err != syscall.EINTR {
break
}
}
return
}
// https://cs.opensource.google/go/go/+/refs/tags/go1.18.1:src/syscall/zsyscall_linux_amd64.go;l=944
func readlen(fd int, p *byte, np int) (n int, err error) {
r0, _, e1 := syscall.Syscall(syscall.SYS_READ, uintptr(fd), uintptr(unsafe.Pointer(p)), uintptr(np))
n = int(r0)
if e1 != 0 {
err = syscall.Errno(e1)
}
return
}
// check pipe error
func handlePipeError(r1 uintptr, errno syscall.Errno) error {
if r1 == unsafe.Sizeof(errno) {
func handlePipeError(r1 int, errno syscall.Errno) syscall.Errno {
if uintptr(r1) >= unsafe.Sizeof(errno) {
return syscall.Errno(errno)
}
return syscall.EPIPE

View File

@ -0,0 +1,119 @@
package forkexec
import (
"os"
"syscall"
"testing"
"github.com/criyle/go-sandbox/pkg/mount"
)
func TestFork_DropCaps(t *testing.T) {
t.Parallel()
r := Runner{
Args: []string{"/bin/echo"},
CloneFlags: syscall.CLONE_NEWUSER,
DropCaps: true,
}
_, err := r.Start()
if err != nil {
t.Fatal(err)
}
}
func TestFork_ETXTBSY(t *testing.T) {
f, err := os.CreateTemp("", "")
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() {
os.Remove(f.Name())
f.Close()
})
if err := f.Chmod(0777); err != nil {
t.Fatal(err)
}
echo, err := os.Open("/bin/echo")
if err != nil {
t.Fatal(err)
}
defer echo.Close()
_, err = f.ReadFrom(echo)
if err != nil {
t.Fatal(err)
}
r := Runner{
Args: []string{f.Name()},
ExecFile: f.Fd(),
}
_, err = r.Start()
e, ok := err.(ChildError)
if !ok {
t.Fatalf("not a child error")
}
if e.Err != syscall.ETXTBSY && e.Location != LocExecve && e.Index != 0 {
t.Fatal(err)
}
}
func TestFork_OK(t *testing.T) {
t.Parallel()
f, err := os.CreateTemp("", "")
if err != nil {
t.Fatal(err)
}
defer os.Remove(f.Name())
if err := f.Chmod(0777); err != nil {
t.Fatal(err)
}
echo, err := os.Open("/bin/echo")
if err != nil {
t.Fatal(err)
}
defer echo.Close()
_, err = f.ReadFrom(echo)
if err != nil {
t.Fatal(err)
}
f.Close()
r := Runner{
Args: []string{f.Name()},
}
_, err = r.Start()
if err != nil {
t.Fatal(err)
}
}
func TestFork_ENOENT(t *testing.T) {
t.Parallel()
m, err := mount.NewBuilder().
WithMount(
mount.Mount{
Source: "NOT_EXISTS",
}).Build()
if err != nil {
t.Fatal(err)
}
r := Runner{
Args: []string{"/bin/echo"},
CloneFlags: syscall.CLONE_NEWNS | syscall.CLONE_NEWUSER,
Mounts: m,
}
_, err = r.Start()
e, ok := err.(ChildError)
if !ok {
t.Fatalf("not a child error")
}
if e.Err != syscall.ENOENT && e.Location != LocExecve {
t.Fatal(err)
}
}

View File

@ -14,7 +14,7 @@ type Runner struct {
// POSIX Resource limit set by set rlimit
RLimits []rlimit.RLimit
// file disriptors map for new process, from 0 to len - 1
// file descriptors map for new process, from 0 to len - 1
Files []uintptr
// work path set by chdir(dir) (current working directory for child)
@ -24,7 +24,7 @@ type Runner struct {
// sandbox profile defines the sandbox profile for sandbox_init syscall
SandboxProfile string
// Parent and child process with sync sataus through a socket pair.
// Parent and child process with sync status through a socket pair.
// SyncFunc will invoke with the child pid. If SyncFunc return some error,
// parent will signal child to stop and report the error
// SyncFunc is called right before execve, thus it could track cpu more accurately

View File

@ -21,7 +21,7 @@ type Runner struct {
// POSIX Resource limit set by set rlimit
RLimits []rlimit.RLimit
// file disriptors map for new process, from 0 to len - 1
// file descriptors map for new process, from 0 to len - 1
Files []uintptr
// work path set by chdir(dir) (current working directory for child)
@ -31,21 +31,6 @@ type Runner struct {
// seccomp syscall filter applied to child
Seccomp *syscall.SockFprog
// ptrace controls child process to call ptrace(PTRACE_TRACEME)
// runtime.LockOSThread is required for tracer to call ptrace syscalls
Ptrace bool
// no_new_privs calls prctl(PR_SET_NO_NEW_PRIVS) to 0 to disable calls to
// setuid processes. It is automatically enabled when seccomp filter is provided
NoNewPrivs bool
// stop before seccomp calls kill(getpid(), SIGSTOP) to wait for tracer to continue
// right before the calls to seccomp. It is automatically enabled when seccomp
// filter and ptrace are provided since kill might not be available after
// seccomp and execve might be traced by ptrace
// cannot stop after seccomp since kill might not be allowed by seccomp filter
StopBeforeSeccomp bool
// clone unshare flag to create linux namespace, effective when clone child
// since unshare syscall does not join the new pid group
CloneFlags uintptr
@ -72,30 +57,51 @@ type Runner struct {
// HostName and DomainName to be set after unshare UTS & user (CAP_SYS_ADMIN)
HostName, DomainName string
// drop_caps calls cap_set(self, 0) to drop all capabilities
// from effective, permitted, inheritable capability sets before execve
// it should avoid calls to set ambient capabilities
DropCaps bool
// UidMappings / GidMappings for unshared user namespaces, no-op if mapping is null
UIDMappings []syscall.SysProcIDMap
GIDMappings []syscall.SysProcIDMap
// GidMappingsEnableSetgroups allows / disallows setgroups syscall.
// deny if GIDMappings is nil
GIDMappingsEnableSetgroups bool
// CgroupFd to use when clone3 with CLONE_INTO_CGROUP with kernel >=5.7 and cgroup v2
CgroupFd uintptr
// Credential holds user and group identities to be assumed
// by a child process started by StartProcess.
Credential *syscall.Credential
// Parent and child process with sync sataus through a socket pair.
// Parent and child process with sync status through a socket pair.
// SyncFunc will invoke with the child pid. If SyncFunc return some error,
// parent will signal child to stop and report the error
// SyncFunc is called right before execve, thus it could track cpu more accurately
SyncFunc func(int) error
// ptrace controls child process to call ptrace(PTRACE_TRACEME)
// runtime.LockOSThread is required for tracer to call ptrace syscalls
Ptrace bool
// no_new_privs calls prctl(PR_SET_NO_NEW_PRIVS) to 0 to disable calls to
// setuid processes. It is automatically enabled when seccomp filter is provided
NoNewPrivs bool
// stop before seccomp calls kill(getpid(), SIGSTOP) to wait for tracer to continue
// right before the calls to seccomp. It is automatically enabled when seccomp
// filter and ptrace are provided since kill might not be available after
// seccomp and execve might be traced by ptrace
// cannot stop after seccomp since kill might not be allowed by seccomp filter
StopBeforeSeccomp bool
// GidMappingsEnableSetgroups allows / disallows setgroups syscall.
// deny if GIDMappings is nil
GIDMappingsEnableSetgroups bool
// drop_caps calls cap_set(self, 0) to drop all capabilities
// from effective, permitted, inheritable capability sets before execve
// it should avoid calls to set ambient capabilities
DropCaps bool
// UnshareCgroupAfterSync specifies whether to unshare cgroup namespace after
// sync (the syncFunc might be add the child to the cgroup)
UnshareCgroupAfterSync bool
// CTTY specifies if set the fd 0 as controlling TTY
CTTY bool
}

View File

@ -1,13 +1,12 @@
package darwin
package forkexec
import (
"io/ioutil"
"os"
"testing"
)
func TestWrite(t *testing.T) {
c, err := ioutil.ReadFile("test.sb")
c, err := os.ReadFile("test.sb")
if err != nil {
t.Error(err)
return

View File

@ -1,4 +1,4 @@
package darwin
package forkexec
import (
"errors"

View File

@ -3,15 +3,8 @@ package forkexec
import (
"syscall"
_ "unsafe" // use go:linkname
"github.com/criyle/go-sandbox/pkg/darwin" // use sandbox_init
)
var _ = darwin.SandboxInit
//go:linkname funcPC syscall.funcPC
func funcPC(f func()) uintptr
//go:linkname syscall3 syscall.syscall
func syscall3(fn, a1, a2, a3 uintptr) (r1, r2 uintptr, err syscall.Errno)
@ -21,41 +14,27 @@ func rawSyscall(fn, a1, a2, a3 uintptr) (r1, r2 uintptr, err syscall.Errno)
//go:linkname rawSyscall6 syscall.rawSyscall6
func rawSyscall6(fn, a1, a2, a3, a4, a5, a6 uintptr) (r1, r2 uintptr, err syscall.Errno)
//go:linkname libc_fork_trampoline syscall.libc_fork_trampoline
func libc_fork_trampoline()
var libc_fork_trampoline_addr uintptr
//go:linkname libc_close_trampoline syscall.libc_close_trampoline
func libc_close_trampoline()
var libc_close_trampoline_addr uintptr
//go:linkname libc_read_trampoline syscall.libc_read_trampoline
func libc_read_trampoline()
var libc_read_trampoline_addr uintptr
//go:linkname libc_write_trampoline syscall.libc_write_trampoline
func libc_write_trampoline()
var libc_write_trampoline_addr uintptr
//go:linkname libc_fcntl_trampoline syscall.libc_fcntl_trampoline
func libc_fcntl_trampoline()
var libc_fcntl_trampoline_addr uintptr
//go:linkname libc_dup2_trampoline syscall.libc_dup2_trampoline
func libc_dup2_trampoline()
var libc_dup2_trampoline_addr uintptr
//go:linkname libc_chdir_trampoline syscall.libc_chdir_trampoline
func libc_chdir_trampoline()
var libc_chdir_trampoline_addr uintptr
//go:linkname libc_setrlimit_trampoline syscall.libc_setrlimit_trampoline
func libc_setrlimit_trampoline()
var libc_setrlimit_trampoline_addr uintptr
//go:linkname libc_execve_trampoline syscall.libc_execve_trampoline
func libc_execve_trampoline()
var libc_execve_trampoline_addr uintptr
//go:linkname libc_exit_trampoline syscall.libc_exit_trampoline
func libc_exit_trampoline()
var libc_exit_trampoline_addr uintptr
//go:linkname libc_sandbox_init_trampoline github.com/criyle/go-sandbox/pkg/darwin.libc_sandbox_init_trampoline
func libc_sandbox_init_trampoline()
//go:linkname libc_sandbox_free_error_trampoline github.com/criyle/go-sandbox/pkg/darwin.libc_sandbox_free_error_trampoline
func libc_sandbox_free_error_trampoline()
var libc_setpgid_trampoline_addr uintptr
//go:linkname fcntl syscall.fcntl
func fcntl(fd int, cmd int, arg int) (val int, err error)

View File

@ -0,0 +1,29 @@
// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
#include "textflag.h"
// See ../runtime/sys_linux_386.s for the reason why we always use int 0x80
// instead of the glibc-specific "CALL 0x10(GS)".
#define INVOKE_SYSCALL INT $0x80
// func RawVforkSyscall(trap, a1, a2, a3 uintptr) (r1, err uintptr)
TEXT ·RawVforkSyscall(SB),NOSPLIT|NOFRAME,$0-24
MOVL trap+0(FP), AX // syscall entry
MOVL a1+4(FP), BX
MOVL a2+8(FP), CX
MOVL a3+12(FP), DX
POPL SI // preserve return address
INVOKE_SYSCALL
PUSHL SI
CMPL AX, $0xfffff001
JLS ok
MOVL $-1, r1+16(FP)
NEGL AX
MOVL AX, err+20(FP)
RET
ok:
MOVL AX, r1+16(FP)
MOVL $0, err+20(FP)
RET

View File

@ -0,0 +1,28 @@
// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
#include "textflag.h"
// func RawVforkSyscall(trap, a1, a2, a3 uintptr) (r1, err uintptr)
TEXT ·RawVforkSyscall(SB),NOSPLIT|NOFRAME,$0-48
MOVQ a1+8(FP), DI
MOVQ a2+16(FP), SI
MOVQ a3+24(FP), DX
MOVQ $0, R10
MOVQ $0, R8
MOVQ $0, R9
MOVQ trap+0(FP), AX // syscall entry
POPQ R12 // preserve return address
SYSCALL
PUSHQ R12
CMPQ AX, $0xfffffffffffff001
JLS ok2
MOVQ $-1, r1+32(FP)
NEGQ AX
MOVQ AX, err+40(FP)
RET
ok2:
MOVQ AX, r1+32(FP)
MOVQ $0, err+40(FP)
RET

View File

@ -0,0 +1,26 @@
// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
#include "textflag.h"
// func RawVforkSyscall(trap, a1, a2, a3 uintptr) (r1, err uintptr)
TEXT ·RawVforkSyscall(SB),NOSPLIT|NOFRAME,$0-24
MOVW trap+0(FP), R7 // syscall entry
MOVW a1+4(FP), R0
MOVW a2+8(FP), R1
MOVW a3+12(FP), R2
SWI $0
MOVW $0xfffff001, R1
CMP R1, R0
BLS ok
MOVW $-1, R1
MOVW R1, r1+16(FP)
RSB $0, R0, R0
MOVW R0, err+20(FP)
RET
ok:
MOVW R0, r1+16(FP)
MOVW $0, R0
MOVW R0, err+20(FP)
RET

View File

@ -0,0 +1,27 @@
// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
#include "textflag.h"
// func RawVforkSyscall(trap, a1, a2, a3 uintptr) (r1, err uintptr)
TEXT ·RawVforkSyscall(SB),NOSPLIT,$0-48
MOVD a1+8(FP), R0
MOVD a2+16(FP), R1
MOVD a3+24(FP), R2
MOVD $0, R3
MOVD $0, R4
MOVD $0, R5
MOVD trap+0(FP), R8 // syscall entry
SVC
CMN $4095, R0
BCC ok
MOVD $-1, R4
MOVD R4, r1+32(FP) // r1
NEG R0, R0
MOVD R0, err+40(FP) // errno
RET
ok:
MOVD R0, r1+32(FP) // r1
MOVD ZR, err+40(FP) // errno
RET

View File

@ -0,0 +1,27 @@
// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
#include "textflag.h"
// func RawVforkSyscall(trap, a1, a2, a3 uintptr) (r1, err uintptr)
TEXT ·RawVforkSyscall(SB),NOSPLIT,$0-48
MOVV a1+8(FP), R4
MOVV a2+16(FP), R5
MOVV a3+24(FP), R6
MOVV $0, R7
MOVV $0, R8
MOVV $0, R9
MOVV trap+0(FP), R11 // syscall entry
SYSCALL
MOVW $-4096, R12
BGEU R12, R4, ok
MOVV $-1, R12
MOVV R12, r1+32(FP) // r1
SUBVU R4, R0, R4
MOVV R4, err+40(FP) // errno
RET
ok:
MOVV R4, r1+32(FP) // r1
MOVV R0, err+40(FP) // errno
RET

View File

@ -0,0 +1,27 @@
// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//go:build linux && (mips64 || mips64le)
#include "textflag.h"
// func RawVforkSyscall(trap, a1, a2, a3 uintptr) (r1, err uintptr)
TEXT ·RawVforkSyscall(SB),NOSPLIT|NOFRAME,$0-48
MOVV a1+8(FP), R4
MOVV a2+16(FP), R5
MOVV a3+24(FP), R6
MOVV R0, R7
MOVV R0, R8
MOVV R0, R9
MOVV trap+0(FP), R2 // syscall entry
SYSCALL
BEQ R7, ok
MOVV $-1, R1
MOVV R1, r1+32(FP) // r1
MOVV R2, err+40(FP) // errno
RET
ok:
MOVV R2, r1+32(FP) // r1
MOVV R0, err+40(FP) // errno
RET

View File

@ -0,0 +1,24 @@
// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//go:build linux && (mips || mipsle)
#include "textflag.h"
// func RawVforkSyscall(trap, a1, a2, a3 uintptr) (r1, err uintptr)
TEXT ·RawVforkSyscall(SB),NOSPLIT|NOFRAME,$0-24
MOVW a1+4(FP), R4
MOVW a2+8(FP), R5
MOVW a3+12(FP), R6
MOVW trap+0(FP), R2 // syscall entry
SYSCALL
BEQ R7, ok
MOVW $-1, R1
MOVW R1, r1+16(FP) // r1
MOVW R2, err+20(FP) // errno
RET
ok:
MOVW R2, r1+16(FP) // r1
MOVW R0, err+20(FP) // errno
RET

View File

@ -0,0 +1,27 @@
// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//go:build linux && (ppc64 || ppc64le)
#include "textflag.h"
// func RawVforkSyscall(trap, a1, a2, a3 uintptr) (r1, err uintptr)
TEXT ·RawVforkSyscall(SB),NOSPLIT|NOFRAME,$0-48
MOVD a1+8(FP), R3
MOVD a2+16(FP), R4
MOVD a3+24(FP), R5
MOVD R0, R6
MOVD R0, R7
MOVD R0, R8
MOVD trap+0(FP), R9 // syscall entry
SYSCALL R9
BVC ok
MOVD $-1, R4
MOVD R4, r1+32(FP) // r1
MOVD R3, err+40(FP) // errno
RET
ok:
MOVD R3, r1+32(FP) // r1
MOVD R0, err+40(FP) // errno
RET

View File

@ -0,0 +1,27 @@
// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
#include "textflag.h"
// func RawVforkSyscall(trap, a1, a2, a3 uintptr) (r1, err uintptr)
TEXT ·RawVforkSyscall(SB),NOSPLIT|NOFRAME,$0-48
MOV a1+8(FP), A0
MOV a2+16(FP), A1
MOV a3+24(FP), A2
MOV ZERO, A3
MOV ZERO, A4
MOV ZERO, A5
MOV trap+0(FP), A7 // syscall entry
ECALL
MOV $-4096, T0
BLTU T0, A0, err
MOV A0, r1+32(FP) // r1
MOV ZERO, err+40(FP) // errno
RET
err:
MOV $-1, T0
MOV T0, r1+32(FP) // r1
SUB A0, ZERO, A0
MOV A0, err+40(FP) // errno
RET

View File

@ -0,0 +1,26 @@
// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
#include "textflag.h"
// func RawVforkSyscall(trap, a1, a2, a3 uintptr) (r1, err uintptr)
TEXT ·RawVforkSyscall(SB),NOSPLIT|NOFRAME,$0-48
MOVD a1+8(FP), R2
MOVD a2+16(FP), R3
MOVD a3+24(FP), R4
MOVD $0, R5
MOVD $0, R6
MOVD $0, R7
MOVD trap+0(FP), R1 // syscall entry
SYSCALL
MOVD $0xfffffffffffff001, R8
CMPUBLT R2, R8, ok2
MOVD $-1, r1+32(FP)
NEG R2, R2
MOVD R2, err+40(FP) // errno
RET
ok2:
MOVD R2, r1+32(FP)
MOVD $0, err+40(FP) // errno
RET

View File

@ -0,0 +1,12 @@
// Package vfork provides the mirror of the un-exported syscall.rawVforkSyscall.
// The assembly code is copied from go1.24 syscall package
package vfork
import "syscall"
// RawVforkSyscall provided the mirrored version from un-exported syscall.rawVforkSyscall
// The go:linkname does not work for assembly function and it was suggested by the go team
// to copy over the assembly functions
//
// See go.dev/issue/71892
func RawVforkSyscall(trap, a1, a2, a3 uintptr) (r1 uintptr, err syscall.Errno)

View File

@ -0,0 +1,31 @@
package forkexec
import (
"syscall"
"unsafe"
)
// SandboxInit calls sandbox_init
func SandboxInit(profile *byte, flags uint64, errorBuf **byte) (err error) {
var r1 uintptr
r1, _, err = syscall3(libc_sandbox_init_trampoline_addr, uintptr(unsafe.Pointer(profile)), uintptr(flags), uintptr(unsafe.Pointer(errorBuf)))
if r1 != 0 {
err = syscall.EINVAL
} else {
err = nil
}
return
}
// SandboxFreeError calls sandbox_free_error
func SandboxFreeError(errorBuf *byte) {
syscall3(libc_sandbox_free_error_trampoline_addr, uintptr(unsafe.Pointer(errorBuf)), 0, 0)
}
var libc_sandbox_init_trampoline_addr uintptr
//go:cgo_import_dynamic libc_sandbox_init sandbox_init "/usr/lib/libSystem.B.dylib"
var libc_sandbox_free_error_trampoline_addr uintptr
//go:cgo_import_dynamic libc_sandbox_free_error sandbox_free_error "/usr/lib/libSystem.B.dylib"

View File

@ -0,0 +1,79 @@
#include "textflag.h"
TEXT libc_sandbox_init_trampoline<>(SB),NOSPLIT,$0-0
JMP libc_sandbox_init(SB)
GLOBL ·libc_sandbox_init_trampoline_addr(SB), RODATA, $8
DATA ·libc_sandbox_init_trampoline_addr(SB)/8, $libc_sandbox_init_trampoline<>(SB)
TEXT libc_sandbox_free_error_trampoline<>(SB),NOSPLIT,$0-0
JMP libc_sandbox_free_error(SB)
GLOBL ·libc_sandbox_free_error_trampoline_addr(SB), RODATA, $8
DATA ·libc_sandbox_free_error_trampoline_addr(SB)/8, $libc_sandbox_free_error_trampoline<>(SB)
TEXT libc_fork_trampoline<>(SB),NOSPLIT,$0-0
JMP libc_fork(SB)
GLOBL ·libc_fork_trampoline_addr(SB), RODATA, $8
DATA ·libc_fork_trampoline_addr(SB)/8, $libc_fork_trampoline<>(SB)
TEXT libc_close_trampoline<>(SB),NOSPLIT,$0-0
JMP libc_close(SB)
GLOBL ·libc_close_trampoline_addr(SB), RODATA, $8
DATA ·libc_close_trampoline_addr(SB)/8, $libc_close_trampoline<>(SB)
TEXT libc_read_trampoline<>(SB),NOSPLIT,$0-0
JMP libc_read(SB)
GLOBL ·libc_read_trampoline_addr(SB), RODATA, $8
DATA ·libc_read_trampoline_addr(SB)/8, $libc_read_trampoline<>(SB)
TEXT libc_write_trampoline<>(SB),NOSPLIT,$0-0
JMP libc_write(SB)
GLOBL ·libc_write_trampoline_addr(SB), RODATA, $8
DATA ·libc_write_trampoline_addr(SB)/8, $libc_write_trampoline<>(SB)
TEXT libc_fcntl_trampoline<>(SB),NOSPLIT,$0-0
JMP libc_fcntl(SB)
GLOBL ·libc_fcntl_trampoline_addr(SB), RODATA, $8
DATA ·libc_fcntl_trampoline_addr(SB)/8, $libc_fcntl_trampoline<>(SB)
TEXT libc_dup2_trampoline<>(SB),NOSPLIT,$0-0
JMP libc_dup2(SB)
GLOBL ·libc_dup2_trampoline_addr(SB), RODATA, $8
DATA ·libc_dup2_trampoline_addr(SB)/8, $libc_dup2_trampoline<>(SB)
TEXT libc_chdir_trampoline<>(SB),NOSPLIT,$0-0
JMP libc_chdir(SB)
GLOBL ·libc_chdir_trampoline_addr(SB), RODATA, $8
DATA ·libc_chdir_trampoline_addr(SB)/8, $libc_chdir_trampoline<>(SB)
TEXT libc_setrlimit_trampoline<>(SB),NOSPLIT,$0-0
JMP libc_setrlimit(SB)
GLOBL ·libc_setrlimit_trampoline_addr(SB), RODATA, $8
DATA ·libc_setrlimit_trampoline_addr(SB)/8, $libc_setrlimit_trampoline<>(SB)
TEXT libc_execve_trampoline<>(SB),NOSPLIT,$0-0
JMP libc_execve(SB)
GLOBL ·libc_execve_trampoline_addr(SB), RODATA, $8
DATA ·libc_execve_trampoline_addr(SB)/8, $libc_execve_trampoline<>(SB)
TEXT libc_exit_trampoline<>(SB),NOSPLIT,$0-0
JMP libc_exit(SB)
GLOBL ·libc_exit_trampoline_addr(SB), RODATA, $8
DATA ·libc_exit_trampoline_addr(SB)/8, $libc_exit_trampoline<>(SB)
TEXT libc_setpgid_trampoline<>(SB),NOSPLIT,$0-0
JMP libc_setpgid(SB)
GLOBL ·libc_setpgid_trampoline_addr(SB), RODATA, $8
DATA ·libc_setpgid_trampoline_addr(SB)/8, $libc_setpgid_trampoline<>(SB)

View File

@ -1,5 +1,3 @@
// +build linux
package memfd
import (
@ -17,12 +15,12 @@ const roSeal = unix.F_SEAL_SEAL | unix.F_SEAL_SHRINK | unix.F_SEAL_GROW | unix.F
func New(name string) (*os.File, error) {
fd, err := unix.MemfdCreate(name, createFlag)
if err != nil {
return nil, fmt.Errorf("memfd: memfd_create failed %v", err)
return nil, fmt.Errorf("memfd: memfd_create: %w", err)
}
file := os.NewFile(uintptr(fd), name)
if file == nil {
unix.Close(fd)
return nil, fmt.Errorf("memfd: NewFile failed for %v", name)
return nil, fmt.Errorf("memfd: new file failed for %q", name)
}
return file, nil
}
@ -31,21 +29,21 @@ func New(name string) (*os.File, error) {
func DupToMemfd(name string, reader io.Reader) (*os.File, error) {
file, err := New(name)
if err != nil {
return nil, fmt.Errorf("DupToMemfd: %v", err)
return nil, fmt.Errorf("memfd: dup: %w", err)
}
// linux syscall sendfile might be more efficient here if reader is a file
if _, err = io.Copy(file, reader); err != nil {
if _, err = file.ReadFrom(reader); err != nil {
file.Close()
return nil, fmt.Errorf("DupToMemfd: io.Copy %v", err)
return nil, fmt.Errorf("memfd: read from: %w", err)
}
// make memfd readonly
if _, err = unix.FcntlInt(file.Fd(), unix.F_ADD_SEALS, roSeal); err != nil {
file.Close()
return nil, fmt.Errorf("DupToMemfd: memfd seal %v", err)
return nil, fmt.Errorf("memfd: seal: %w", err)
}
if _, err := file.Seek(0, 0); err != nil {
file.Close()
return nil, fmt.Errorf("DupToMemfd: file seek %v", err)
return nil, fmt.Errorf("memfd: seek: %w", err)
}
return file, nil
}

View File

@ -0,0 +1,80 @@
package memfd
import (
"bytes"
"io"
"os"
"testing"
)
func TestNew(t *testing.T) {
f, err := New("test-memfd")
if err != nil {
t.Fatalf("New() error: %v", err)
}
defer f.Close()
// Write and read to verify it's a valid file
data := []byte("hello world")
n, err := f.Write(data)
if err != nil {
t.Fatalf("Write error: %v", err)
}
if n != len(data) {
t.Errorf("Write n = %d, want %d", n, len(data))
}
_, err = f.Seek(0, io.SeekStart)
if err != nil {
t.Fatalf("Seek error: %v", err)
}
read := make([]byte, len(data))
n, err = f.Read(read)
if err != nil && err != io.EOF {
t.Fatalf("Read error: %v", err)
}
if string(read[:n]) != string(data) {
t.Errorf("Read = %q, want %q", string(read[:n]), string(data))
}
}
func TestDupToMemfd(t *testing.T) {
content := []byte("memfd content")
r := bytes.NewReader(content)
f, err := DupToMemfd("dup-memfd", r)
if err != nil {
t.Fatalf("DupToMemfd error: %v", err)
}
defer f.Close()
// Should be sealed (readonly), so writing should fail
_, err = f.Write([]byte("fail"))
if err == nil {
t.Error("expected write to sealed memfd to fail, but it succeeded")
}
// Should be able to read the content
_, err = f.Seek(0, io.SeekStart)
if err != nil {
t.Fatalf("Seek error: %v", err)
}
got, err := io.ReadAll(f)
if err != nil {
t.Fatalf("ReadAll error: %v", err)
}
if string(got) != string(content) {
t.Errorf("ReadAll = %q, want %q", string(got), string(content))
}
}
func TestDupToMemfd_ErrorPropagation(t *testing.T) {
// Pass a reader that always errors
r := errorReader{}
_, err := DupToMemfd("dup-memfd-err", r)
if err == nil {
t.Error("expected error from DupToMemfd, got nil")
}
}
type errorReader struct{}
func (errorReader) Read([]byte) (int, error) { return 0, os.ErrInvalid }

View File

@ -1,4 +1,4 @@
// +build !linux
//go:build !linux
package memfd
@ -9,7 +9,7 @@ import (
"runtime"
)
var errNotImplemented = fmt.Errorf("memfd: unsupported on platform %s", runtime.GOOS)
var errNotImplemented = fmt.Errorf("memfd: unsupported on platform: %s", runtime.GOOS)
func New(name string) (*os.File, error) {
return nil, errNotImplemented

View File

@ -8,7 +8,7 @@ import (
)
const (
bind = unix.MS_BIND | unix.MS_NOSUID | unix.MS_PRIVATE
bind = unix.MS_BIND | unix.MS_NOSUID | unix.MS_PRIVATE | unix.MS_REC
mFlag = unix.MS_NOSUID | unix.MS_NOATIME | unix.MS_NODEV
)
@ -22,16 +22,12 @@ func NewDefaultBuilder() *Builder {
}
// Build creates sequence of syscalls for fork_exec
// skipNotExists skips bind mounts that source not exists
func (b *Builder) Build(skipNotExists bool) ([]SyscallParams, error) {
func (b *Builder) Build() ([]SyscallParams, error) {
var err error
ret := make([]SyscallParams, 0, len(b.Mounts))
for _, m := range b.Mounts {
var mknod bool
if mknod, err = isBindMountFileOrNotExists(m); err != nil {
if skipNotExists {
continue
}
return nil, err
}
sp, err := m.ToSyscall()
@ -44,8 +40,23 @@ func (b *Builder) Build(skipNotExists bool) ([]SyscallParams, error) {
return ret, nil
}
// FilterNotExist removes bind mount that does not exists
func (b *Builder) FilterNotExist() *Builder {
rt := b.Mounts[:0]
for _, m := range b.Mounts {
if m.IsBindMount() {
if _, err := os.Stat(m.Source); os.IsNotExist(err) {
continue
}
}
rt = append(rt, m)
}
b.Mounts = rt
return b
}
func isBindMountFileOrNotExists(m Mount) (bool, error) {
if m.Flags&unix.MS_BIND == unix.MS_BIND {
if m.IsBindMount() {
if fi, err := os.Stat(m.Source); os.IsNotExist(err) {
return false, err
} else if !fi.IsDir() {
@ -55,13 +66,13 @@ func isBindMountFileOrNotExists(m Mount) (bool, error) {
return false, nil
}
// WithMounts add mounts to builder
// WithMounts adds mounts to builder
func (b *Builder) WithMounts(m []Mount) *Builder {
b.Mounts = append(b.Mounts, m...)
return b
}
// WithMount add single mount to builder
// WithMount adds single mount to builder
func (b *Builder) WithMount(m Mount) *Builder {
b.Mounts = append(b.Mounts, m)
return b
@ -81,7 +92,7 @@ func (b *Builder) WithBind(source, target string, readonly bool) *Builder {
return b
}
// WithTmpfs add a tmpfs mount to builder
// WithTmpfs adds a tmpfs mount to builder
func (b *Builder) WithTmpfs(target, data string) *Builder {
b.Mounts = append(b.Mounts, Mount{
Source: "tmpfs",
@ -93,13 +104,22 @@ func (b *Builder) WithTmpfs(target, data string) *Builder {
return b
}
// WithProc add proc file system
// WithProc adds proc file system mounted read-only
func (b *Builder) WithProc() *Builder {
return b.WithProcRW(false)
}
// WithProcRW adds proc file system, possibly read-write
func (b *Builder) WithProcRW(canWrite bool) *Builder {
var flags uintptr = unix.MS_NOSUID | unix.MS_NODEV | unix.MS_NOEXEC
if !canWrite {
flags |= unix.MS_RDONLY
}
b.Mounts = append(b.Mounts, Mount{
Source: "proc",
Target: "proc",
FsType: "proc",
Flags: unix.MS_NOSUID | unix.MS_RDONLY,
Flags: flags,
})
return b
}

View File

@ -0,0 +1,124 @@
package mount
import (
"os"
"strings"
"testing"
)
func TestBuilder_WithBind(t *testing.T) {
b := NewBuilder().WithBind("/src", "/dst", true)
if len(b.Mounts) != 1 {
t.Fatalf("expected 1 mount, got %d", len(b.Mounts))
}
m := b.Mounts[0]
if m.Source != "/src" || m.Target != "/dst" {
t.Errorf("unexpected mount: %+v", m)
}
if !m.IsBindMount() {
t.Errorf("expected bind mount")
}
if !m.IsReadOnly() {
t.Errorf("expected readonly mount")
}
}
func TestBuilder_WithTmpfs(t *testing.T) {
b := NewBuilder().WithTmpfs("/tmp", "size=64m")
if len(b.Mounts) != 1 {
t.Fatalf("expected 1 mount, got %d", len(b.Mounts))
}
m := b.Mounts[0]
if !m.IsTmpFs() {
t.Errorf("expected tmpfs mount")
}
if m.Target != "/tmp" || m.Data != "size=64m" {
t.Errorf("unexpected mount: %+v", m)
}
}
func TestBuilder_WithProc(t *testing.T) {
b := NewBuilder().WithProc()
if len(b.Mounts) != 1 {
t.Fatalf("expected 1 mount, got %d", len(b.Mounts))
}
m := b.Mounts[0]
if m.FsType != "proc" {
t.Errorf("expected proc fsType")
}
if !m.IsReadOnly() {
t.Errorf("expected readonly proc mount")
}
}
func TestBuilder_WithProcRW(t *testing.T) {
b := NewBuilder().WithProcRW(true)
if len(b.Mounts) != 1 {
t.Fatalf("expected 1 mount, got %d", len(b.Mounts))
}
m := b.Mounts[0]
if m.FsType != "proc" {
t.Errorf("expected proc fsType")
}
if m.IsReadOnly() {
t.Errorf("expected read-write proc mount")
}
}
func TestBuilder_WithMounts(t *testing.T) {
m1 := Mount{Source: "/a", Target: "/b"}
m2 := Mount{Source: "/c", Target: "/d"}
b := NewBuilder().WithMounts([]Mount{m1, m2})
if len(b.Mounts) != 2 {
t.Fatalf("expected 2 mounts, got %d", len(b.Mounts))
}
}
func TestBuilder_WithMount(t *testing.T) {
m := Mount{Source: "/a", Target: "/b"}
b := NewBuilder().WithMount(m)
if len(b.Mounts) != 1 {
t.Fatalf("expected 1 mount, got %d", len(b.Mounts))
}
}
func TestBuilder_String(t *testing.T) {
b := NewBuilder().
WithBind("/src", "/dst", false).
WithTmpfs("/tmp", "size=1m").
WithProc()
s := b.String()
if !strings.HasPrefix(s, "Mounts: ") {
t.Errorf("unexpected prefix: %q", s)
}
if !strings.Contains(s, "bind[/src:/dst:rw]") {
t.Errorf("missing bind: %q", s)
}
if !strings.Contains(s, "tmpfs[/tmp]") {
t.Errorf("missing tmpfs: %q", s)
}
if !strings.Contains(s, "proc[ro]") {
t.Errorf("missing proc: %q", s)
}
}
func TestBuilder_FilterNotExist(t *testing.T) {
tmpDir := t.TempDir()
tmpFilePath := tmpDir + "/mounttest"
f, err := os.Create(tmpFilePath)
if err != nil {
t.Fatal(err)
}
f.Close()
b := NewBuilder().
WithBind(f.Name(), "/dst1", false).
WithBind("/not/exist", "/dst2", false)
b.FilterNotExist()
if len(b.Mounts) != 1 {
t.Errorf("expected 1 mount after filter, got %d", len(b.Mounts))
}
if b.Mounts[0].Source != f.Name() {
t.Errorf("unexpected mount: %+v", b.Mounts[0])
}
}

View File

@ -3,21 +3,69 @@ package mount
import (
"fmt"
"os"
"path/filepath"
"syscall"
)
// Mount calls mount syscall
func (m *Mount) Mount() error {
if err := os.MkdirAll(m.Target, 0755); err != nil {
return err
if err := ensureMountTargetExists(m.Source, m.Target); err != nil {
return fmt.Errorf("mkdir: %w", err)
}
if err := syscall.Mount(m.Source, m.Target, m.FsType, m.Flags, m.Data); err != nil {
return err
return fmt.Errorf("mount: %w", err)
}
// Read-only bind mount need to be remounted
const bindRo = syscall.MS_BIND | syscall.MS_RDONLY
const mask = syscall.MS_NOSUID | syscall.MS_NODEV | syscall.MS_NOEXEC | syscall.MS_NOATIME | syscall.MS_NODIRATIME | syscall.MS_RELATIME
if m.Flags&bindRo == bindRo {
if err := syscall.Mount("", m.Target, m.FsType, m.Flags|syscall.MS_REMOUNT, m.Data); err != nil {
// Ensure the flag retains for bind mount
var s syscall.Statfs_t
if err := syscall.Statfs(m.Source, &s); err != nil {
return fmt.Errorf("statfs: %w", err)
}
flag := m.Flags | syscall.MS_REMOUNT | uintptr(s.Flags&mask)
if err := syscall.Mount("", m.Target, m.FsType, flag, m.Data); err != nil {
return fmt.Errorf("remount: %w", err)
}
}
return nil
}
// IsBindMount returns if it is a bind mount
func (m Mount) IsBindMount() bool {
return m.Flags&syscall.MS_BIND == syscall.MS_BIND
}
// IsReadOnly returns if it is a readonly mount
func (m Mount) IsReadOnly() bool {
return m.Flags&syscall.MS_RDONLY == syscall.MS_RDONLY
}
// IsTmpFs returns if the fsType is tmpfs
func (m Mount) IsTmpFs() bool {
return m.FsType == "tmpfs"
}
func ensureMountTargetExists(source, target string) error {
isFile := false
if fi, err := os.Stat(source); err == nil {
isFile = !fi.IsDir()
}
dir := target
if isFile {
dir = filepath.Dir(target)
}
if err := os.MkdirAll(dir, 0755); err != nil {
return err
}
if isFile {
if err := syscall.Mknod(target, 0755, 0); err != nil {
// double check if file exists
f, err1 := os.Lstat(target)
if err1 == nil && f.Mode().IsRegular() {
return nil
}
return err
}
}
@ -25,19 +73,19 @@ func (m *Mount) Mount() error {
}
func (m Mount) String() string {
switch {
case m.Flags&syscall.MS_BIND == syscall.MS_BIND:
flag := "rw"
if m.Flags&syscall.MS_RDONLY == syscall.MS_RDONLY {
flag = "ro"
}
switch {
case m.Flags&syscall.MS_BIND == syscall.MS_BIND:
return fmt.Sprintf("bind[%s:%s:%s]", m.Source, m.Target, flag)
case m.FsType == "tmpfs":
return fmt.Sprintf("tmpfs[%s]", m.Target)
case m.FsType == "proc":
return fmt.Sprintf("proc[]")
return fmt.Sprintf("proc[%s]", flag)
default:
return fmt.Sprintf("mount[%s,%s:%s:%x,%s]", m.FsType, m.Source, m.Target, m.Flags, m.Data)

View File

@ -0,0 +1,112 @@
package mount
import (
"os"
"path/filepath"
"syscall"
"testing"
)
func TestMount_IsBindMount(t *testing.T) {
m := Mount{Flags: syscall.MS_BIND}
if !m.IsBindMount() {
t.Errorf("expected IsBindMount true")
}
m.Flags = 0
if m.IsBindMount() {
t.Errorf("expected IsBindMount false")
}
}
func TestMount_IsReadOnly(t *testing.T) {
m := Mount{Flags: syscall.MS_RDONLY}
if !m.IsReadOnly() {
t.Errorf("expected IsReadOnly true")
}
m.Flags = 0
if m.IsReadOnly() {
t.Errorf("expected IsReadOnly false")
}
}
func TestMount_IsTmpFs(t *testing.T) {
m := Mount{FsType: "tmpfs"}
if !m.IsTmpFs() {
t.Errorf("expected IsTmpFs true")
}
m.FsType = "other"
if m.IsTmpFs() {
t.Errorf("expected IsTmpFs false")
}
}
func TestMount_String(t *testing.T) {
tests := []struct {
m Mount
want string
}{
{
m: Mount{Source: "/src", Target: "/dst", Flags: syscall.MS_BIND, FsType: "", Data: ""},
want: "bind[/src:/dst:rw]",
},
{
m: Mount{Source: "/src", Target: "/dst", Flags: syscall.MS_BIND | syscall.MS_RDONLY, FsType: "", Data: ""},
want: "bind[/src:/dst:ro]",
},
{
m: Mount{Source: "", Target: "/tmp", FsType: "tmpfs"},
want: "tmpfs[/tmp]",
},
{
m: Mount{Source: "", Target: "proc", FsType: "proc", Flags: syscall.MS_RDONLY},
want: "proc[ro]",
},
{
m: Mount{Source: "src", Target: "dst", FsType: "other", Flags: 0, Data: "data"},
want: "mount[other,src:dst:0,data]",
},
}
for _, tt := range tests {
got := tt.m.String()
if got != tt.want {
t.Errorf("Mount.String() = %q, want %q", got, tt.want)
}
}
}
func TestEnsureMountTargetExists_Dir(t *testing.T) {
tmpDir := t.TempDir()
target := filepath.Join(tmpDir, "foo/bar")
err := ensureMountTargetExists(tmpDir, target)
if err != nil {
t.Fatalf("ensureMountTargetExists error: %v", err)
}
info, err := os.Stat(target)
if err != nil {
t.Fatalf("stat error: %v", err)
}
if !info.IsDir() {
t.Errorf("expected directory at %s", target)
}
}
func TestEnsureMountTargetExists_File(t *testing.T) {
tmpDir := t.TempDir()
srcFile := filepath.Join(tmpDir, "srcfile")
if err := os.WriteFile(srcFile, []byte("x"), 0644); err != nil {
t.Fatalf("write srcfile: %v", err)
}
target := filepath.Join(tmpDir, "targetfile")
err := ensureMountTargetExists(srcFile, target)
if err != nil {
t.Fatalf("ensureMountTargetExists error: %v", err)
}
// Should be a file or at least exist
info, err := os.Lstat(target)
if err != nil {
t.Fatalf("lstat error: %v", err)
}
if info.IsDir() {
t.Errorf("expected file at %s, got directory", target)
}
}

View File

@ -13,9 +13,9 @@ import (
// at most max bytes to a buffer
type Buffer struct {
W *os.File
Max int64
Buffer *bytes.Buffer
Done <-chan struct{}
Max int64
}
// NewPipe create a pipe with a goroutine to copy its read-end to writer
@ -28,9 +28,11 @@ func NewPipe(writer io.Writer, n int64) (<-chan struct{}, *os.File, error) {
}
done := make(chan struct{})
go func() {
defer close(done)
defer r.Close()
io.CopyN(writer, r, int64(n))
close(done)
// ensure no blocking / SIGPIPE on the other end
io.Copy(io.Discard, r)
r.Close()
}()
return done, w, nil
}

101
pkg/pipe/buffer_test.go Normal file
View File

@ -0,0 +1,101 @@
package pipe
import (
"io"
"strings"
"testing"
"time"
)
func TestNewBuffer_WriteAndRead(t *testing.T) {
const max = 10
buf, err := NewBuffer(max)
if err != nil {
t.Fatalf("NewBuffer error: %v", err)
}
defer buf.W.Close()
// Write less than max bytes
input := "hello"
n, err := buf.W.Write([]byte(input))
if err != nil {
t.Fatalf("Write error: %v", err)
}
if n != len(input) {
t.Errorf("Write bytes = %d, want %d", n, len(input))
}
buf.W.Close()
<-buf.Done
got := buf.Buffer.String()
if got != input {
t.Errorf("Buffer content = %q, want %q", got, input)
}
}
func TestNewBuffer_MaxBytes(t *testing.T) {
const max = 5
buf, err := NewBuffer(max)
if err != nil {
t.Fatalf("NewBuffer error: %v", err)
}
defer buf.W.Close()
// Write more than max bytes
input := "toolonginput"
_, err = io.Copy(buf.W, strings.NewReader(input))
if err != nil {
t.Fatalf("Copy error: %v", err)
}
buf.W.Close()
<-buf.Done
got := buf.Buffer.String()
if len(got) != int(max+1) {
t.Errorf("Buffer length = %d, want %d", len(got), max+1)
}
if got != input[:max+1] {
t.Errorf("Buffer content = %q, want %q", got, input[:max+1])
}
}
func TestBuffer_String(t *testing.T) {
const max = 8
buf, err := NewBuffer(max)
if err != nil {
t.Fatalf("NewBuffer error: %v", err)
}
defer buf.W.Close()
_, _ = buf.W.Write([]byte("abc"))
buf.W.Close()
<-buf.Done
want := "Buffer[3/8]"
if buf.String() != want {
t.Errorf("String() = %q, want %q", buf.String(), want)
}
}
func TestNewBuffer_DoneCloses(t *testing.T) {
const max = 4
buf, err := NewBuffer(max)
if err != nil {
t.Fatalf("NewBuffer error: %v", err)
}
defer buf.W.Close()
done := make(chan struct{})
go func() {
_, _ = buf.W.Write([]byte("test"))
buf.W.Close()
close(done)
}()
select {
case <-buf.Done:
// ok
case <-time.After(1 * time.Second):
t.Fatal("timeout waiting for Done channel")
}
}

View File

@ -17,6 +17,8 @@ type RLimits struct {
FileSize uint64 // in bytes
Stack uint64 // in bytes
AddressSpace uint64 // in bytes
OpenFile uint64 // count
DisableCore bool // set core to 0
}
// RLimit is the resource limits defined by Linux setrlimit
@ -70,15 +72,28 @@ func (r *RLimits) PrepareRLimit() []RLimit {
Rlim: getRlimit(r.AddressSpace, r.AddressSpace),
})
}
if r.OpenFile > 0 {
ret = append(ret, RLimit{
Res: syscall.RLIMIT_NOFILE,
Rlim: getRlimit(r.OpenFile, r.OpenFile),
})
}
if r.DisableCore {
ret = append(ret, RLimit{
Res: syscall.RLIMIT_CORE,
Rlim: getRlimit(0, 0),
})
}
return ret
}
func (r RLimit) String() string {
if r.Res == syscall.RLIMIT_CPU {
return fmt.Sprintf("CPU[%d s:%d s]", r.Rlim.Cur, r.Rlim.Max)
}
t := ""
switch r.Res {
case syscall.RLIMIT_CPU:
return fmt.Sprintf("CPU[%d s:%d s]", r.Rlim.Cur, r.Rlim.Max)
case syscall.RLIMIT_NOFILE:
return fmt.Sprintf("OpenFile[%d:%d]", r.Rlim.Cur, r.Rlim.Max)
case syscall.RLIMIT_DATA:
t = "Data"
case syscall.RLIMIT_FSIZE:
@ -87,6 +102,8 @@ func (r RLimit) String() string {
t = "Stack"
case syscall.RLIMIT_AS:
t = "AddressSpace"
case syscall.RLIMIT_CORE:
t = "Core"
}
return fmt.Sprintf("%s[%v:%v]", t, runner.Size(r.Rlim.Cur), runner.Size(r.Rlim.Max))
}

136
pkg/rlimit/rlimit_test.go Normal file
View File

@ -0,0 +1,136 @@
//go:build linux
package rlimit
import (
"syscall"
"testing"
)
func TestPrepareRLimit(t *testing.T) {
tests := []struct {
name string
rl RLimits
expect []int
}{
{
name: "Empty",
rl: RLimits{},
expect: []int{},
},
{
name: "CPU only",
rl: RLimits{CPU: 1},
expect: []int{syscall.RLIMIT_CPU},
},
{
name: "Data only",
rl: RLimits{Data: 1024},
expect: []int{syscall.RLIMIT_DATA},
},
{
name: "All fields",
rl: RLimits{CPU: 1, CPUHard: 2, Data: 1024, FileSize: 2048, Stack: 4096, AddressSpace: 8192, OpenFile: 16, DisableCore: true},
expect: []int{syscall.RLIMIT_CPU, syscall.RLIMIT_DATA, syscall.RLIMIT_FSIZE, syscall.RLIMIT_STACK, syscall.RLIMIT_AS, syscall.RLIMIT_NOFILE, syscall.RLIMIT_CORE},
},
{
name: "DisableCore only",
rl: RLimits{DisableCore: true},
expect: []int{syscall.RLIMIT_CORE},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
rls := tt.rl.PrepareRLimit()
if len(rls) != len(tt.expect) {
t.Fatalf("expected %d rlimits, got %d", len(tt.expect), len(rls))
}
for i, r := range rls {
if r.Res != tt.expect[i] {
t.Errorf("expected Res %d at %d, got %d", tt.expect[i], i, r.Res)
}
}
})
}
}
func TestRLimitString(t *testing.T) {
tests := []struct {
name string
rl RLimit
want string
}{
{
name: "CPU",
rl: RLimit{Res: syscall.RLIMIT_CPU, Rlim: syscall.Rlimit{Cur: 1, Max: 2}},
want: "CPU[1 s:2 s]",
},
{
name: "NOFILE",
rl: RLimit{Res: syscall.RLIMIT_NOFILE, Rlim: syscall.Rlimit{Cur: 10, Max: 20}},
want: "OpenFile[10:20]",
},
{
name: "DATA",
rl: RLimit{Res: syscall.RLIMIT_DATA, Rlim: syscall.Rlimit{Cur: 1024, Max: 2048}},
want: "Data[1.0 KiB:2.0 KiB]",
},
{
name: "FSIZE",
rl: RLimit{Res: syscall.RLIMIT_FSIZE, Rlim: syscall.Rlimit{Cur: 100, Max: 200}},
want: "File[100 B:200 B]",
},
{
name: "STACK",
rl: RLimit{Res: syscall.RLIMIT_STACK, Rlim: syscall.Rlimit{Cur: 4096, Max: 8192}},
want: "Stack[4.0 KiB:8.0 KiB]",
},
{
name: "AS",
rl: RLimit{Res: syscall.RLIMIT_AS, Rlim: syscall.Rlimit{Cur: 123, Max: 456}},
want: "AddressSpace[123 B:456 B]",
},
{
name: "CORE",
rl: RLimit{Res: syscall.RLIMIT_CORE, Rlim: syscall.Rlimit{Cur: 0, Max: 0}},
want: "Core[0 B:0 B]",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := tt.rl.String()
if got != tt.want {
t.Errorf("got %q, want %q", got, tt.want)
}
})
}
}
func TestRLimitsString(t *testing.T) {
rl := RLimits{
CPU: 1,
CPUHard: 2,
Data: 1024,
FileSize: 2048,
Stack: 4096,
AddressSpace: 8192,
OpenFile: 16,
DisableCore: true,
}
want := "RLimits[CPU[1 s:2 s],Data[1.0 KiB:1.0 KiB],File[2.0 KiB:2.0 KiB],Stack[4.0 KiB:4.0 KiB],AddressSpace[8.0 KiB:8.0 KiB],OpenFile[16:16],Core[0 B:0 B]]"
got := rl.String()
if got != want {
t.Errorf("got %q, want %q", got, want)
}
}
func TestRLimitsString_Empty(t *testing.T) {
rl := RLimits{}
want := "RLimits[]"
got := rl.String()
if got != want {
t.Errorf("got %q, want %q", got, want)
}
}

View File

@ -1,5 +0,0 @@
// Package seccomp provides a generated filter format for seccomp filter
package seccomp
// Filter is the BPF seccomp filter value
type Filter []byte

View File

@ -1,15 +1,16 @@
// Package seccomp provides a generated filter format for seccomp filter
package seccomp
import (
"syscall"
"unsafe"
)
import "syscall"
// Filter is the BPF seccomp filter value
type Filter []syscall.SockFilter
// SockFprog converts Filter to SockFprog for seccomp syscall
func (f Filter) SockFprog() *syscall.SockFprog {
b := []byte(f)
b := []syscall.SockFilter(f)
return &syscall.SockFprog{
Len: uint16(len(b) / 8),
Filter: (*syscall.SockFilter)(unsafe.Pointer(&b[0])),
Len: uint16(len(b)),
Filter: &b[0],
}
}

View File

@ -1,4 +1,4 @@
package seccomp
package libseccomp
// Action is seccomp trap action
type Action uint32
@ -19,16 +19,6 @@ const (
MsgHandle
)
// WithReturnCode set the return code when action is trace or ban
func (a Action) WithReturnCode(code int16) Action {
return a.Action() | Action(code)<<16
}
// ReturnCode get the return code
func (a Action) ReturnCode() int16 {
return int16(a >> 16)
}
// Action get the basic action
func (a Action) Action() Action {
return Action(a & 0xffff)

View File

@ -1,23 +1,24 @@
package libseccomp
import (
"github.com/criyle/go-sandbox/pkg/seccomp"
libseccomp "github.com/seccomp/libseccomp-golang"
libseccomp "github.com/elastic/go-seccomp-bpf"
)
// ToSeccompAction convert action to libseccomp compatible action
func ToSeccompAction(a seccomp.Action) libseccomp.ScmpAction {
var action libseccomp.ScmpAction
func ToSeccompAction(a Action) libseccomp.Action {
var action libseccomp.Action
switch a.Action() {
case seccomp.ActionAllow:
action = libseccomp.ActAllow
case seccomp.ActionErrno:
action = libseccomp.ActErrno
case seccomp.ActionTrace:
action = libseccomp.ActTrace
case ActionAllow:
action = libseccomp.ActionAllow
case ActionErrno:
action = libseccomp.ActionErrno
case ActionTrace:
action = libseccomp.ActionTrace
default:
action = libseccomp.ActKill
action = libseccomp.ActionKillProcess
}
action = action.SetReturnCode(a.ReturnCode())
// the least 16 bit of ret value is SECCOMP_RET_DATA
// although it might not officially supported by go-seccomp-bpf
// action = action.WithReturnData(int(a.ReturnCode()))
return action
}

View File

@ -1,75 +1,61 @@
package libseccomp
import (
"io/ioutil"
"os"
"syscall"
"github.com/criyle/go-sandbox/pkg/seccomp"
libseccomp "github.com/seccomp/libseccomp-golang"
libseccomp "github.com/elastic/go-seccomp-bpf"
"golang.org/x/net/bpf"
)
// Builder is used to build the filter
type Builder struct {
Allow, Trace []string
Default seccomp.Action
Default Action
}
var actTrace = libseccomp.ActTrace.SetReturnCode(seccomp.MsgHandle)
var actTrace = libseccomp.ActionTrace
// Build builds the filter
func (b *Builder) Build() (seccomp.Filter, error) {
filter, err := libseccomp.NewFilter(ToSeccompAction(b.Default))
policy := libseccomp.Policy{
DefaultAction: ToSeccompAction(b.Default),
Syscalls: []libseccomp.SyscallGroup{
{
Action: libseccomp.ActionAllow,
Names: b.Allow,
},
{
Action: actTrace,
Names: b.Trace,
},
},
}
program, err := policy.Assemble()
if err != nil {
return nil, err
}
defer filter.Release()
if err = addFilterActions(filter, b.Allow, libseccomp.ActAllow); err != nil {
return nil, err
}
if err = addFilterActions(filter, b.Trace, actTrace); err != nil {
return nil, err
}
return ExportBPF(filter)
return ExportBPF(program)
}
// ExportBPF convert libseccomp filter to kernel readable BPF content
func ExportBPF(filter *libseccomp.ScmpFilter) (seccomp.Filter, error) {
r, w, err := os.Pipe()
func ExportBPF(filter []bpf.Instruction) (seccomp.Filter, error) {
raw, err := bpf.Assemble(filter)
if err != nil {
return nil, err
}
defer r.Close()
// export BPF to pipe
go func() {
filter.ExportBPF(w)
w.Close()
}()
// get BPF binary
bin, err := ioutil.ReadAll(r)
if err != nil {
return nil, err
}
return seccomp.Filter(bin), nil
return sockFilter(raw), nil
}
func addFilterActions(filter *libseccomp.ScmpFilter, names []string, action libseccomp.ScmpAction) error {
for _, s := range names {
if err := addFilterAction(filter, s, action); err != nil {
return err
func sockFilter(raw []bpf.RawInstruction) []syscall.SockFilter {
filter := make([]syscall.SockFilter, 0, len(raw))
for _, instruction := range raw {
filter = append(filter, syscall.SockFilter{
Code: instruction.Op,
Jt: instruction.Jt,
Jf: instruction.Jf,
K: instruction.K,
})
}
}
return nil
}
func addFilterAction(filter *libseccomp.ScmpFilter, name string, action libseccomp.ScmpAction) error {
syscallID, err := libseccomp.GetSyscallFromName(name)
if err != nil {
return err
}
if err = filter.AddRule(syscallID, action); err != nil {
return err
}
return nil
return filter
}

View File

@ -4,7 +4,6 @@ import (
"testing"
"github.com/criyle/go-sandbox/pkg/seccomp"
libseccomp "github.com/seccomp/libseccomp-golang"
)
var (
@ -22,8 +21,7 @@ var (
)
func TestBuildFilter(t *testing.T) {
defaultAction := libseccomp.ActKill
_, err := buildFilterMock(defaultAction)
_, err := buildFilterMock()
if err != nil {
t.Error("BuildFilter failed")
}
@ -35,17 +33,17 @@ func BenchmarkBuildDefaultFilter(b *testing.B) {
builder := Builder{
Allow: defaultSyscallAllows,
Trace: defaultSyscallTraces,
Default: seccomp.ActionTrace,
Default: ActionTrace,
}
builder.Build()
}
}
func buildFilterMock(d libseccomp.ScmpAction) (seccomp.Filter, error) {
func buildFilterMock() (seccomp.Filter, error) {
b := Builder{
Allow: []string{"fork"},
Trace: []string{"execve"},
Default: seccomp.ActionTrace,
Default: ActionTrace,
}
return b.Build()
}

View File

@ -1,10 +1,21 @@
package libseccomp
import (
libseccomp "github.com/seccomp/libseccomp-golang"
"fmt"
"github.com/elastic/go-seccomp-bpf/arch"
)
var info, errInfo = arch.GetInfo("")
// ToSyscallName convert syscallno to syscall name
func ToSyscallName(sysno uint) (string, error) {
return libseccomp.ScmpSyscall(sysno).GetName()
if errInfo != nil {
return "", errInfo
}
n, ok := info.SyscallNumbers[int(sysno)]
if !ok {
return "", fmt.Errorf("syscall number does not exist: %d", sysno)
}
return n, nil
}

View File

@ -0,0 +1,98 @@
package unixsocket
import "testing"
func BenchmarkBaseline(b *testing.B) {
s, t, err := NewSocketPair()
if err != nil {
b.Fatal(err)
}
m := make([]byte, 1024)
b.ResetTimer()
go func() {
msg := []byte("message")
for i := 0; i < b.N; i++ {
s.SendMsg(msg, Msg{})
}
}()
for i := 0; i < b.N; i++ {
t.RecvMsg(m)
}
}
func BenchmarkGoroutine(b *testing.B) {
s, t, err := NewSocketPair()
if err != nil {
b.Fatal(err)
}
m := make([]byte, 1024)
b.ResetTimer()
go func() {
msg := []byte("message")
for i := 0; i < b.N; i++ {
s.SendMsg(msg, Msg{})
}
}()
for i := 0; i < b.N; i++ {
c := make(chan struct{})
go func() {
defer close(c)
t.RecvMsg(m)
}()
<-c
}
}
func BenchmarkChannel(b *testing.B) {
c := make(chan []byte)
benchGoroutine(b, c)
}
func BenchmarkChannelBuffed(b *testing.B) {
c := make(chan []byte, 1)
benchGoroutine(b, c)
}
func BenchmarkChannelBuffed4(b *testing.B) {
c := make(chan []byte, 4)
benchGoroutine(b, c)
}
func BenchmarkEmptyGoroutine(b *testing.B) {
for i := 0; i < b.N; i++ {
c := make(chan struct{})
go func() {
close(c)
}()
<-c
}
}
func benchGoroutine(b *testing.B, c chan []byte) {
s, t, err := NewSocketPair()
if err != nil {
b.Fatal(err)
}
go func() {
msg := []byte("message")
for i := 0; i < b.N; i++ {
s.SendMsg(msg, Msg{})
}
}()
b.ResetTimer()
go func() {
m := make([]byte, 1024)
for i := 0; i < b.N; i++ {
t.RecvMsg(m)
c <- m
}
}()
for i := 0; i < b.N; i++ {
<-c
}
}

View File

@ -7,22 +7,18 @@ import (
"fmt"
"net"
"os"
"sync"
"syscall"
)
// oob size default to page size
const oobSize = 4096
// use pool to minimize allocate gabage collector overhead
var oobPool = sync.Pool{
New: func() interface{} {
return make([]byte, oobSize)
},
}
const oobSize = 4 << 10 // 4kb
// Socket wrappers a unix socket connection
type Socket net.UnixConn
type Socket struct {
*net.UnixConn
sendBuff []byte
recvBuff []byte
}
// Msg is the oob msg with the message
type Msg struct {
@ -30,51 +26,61 @@ type Msg struct {
Cred *syscall.Ucred // unix credential
}
func newSocket(conn *net.UnixConn) *Socket {
return &Socket{
UnixConn: conn,
sendBuff: make([]byte, oobSize),
recvBuff: make([]byte, oobSize),
}
}
// NewSocket creates Socket conn struct using existing unix socket fd
// creates by socketpair or net.DialUnix and mark it as close_on_exec (avoid fd leak)
// it need SOCK_SEQPACKET socket for reliable transfer
// it will need SO_PASSCRED to pass unix credential, Notice: in the documentation,
// if cred is not specified, self information will be sent
func NewSocket(fd int) (*Socket, error) {
syscall.SetNonblock(fd, true)
syscall.CloseOnExec(fd)
file := os.NewFile(uintptr(fd), "unix-socket")
if file == nil {
return nil, fmt.Errorf("NewSocket: %d is not a valid fd", fd)
return nil, fmt.Errorf("new socket: %d is not a valid fd", fd)
}
defer file.Close()
syscall.CloseOnExec(int(file.Fd()))
conn, err := net.FileConn(file)
if err != nil {
return nil, err
return nil, fmt.Errorf("new socket: fileconn: %w", err)
}
unixConn, ok := conn.(*net.UnixConn)
if !ok {
conn.Close()
return nil, fmt.Errorf("NewSocket: %d is not a valid unix socket connection", fd)
return nil, fmt.Errorf("new socket: %d is not a valid unix socket connection", fd)
}
return (*Socket)(unixConn), nil
return newSocket(unixConn), nil
}
// NewSocketPair creates connected unix socketpair using SOCK_SEQPACKET
func NewSocketPair() (*Socket, *Socket, error) {
fd, err := syscall.Socketpair(syscall.AF_LOCAL, syscall.SOCK_SEQPACKET|syscall.SOCK_CLOEXEC, 0)
if err != nil {
return nil, nil, fmt.Errorf("NewSocketPair: failed to call socketpair %v", err)
return nil, nil, fmt.Errorf("new socket pair: socketpair: %w", err)
}
ins, err := NewSocket(fd[0])
if err != nil {
syscall.Close(fd[0])
syscall.Close(fd[1])
return nil, nil, fmt.Errorf("NewSocketPair: failed to call NewSocket on sender %v", err)
return nil, nil, fmt.Errorf("new socket pair: sender: %w", err)
}
outs, err := NewSocket(fd[1])
if err != nil {
ins.Close()
syscall.Close(fd[1])
return nil, nil, fmt.Errorf("NewSocketPair: failed to call NewSocket receiver %v", err)
return nil, nil, fmt.Errorf("new socket pair: receiver: %w", err)
}
return ins, outs, nil
@ -82,7 +88,7 @@ func NewSocketPair() (*Socket, *Socket, error) {
// SetPassCred set sockopt for pass cred for unix socket
func (s *Socket) SetPassCred(option int) error {
sysconn, err := (*net.UnixConn)(s).SyscallConn()
sysconn, err := s.SyscallConn()
if err != nil {
return err
}
@ -92,21 +98,16 @@ func (s *Socket) SetPassCred(option int) error {
}
// SendMsg sendmsg to unix socket and encode possible unix right / credential
func (s *Socket) SendMsg(b []byte, m *Msg) error {
buf := oobPool.Get().([]byte)
defer oobPool.Put(buf)
oob := bytes.NewBuffer(buf[:0])
if m != nil {
func (s *Socket) SendMsg(b []byte, m Msg) error {
oob := bytes.NewBuffer(s.sendBuff[:0])
if len(m.Fds) > 0 {
oob.Write(syscall.UnixRights(m.Fds...))
}
if m.Cred != nil {
oob.Write(syscall.UnixCredentials(m.Cred))
}
}
_, _, err := (*net.UnixConn)(s).WriteMsgUnix(b, oob.Bytes(), nil)
_, _, err := s.WriteMsgUnix(b, oob.Bytes(), nil)
if err != nil {
return err
}
@ -114,28 +115,33 @@ func (s *Socket) SendMsg(b []byte, m *Msg) error {
}
// RecvMsg recvmsg from unix socket and parse possible unix right / credential
func (s *Socket) RecvMsg(b []byte) (int, *Msg, error) {
oob := oobPool.Get().([]byte)
defer oobPool.Put(oob)
n, oobn, _, _, err := (*net.UnixConn)(s).ReadMsgUnix(b, oob)
func (s *Socket) RecvMsg(b []byte) (int, Msg, error) {
var msg Msg
n, oobn, _, _, err := s.ReadMsgUnix(b, s.recvBuff)
if err != nil {
return 0, nil, err
return 0, msg, err
}
// parse oob msg
msgs, err := syscall.ParseSocketControlMessage(oob[:oobn])
msgs, err := syscall.ParseSocketControlMessage(s.recvBuff[:oobn])
if err != nil {
return 0, nil, err
return 0, msg, err
}
msg, err := parseMsg(msgs)
msg, err = parseMsg(msgs)
if err != nil {
return 0, nil, err
return 0, msg, err
}
return n, msg, nil
}
func parseMsg(msgs []syscall.SocketControlMessage) (*Msg, error) {
var msg Msg
func parseMsg(msgs []syscall.SocketControlMessage) (msg Msg, err error) {
defer func() {
if err != nil {
for _, f := range msg.Fds {
syscall.Close(f)
}
msg.Fds = nil
}
}()
for _, m := range msgs {
if m.Header.Level != syscall.SOL_SOCKET {
continue
@ -145,17 +151,17 @@ func parseMsg(msgs []syscall.SocketControlMessage) (*Msg, error) {
case syscall.SCM_CREDENTIALS:
cred, err := syscall.ParseUnixCredentials(&m)
if err != nil {
return nil, err
return msg, err
}
msg.Cred = cred
case syscall.SCM_RIGHTS:
fds, err := syscall.ParseUnixRights(&m)
if err != nil {
return nil, err
return msg, err
}
msg.Fds = fds
}
}
return &msg, nil
return msg, nil
}

View File

@ -0,0 +1,142 @@
package unixsocket
import (
"bytes"
"os"
"syscall"
"testing"
)
func TestBaseline(t *testing.T) {
a, b, err := NewSocketPair()
if err != nil {
t.Fatal(err)
}
m := make([]byte, 1024)
go func() {
msg := []byte("message")
a.SendMsg(msg, Msg{})
}()
n, _, err := b.RecvMsg(m)
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(m[:n], []byte("message")) {
t.Fatal("not equal")
}
}
func TestSendRecvMsg_Fds(t *testing.T) {
a, b, err := NewSocketPair()
if err != nil {
t.Fatal(err)
}
defer a.Close()
defer b.Close()
// Create a file to send its fd
tmpfile, err := os.CreateTemp("", "unixsocket-fd")
if err != nil {
t.Fatal(err)
}
defer os.Remove(tmpfile.Name())
defer tmpfile.Close()
msg := []byte("fdtest")
go func() {
a.SendMsg(msg, Msg{Fds: []int{int(tmpfile.Fd())}})
}()
buf := make([]byte, 64)
n, m, err := b.RecvMsg(buf)
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(buf[:n], msg) {
t.Errorf("RecvMsg got %q, want %q", buf[:n], msg)
}
if len(m.Fds) != 1 {
t.Errorf("expected 1 fd, got %d", len(m.Fds))
}
if m.Fds != nil {
syscall.Close(m.Fds[0])
}
}
func TestSendRecvMsg_Cred(t *testing.T) {
if os.Geteuid() != 0 {
t.Skip("skipping credential test: requires root privileges")
return
}
a, b, err := NewSocketPair()
if err != nil {
t.Fatal(err)
}
defer a.Close()
defer b.Close()
// Enable credential passing
if err := a.SetPassCred(1); err != nil {
t.Fatal(err)
}
if err := b.SetPassCred(1); err != nil {
t.Fatal(err)
}
msg := []byte("credtest")
go func() {
a.SendMsg(msg, Msg{Cred: &syscall.Ucred{Pid: 123, Uid: 456, Gid: 789}})
}()
buf := make([]byte, 64)
n, m, err := b.RecvMsg(buf)
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(buf[:n], msg) {
t.Errorf("RecvMsg got %q, want %q", buf[:n], msg)
}
if m.Cred == nil {
t.Error("expected credential, got nil")
}
}
func TestNewSocketPair_Close(t *testing.T) {
a, b, err := NewSocketPair()
if err != nil {
t.Fatal(err)
}
if err := a.Close(); err != nil {
t.Errorf("a.Close() error: %v", err)
}
if err := b.Close(); err != nil {
t.Errorf("b.Close() error: %v", err)
}
}
func TestNewSocket_InvalidFd(t *testing.T) {
// Use an invalid fd
_, err := NewSocket(-1)
if err == nil {
t.Error("expected error for invalid fd, got nil")
}
}
func TestSetPassCred_InvalidSocket(t *testing.T) {
a, b, err := NewSocketPair()
if err != nil {
t.Fatal(err)
}
defer a.Close()
defer b.Close()
// Close the socket to make it invalid
a.Close()
err = a.SetPassCred(1)
if err == nil {
t.Error("expected error on SetPassCred for closed socket, got nil")
}
}

View File

@ -36,30 +36,37 @@ func getIovecs(base *byte, l int) []unix.Iovec {
}
func vmReadStr(pid int, addr uintptr, buff []byte) error {
// Deal with unaligned addr
n := 0
r := pageSize - int(addr%uintptr(pageSize))
if r == 0 {
r = pageSize
// Handle unaligned address: calculate remaining bytes to page boundary
totalRead := 0 // Total bytes read so far
// Calculate distance to next page boundary, nextRead is the number of bytes to read
nextRead := pageSize - int(addr%uintptr(pageSize))
if nextRead == 0 {
nextRead = pageSize // If exactly at page boundary, use full page size
}
// Read in a loop until buffer is full or termination condition is met
for len(buff) > 0 {
if l := len(buff); r < l {
r = l
// If remaining buffer is smaller than planned read size, reduce read size
if restToRead := len(buff); restToRead < nextRead {
nextRead = restToRead
}
nn, err := vmRead(pid, addr+uintptr(n), buff[:r])
// Read data from current position
curRead, err := vmRead(pid, addr+uintptr(totalRead), buff[:nextRead])
if err != nil {
return err
return err // Read error
}
if curRead == 0 {
break // No more data to read
}
if hasNull(buff[:curRead]) {
break // Found string terminator
}
if hasNull(buff[:nn]) {
return nil
}
n += nn
buff = buff[nn:]
r = pageSize
// Update counters and buffer
totalRead += curRead // Update total bytes read
buff = buff[curRead:] // Move buffer pointer
nextRead = pageSize // Reset to full page size
}
return nil
}

View File

@ -0,0 +1,250 @@
package ptracer
import (
"bytes"
"fmt"
"os"
"os/exec"
"testing"
)
// TestHasNull tests the hasNull function
func TestHasNull(t *testing.T) {
tests := []struct {
name string
data []byte
want bool
}{
{
name: "empty buffer",
data: []byte{},
want: false,
},
{
name: "no null",
data: []byte("hello"),
want: false,
},
{
name: "has null at start",
data: []byte{0, 1, 2, 3},
want: true,
},
{
name: "has null at end",
data: []byte{1, 2, 3, 0},
want: true,
},
{
name: "has null in middle",
data: []byte{1, 0, 3, 4},
want: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := hasNull(tt.data); got != tt.want {
t.Errorf("hasNull() = %v, want %v", got, tt.want)
}
})
}
}
// Helper function: creates a child process and returns its PID
func createTestProcess(t *testing.T) (int, func()) {
cmd := exec.Command("sleep", "10") // use sleep command to create a running process
if err := cmd.Start(); err != nil {
t.Fatalf("Failed to start test process: %v", err)
}
cleanup := func() {
cmd.Process.Kill()
cmd.Wait()
}
return cmd.Process.Pid, cleanup
}
// findReadableMemoryRegion finds a readable memory region in the process's address space
func findReadableMemoryRegion(t *testing.T, pid int, minSize int) uintptr {
maps, err := os.ReadFile(fmt.Sprintf("/proc/%d/maps", pid))
if err != nil {
t.Fatalf("Failed to read process maps: %v", err)
}
for _, line := range bytes.Split(maps, []byte{'\n'}) {
if len(line) == 0 {
continue
}
if bytes.Contains(line, []byte("r-x")) {
// Parse address range: start-end
var start, end uint64
_, err := fmt.Sscanf(string(line), "%x-%x", &start, &end)
if err != nil {
continue
}
// Calculate region size
size := end - start
if size >= uint64(minSize) {
return uintptr(start)
}
}
}
t.Fatalf("Failed to find readable memory region with minimum size %d", minSize)
return 0
}
// TestVmRead tests the vmRead function
func TestVmRead(t *testing.T) {
pid, cleanup := createTestProcess(t)
defer cleanup()
buff := make([]byte, 100)
// Ensure the found memory region is large enough
baseAddr := findReadableMemoryRegion(t, pid, len(buff))
n, err := vmRead(pid, baseAddr, buff)
if err != nil {
t.Errorf("vmRead() error = %v", err)
}
if n == 0 {
t.Error("vmRead returned 0 bytes")
}
}
// vmReadStrTestCase defines a test case for vmReadStr
type vmReadStrTestCase struct {
name string
buffSize int
addrAlign uintptr // address alignment, used to test different alignment scenarios
wantErr bool
}
// getVmReadStrTestCases returns test cases for vmReadStr testing
func getVmReadStrTestCases() []vmReadStrTestCase {
return []vmReadStrTestCase{
{
name: "small_buffer_aligned",
buffSize: 10,
addrAlign: 0,
wantErr: false,
},
{
name: "small_buffer_unaligned",
buffSize: 10,
addrAlign: 1,
wantErr: false,
},
{
name: "exact_page_size",
buffSize: pageSize,
addrAlign: 0,
wantErr: false,
},
{
name: "cross_page_boundary",
buffSize: pageSize + 100,
addrAlign: uintptr(pageSize - 50),
wantErr: false,
},
{
name: "large_buffer_unaligned",
buffSize: pageSize * 2,
addrAlign: 123,
wantErr: false,
},
{
name: "buffer_smaller_than_to_boundary",
buffSize: 10,
addrAlign: uintptr(pageSize - 100), // distance to page boundary is 100 bytes, but buffer is only 10 bytes
wantErr: false,
},
}
}
// TestVmReadStr tests the vmReadStr function
func TestVmReadStr(t *testing.T) {
pid, cleanup := createTestProcess(t)
defer cleanup()
testCases := getVmReadStrTestCases()
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
buff := make([]byte, tc.buffSize)
// Ensure the found memory region is large enough
baseAddr := findReadableMemoryRegion(t, pid, tc.buffSize)
// Use test case specified alignment offset
testAddr := baseAddr + tc.addrAlign
// Record buffer content before reading
originalBuff := make([]byte, len(buff))
copy(originalBuff, buff)
err := vmReadStr(pid, testAddr, buff)
if (err != nil) != tc.wantErr {
t.Errorf("vmReadStr() error = %v, wantErr %v", err, tc.wantErr)
}
// Verify reading results
if !bytes.Equal(buff, originalBuff) {
// For vmReadStr, we only care that some data was read
// and that the function completed without error
t.Logf("Data was read successfully for case: %s", tc.name)
}
// Special case: check buffer size smaller than distance to boundary
if tc.name == "buffer_smaller_than_to_boundary" {
distToBoundary := pageSize - int(testAddr%uintptr(pageSize))
if distToBoundary > len(buff) {
t.Logf("Verified buffer handling when smaller than distance to boundary: dist=%d, buff=%d",
distToBoundary, len(buff))
}
}
})
}
}
// TestSliceBehavior tests slice behavior
func TestSliceBehavior(t *testing.T) {
tests := []struct {
name string
buffSize int
nextRead int
expected int
}{
{
name: "small_buffer_large_read",
buffSize: 10,
nextRead: 4096,
expected: 10, // must be limited to buffer size
},
{
name: "large_buffer_small_read",
buffSize: 8192,
nextRead: 4096,
expected: 4096, // can use full read amount
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
buff := make([]byte, tt.buffSize)
// safely calculate actual read amount
actualRead := tt.nextRead
if tt.buffSize < actualRead {
actualRead = tt.buffSize
}
slice := buff[:actualRead]
if len(slice) != tt.expected {
t.Errorf("Expected slice len %d, got %d", tt.expected, len(slice))
}
})
}
}

View File

@ -1,4 +1,4 @@
// +build !linux
//go:build !linux
package ptracer

Some files were not shown because too many files have changed in this diff Show More