From f5dff9ca57c389859523b5e5ae8b94da19c99e46 Mon Sep 17 00:00:00 2001 From: criyle Date: Fri, 25 Dec 2020 17:25:09 -0800 Subject: [PATCH] Refactor package structure & debug outputs --- README.md | 185 ++++++++---------- cmd/cinit/init_linux.go | 7 +- cmd/executorserver/config/config.go | 2 +- cmd/executorserver/grpc.go | 16 +- cmd/executorserver/main.go | 4 +- cmd/executorserver/model/model.go | 17 +- cmd/ffi/cinterface.go | 2 +- env/doc.go | 8 +- env/env_darwin.go | 4 +- env/env_linux.go | 9 +- env/env_windows.go | 4 +- .../linuxcontainer}/cgroup_fake_pool_linux.go | 2 +- .../linuxcontainer}/cgroup_wrapper_linux.go | 4 +- .../linuxcontainer}/cgrouppool_linux.go | 4 +- .../linuxcontainer}/envbuilder_linux.go | 7 +- .../linuxcontainer}/environment_linux.go | 4 +- .../linuxcontainer}/envprocess_linux.go | 4 +- .../linuxcontainer}/interface_linux.go | 2 +- {pkg => env}/macsandbox/builder_darwin.go | 2 +- {pkg => env}/macsandbox/environment_darwin.go | 4 +- {pkg => env}/macsandbox/process_darwin.go | 2 +- {pkg => env}/macsandbox/profile_darwin.go | 0 env/pool/doc.go | 2 + {pkg => env}/pool/interface.go | 2 +- {pkg => env}/winc/builder_windows.go | 2 +- {pkg => env}/winc/environment_windows.go | 4 +- {pkg => env}/winc/job_object_windows.go | 2 +- {pkg => env}/winc/mksyscall.go | 0 {pkg => env}/winc/process_windows.go | 2 +- {pkg => env}/winc/syscall_windows.go | 0 {pkg => env}/winc/token_windows.go | 0 {pkg => env}/winc/zsyscall_windows.go | 0 {pkg/envexec => envexec}/cmd.go | 0 {pkg/envexec => envexec}/const.go | 0 {pkg/envexec => envexec}/doc.go | 0 {pkg/envexec => envexec}/file_collect.go | 28 +-- {pkg/envexec => envexec}/file_copyin.go | 0 {pkg/envexec => envexec}/file_prepare.go | 0 {pkg/envexec => envexec}/file_util_linux.go | 0 {pkg/envexec => envexec}/file_util_others.go | 0 {pkg/envexec => envexec}/group.go | 0 {pkg/envexec => envexec}/interface.go | 0 {pkg/envexec => envexec}/run_single.go | 0 {pkg/envexec => envexec}/run_status.go | 0 {pkg/envexec => envexec}/single.go | 0 {pkg/envexec => envexec}/util.go | 0 go.mod | 6 +- go.sum | 14 +- pkg/pool/doc.go | 2 - worker/file.go | 2 +- worker/model.go | 48 ++++- worker/waiter.go | 2 +- worker/worker.go | 8 +- 53 files changed, 216 insertions(+), 201 deletions(-) rename {pkg/pool => env/linuxcontainer}/cgroup_fake_pool_linux.go (96%) rename {pkg/pool => env/linuxcontainer}/cgroup_wrapper_linux.go (96%) rename {pkg/pool => env/linuxcontainer}/cgrouppool_linux.go (96%) rename {pkg/pool => env/linuxcontainer}/envbuilder_linux.go (87%) rename {pkg/pool => env/linuxcontainer}/environment_linux.go (97%) rename {pkg/pool => env/linuxcontainer}/envprocess_linux.go (94%) rename {pkg/pool => env/linuxcontainer}/interface_linux.go (94%) rename {pkg => env}/macsandbox/builder_darwin.go (96%) rename {pkg => env}/macsandbox/environment_darwin.go (97%) rename {pkg => env}/macsandbox/process_darwin.go (90%) rename {pkg => env}/macsandbox/profile_darwin.go (100%) create mode 100644 env/pool/doc.go rename {pkg => env}/pool/interface.go (95%) rename {pkg => env}/winc/builder_windows.go (97%) rename {pkg => env}/winc/environment_windows.go (99%) rename {pkg => env}/winc/job_object_windows.go (98%) rename {pkg => env}/winc/mksyscall.go (100%) rename {pkg => env}/winc/process_windows.go (96%) rename {pkg => env}/winc/syscall_windows.go (100%) rename {pkg => env}/winc/token_windows.go (100%) rename {pkg => env}/winc/zsyscall_windows.go (100%) rename {pkg/envexec => envexec}/cmd.go (100%) rename {pkg/envexec => envexec}/const.go (100%) rename {pkg/envexec => envexec}/doc.go (100%) rename {pkg/envexec => envexec}/file_collect.go (83%) rename {pkg/envexec => envexec}/file_copyin.go (100%) rename {pkg/envexec => envexec}/file_prepare.go (100%) rename {pkg/envexec => envexec}/file_util_linux.go (100%) rename {pkg/envexec => envexec}/file_util_others.go (100%) rename {pkg/envexec => envexec}/group.go (100%) rename {pkg/envexec => envexec}/interface.go (100%) rename {pkg/envexec => envexec}/run_single.go (100%) rename {pkg/envexec => envexec}/run_status.go (100%) rename {pkg/envexec => envexec}/single.go (100%) rename {pkg/envexec => envexec}/util.go (100%) delete mode 100644 pkg/pool/doc.go diff --git a/README.md b/README.md index 3772a96..a84d3cf 100644 --- a/README.md +++ b/README.md @@ -1,38 +1,67 @@ # go-judge -[![GoDoc](https://godoc.org/github.com/criyle/go-judge?status.svg)](https://godoc.org/github.com/criyle/go-judge) [![Go Report Card](https://goreportcard.com/badge/github.com/criyle/go-judge)](https://goreportcard.com/report/github.com/criyle/go-judge) [![Release](https://img.shields.io/github/v/tag/criyle/go-judge)](https://github.com/criyle/go-judge/releases/latest) ![Build](https://github.com/criyle/go-judge/workflows/Build/badge.svg) +[![Go Reference](https://pkg.go.dev/badge/github.com/criyle/go-judge.svg)](https://pkg.go.dev/github.com/criyle/go-judge) [![Go Report Card](https://goreportcard.com/badge/github.com/criyle/go-judge)](https://goreportcard.com/report/github.com/criyle/go-judge) [![Release](https://img.shields.io/github/v/tag/criyle/go-judge)](https://github.com/criyle/go-judge/releases/latest) ![Build](https://github.com/criyle/go-judge/workflows/Build/badge.svg) ## Executor Service ### Architecture -#### Overall Architecture - ```text -+----------------------------------------------------------------------------------+ -| Transport Layer (HTTP / WebSocket / FFI / ...) | -+----------------------------------------------------------------------------------+ -| Executor Worker | -+-----------------------------------------------------------+----------------------+ -| EnvExec + Environment Pool + Environment Builder | File Store | -+--------------------+----------------+---------------------+--------+-------+-----+ -| Linux (go-sandbox) | Windows (winc) | macOS (app sandbox) | Memory | Local | ... | -+--------------------+----------------+---------------------+--------+-------+-----+ ++---------------------------------------------------------------------------+ +| Transport Layer (HTTP / WebSocket / FFI / ...) | ++---------------------------------------------------------------------------+ +| Executor Worker | ++-----------------------------------------------------------+---------------+ +| Environment Pool + EnvExec + Environment Builder | File Store | ++--------------------+----------------+---------------------+--------+------+ +| Linux (go-sandbox) | Windows (winc) | macOS (app sandbox) | Memory | Disk | ++--------------------+----------------+---------------------+--------+------+ ``` -A rest service to run program in restricted environment and it is basically a wrapper for `pkg/envexec` to run single / multiple programs. +### REST API -- /run POST execute program in the restricted environment +A REST service to run program in restricted environment and it is basically a wrapper for `envexec` to run single / multiple programs. + +- /run POST execute program in the restricted environment (examples below) - /file GET list all cached file - /file POST prepare a file in the executor service (in memory), returns fileId (can be referenced in /run parameter) - /file/:fileId GET downloads file from executor service (in memory), returns file content - /file/:fileId DELETE delete file specified by fileId - /ws WebSocket for /run -- /metrics prometheus metrics (specifies `METRICS=1` environment variable to enable metrics) -- /debug (specifies `DEBUG=1` environment variable to enable go runtime debug endpoint) -- /version gets build git version (e.g. `v0.6.4-1-g20d2815`) together with runtime information (go version, os, platform) +- /metrics prometheus metrics (specifies `ES_ENABLE_METRICS=1` environment variable to enable metrics) +- /debug (specifies `ES_ENABLE_DEBUG=1` environment variable to enable go runtime debug endpoint) +- /version gets build git version (e.g. `v0.9.4`) together with runtime information (go version, os, platform) -### Install & Run Developing Server +### Command Line Arguments + +- The default binding address for the executor server is `:5050`. Can be specified with `-http-addr` flag. +- By default gRPC endpoint is disabled, to enable gRPC endpoint, add `-enable-grpc` flag. +- The default binding address for the gRPC executor server is `:5051`. Can be specified with `-grpc-addr` flag. +- The default concurrency is `4`, Can be specified with `-parallelism` flag. +- The default file store is in memory, local cache can be specified with `-dir` flag. +- The default log level is debug, use `-silent` to disable logs or use `-release` to enable release logger (auto turn on if in docker). +- The default CGroup prefix is `executor_server`, Can be specified with `-cgroup-prefix` flag. +- `-auth-token` to add token-based authentication to REST / gRPC +- `-src-prefix` to restrict `src` copyIn path (need to be absolute path) +- `-time-limit-checker-interval` specifies time limit checker interval (default 100ms) (valid value: \[1ms, 1s\]) +- `-output-limit` specifies size limit of POSIX rlimit of output +- `-cpuset` specifies `cpuset.cpus` cgroup for each container +- `-container-cred-start` specifies container `setuid` / `setgid` credential start point (default: 10000) + - for example, by default container 0 will run with 10001 uid & gid and container 1 will run with 10002 uid & gid... +- `-enable-cpu-rate` enabled `cpu` cgroup to control cpu rate using cfs_quota & cfs_period control +- `-cpu-cfs-period` specifies cfs_period if cpu rate is enabled (default 100ms) (valid value: \[1ms, 1s\]) +- `-seccomp-conf` specifies `seecomp` filter setting to load when running program + - for example, by `strace -c prog` to get all `syscall` needed and restrict to that sub set + - however, the `syscall` count in one platform(e.g. x86_64) is not suitable for all platform, so this option is not recommended + - the program killed by seccomp filter will have status `Dangerous Syscall` +- By default, the GO debug endpoints are disabled, to enable, specifies `-enable-debug` +- By default, the prometheus metrics endpoints are disabled, to enable, specifies `-enable-metrics` + +### Environment Variables + +Environment variable will be override by command line arguments if they both present and all command line arguments have its correspond environment variable (e.g. `ES_HTTP_ADDR`). Run `executorserver --help` to see all the environment variable configurations. + +### Install & Run Install GO 1.13+ from [download](https://golang.org/dl/) @@ -51,41 +80,7 @@ Build by your own `docker build -t executorserver -f Dockerfile.exec .` The `executorserver` need root privilege to create `cgroup`. Either creates sub-directory `/sys/fs/cgroup/cpuacct/executor_server`, `/sys/fs/cgroup/memory/executor_server`, `/sys/fs/cgroup/pids/executor_server` and make execution user readable or use `sudo` to run it. -#### Command Line Arguments - -- The default binding address for the executor server is `:5050`. Can be specified with `-http-addr` flag. -- The default binding address for the gRPC executor server is `:5051`. Can be specified with `-grpc-addr` flag. (Notice: need to set `ES_ENABLE_GRPC=1` environment variable to enable GRPC endpoint) -- The default concurrency is `4`, Can be specified with `-parallelism` flag. -- The default file store is in memory, local cache can be specified with `-dir` flag. -- The default log level is debug, use `-silent` to disable logs or use `-release` to enable release logger (auto turn on if in docker). -- The default CGroup prefix is `executor_server`, Can be specified with `-cgroup-prefix` flag. -- `-auth-token` to add token-based authentication to REST / gRPC -- `-src-prefix` to restrict `src` copyIn path (need to be absolute path) -- `-time-limit-checker-interval` specifies time limit checker interval (default 100ms) (valid value: \[1ms, 1s\]) -- `-output-limit` specifies size limit of POSIX rlimit of output -- `-cpuset` specifies `cpuset.cpus` cgroup for each container -- `-container-cred-start` specifies container `setuid` / `setgid` credential start point (default: 10000) - - for example, by default container 0 will run with 10001 uid & gid and container 1 will run with 10002 uid & gid... -- `-enable-cpu-rate` enabled `cpu` cgroup to control cpu rate using cfs_quota & cfs_period control -- `-cpu-cfs-period` specifies cfs_period if cpu rate is enabled (default 100ms) (valid value: \[1ms, 1s\]) -- `-seccomp-conf` specifies `seecomp` filter setting to load when running program - - for example, by `strace -c prog` to get all `syscall` needed and restrict to that sub set - - however, the `syscall` count in one platform(e.g. x86_64) is not suitable for all platform, so this option is not recommended - - the program killed by seccomp filter will have status `Dangerous Syscall` - -#### Environment Variables - -Environment variable will be override by command line arguments if they both present. All command line arguments have its correspond environment variable. - -- The http binding address specifies as `ES_HTTP_ADDR=addr` -- The grpc binding address specifies as `ES_GRPC_ADDR=addr` -- The parallelism specifies as `ES_PARALLELISM=4` -- The token specifies as `ES_AUTH_TOKEN=token` -- `ES_ENABLE_GRPC=1` enables gRPC -- `ES_ENABLE_METRICS=1` enables metrics -- `ES_ENABLE_DEBUG=1` enables debug - -### Build Shared object +#### Build Shared object Build container init `cinit`: @@ -101,7 +96,7 @@ For example, in JavaScript, run with `ffi-napi` (seems node 14 is not supported Build `go build ./cmd/executorproxy` -Run `./executorproxy`, connect to gRPC endpoint and offers REST endpoint. +Run `./executorproxy`, connect to gRPC endpoint expose as a REST endpoint. ### Build Executor Shell @@ -111,30 +106,20 @@ Run `./executorshell`, connect to gRPC endpoint with interactive shell. ### Container Root Filesystem -- [x] necessary lib / exec / compiler / header readonly bind mounted from current file system: /lib /lib64 /bin /usr -- [x] work directory tmpfs mount: /w (work dir), /tmp (compiler temp files) +For linux platform, the default mounts points are bind mounting host's `/lib`, `/lib64`, `/usr`, `/bin`, `/etc/alternatives`, `/etc/fpc.cfg`, `/dev/null`, `/dev/urandom` and mounts tmpfs at `/w`, `/tmp` and creates `/proc`. -The following mounts point are examples that can be configured through config file later +To customize mount points, please look at example `mount.yaml` file. -- additional compiler scripts / exec readonly bind mounted: /c -- additional header readonly bind mounted: /i +### Packages -### Utilities - -- pkg/envexec: run single / group of programs in parallel within restricted environment and resource constraints -- pkg/pool: reference implementation for Cgroup & Environment Pool +- envexec: run single / group of programs in parallel within restricted environment and resource constraints +- env: reference implementation environments to inject into envexec ### Windows Support -Build `executorserver` by: - -`go build ./cmd/executorserver/` - -Build `executor_server.dll`: (need to install `gcc` as well) - -`go build -buildmode=c-shared -o executor_server.so ./cmd/ffi/` - -Run: `./executorserver` +- Build `executorserver` by: `go build ./cmd/executorserver/` +- Build `executor_server.dll`: (need to install `gcc` as well) `go build -buildmode=c-shared -o executor_server.so ./cmd/ffi/` +- Run: `./executorserver` #### Windows Security @@ -144,26 +129,19 @@ Run: `./executorserver` ### MacOS Support -Build `executorserver` by: - -`go build ./cmd/executorserver/` - -Build `executor_server.dylib`: (need to install `XCode`) - -`go build -buildmode=c-shared -o executor_server.dylib ./cmd/ffi/` - -Run: `./executorserver` +- Build `executorserver` by: `go build ./cmd/executorserver/` +- Build `executor_server.dylib`: (need to install `XCode`) `go build -buildmode=c-shared -o executor_server.dylib ./cmd/ffi/` +- Run: `./executorserver` #### MacOS Security -- `sandbox-init` profile deny network access and file read / write +- `sandbox-init` profile deny network access and file read / write and read / write to `/Users` directory ### Benchmark -By `wrk` with `t.lua`: +By `wrk` with `t.lua`: `wrk -s t.lua -c 1 -t 1 -d 30s --latency http://localhost:5050/run`. -- Tested single thread ~140-160 op/s macOS Docker Desktop & ~400-460 op/s Windows 10 WSL2. -- Tested multi thread ~1100-1200 op/s Windows 10 WSL2 +However, these results are not the real use cases since the running time depends on the actual program specifies in the request. Normally, the executor server consumes ~1ms more compare to running without sandbox. ```lua wrk.method = "POST" @@ -171,41 +149,32 @@ wrk.body = '{"cmd":[{"args":["/bin/cat","a.hs"],"env":["PATH=/usr/bin:/bin"]," wrk.headers["Content-Type"] = "application/json;charset=UTF-8" ``` -`wrk -s t.lua -c 1 -t 1 -d 30s --latency http://localhost:5050/run` +- Single thread ~400-460 op/s Windows 10 WSL2 +- Multi thread ~1100-1200 op/s Windows 10 WSL2 -e.g.: +Single thread: ```text Running 30s test @ http://localhost:5050/run 1 threads and 1 connections Thread Stats Avg Stdev Max +/- Stdev - Latency 2.17ms 446.05us 22.37ms 93.88% - Req/Sec 463.26 26.15 500.00 78.67% + Latency 2.23ms 287.79us 14.29ms 86.51% + Req/Sec 451.05 17.60 494.00 76.00% Latency Distribution - 50% 2.08ms - 75% 2.27ms + 50% 2.19ms + 75% 2.33ms 90% 2.50ms - 99% 3.13ms - 13846 requests in 30.01s, 3.65MB read -Requests/sec: 461.30 -Transfer/sec: 124.38KB + 99% 2.93ms + 13482 requests in 30.02s, 3.58MB read +Requests/sec: 449.15 +Transfer/sec: 121.98KB ``` -## TODO +### TODO -- [x] Github actions to auto build -- [x] Configure mounts using YAML config file -- [x] Investigate root-free running mechanism (no cgroup && not set uid / gid) -- [x] Investigate RLimit settings (cpu, data, fsize, stack, noFile) -- [x] Add WebSocket for job submission -- [x] Windows support -- [x] MacOS support -- [x] GRPC + protobuf support -- [x] Token-based authentication -- [x] Prometheus metrics support -- [x] Customize container workDir, hostName & domainName +- [ ] Github actions to auto build docker image -## API interface +### REST API Interface ```typescript interface LocalFile { diff --git a/cmd/cinit/init_linux.go b/cmd/cinit/init_linux.go index f852356..9e197ca 100644 --- a/cmd/cinit/init_linux.go +++ b/cmd/cinit/init_linux.go @@ -1,7 +1,12 @@ package main -import "github.com/criyle/go-sandbox/container" +import ( + "os" + + "github.com/criyle/go-sandbox/container" +) func main() { container.Init() + os.Exit(2) } diff --git a/cmd/executorserver/config/config.go b/cmd/executorserver/config/config.go index 2d1e012..cace174 100644 --- a/cmd/executorserver/config/config.go +++ b/cmd/executorserver/config/config.go @@ -4,7 +4,7 @@ import ( "os" "time" - "github.com/criyle/go-judge/pkg/envexec" + "github.com/criyle/go-judge/envexec" "github.com/koding/multiconfig" ) diff --git a/cmd/executorserver/grpc.go b/cmd/executorserver/grpc.go index 44a8278..ba9f94c 100644 --- a/cmd/executorserver/grpc.go +++ b/cmd/executorserver/grpc.go @@ -7,8 +7,10 @@ import ( "path/filepath" "strings" "sync" + "time" "github.com/creack/pty" + "github.com/criyle/go-judge/envexec" "github.com/criyle/go-judge/filestore" "github.com/criyle/go-judge/pb" "github.com/criyle/go-judge/worker" @@ -100,9 +102,9 @@ func convertPBResult(r worker.Result) *pb.Response_Result { Status: pb.Response_Result_StatusType(r.Status), ExitStatus: int32(r.ExitStatus), Error: r.Error, - Time: r.Time, - RunTime: r.RunTime, - Memory: r.Memory, + Time: uint64(r.Time), + RunTime: uint64(r.RunTime), + Memory: uint64(r.Memory), Files: r.Files, FileIDs: r.FileIDs, } @@ -172,10 +174,10 @@ func convertPBCmd(c *pb.Request_CmdType, srcPrefix string) (cm worker.Cmd, strea Args: c.GetArgs(), Env: c.GetEnv(), TTY: c.GetTty(), - CPULimit: c.GetCPULimit(), - RealCPULimit: c.GetRealCPULimit(), - MemoryLimit: c.GetMemoryLimit(), - StackLimit: c.GetStackLimit(), + CPULimit: time.Duration(c.GetCPULimit()), + RealCPULimit: time.Duration(c.GetRealCPULimit()), + MemoryLimit: envexec.Size(c.GetMemoryLimit()), + StackLimit: envexec.Size(c.GetStackLimit()), ProcLimit: c.GetProcLimit(), CPURateLimit: c.GetCPURateLimit(), CopyOut: c.GetCopyOut(), diff --git a/cmd/executorserver/main.go b/cmd/executorserver/main.go index d7c4240..eab2569 100644 --- a/cmd/executorserver/main.go +++ b/cmd/executorserver/main.go @@ -15,10 +15,10 @@ import ( "github.com/criyle/go-judge/cmd/executorserver/config" "github.com/criyle/go-judge/env" + "github.com/criyle/go-judge/env/pool" + "github.com/criyle/go-judge/envexec" "github.com/criyle/go-judge/filestore" "github.com/criyle/go-judge/pb" - "github.com/criyle/go-judge/pkg/envexec" - "github.com/criyle/go-judge/pkg/pool" "github.com/criyle/go-judge/worker" ginpprof "github.com/gin-contrib/pprof" ginzap "github.com/gin-contrib/zap" diff --git a/cmd/executorserver/model/model.go b/cmd/executorserver/model/model.go index dcfc6e1..44a5504 100644 --- a/cmd/executorserver/model/model.go +++ b/cmd/executorserver/model/model.go @@ -5,8 +5,9 @@ import ( "os" "path/filepath" "strings" + "time" - "github.com/criyle/go-judge/pkg/envexec" + "github.com/criyle/go-judge/envexec" "github.com/criyle/go-judge/worker" ) @@ -127,9 +128,9 @@ func convertResult(r worker.Result) Result { Status: Status(r.Status), ExitStatus: r.ExitStatus, Error: r.Error, - Time: r.Time, - RunTime: r.RunTime, - Memory: r.Memory, + Time: uint64(r.Time), + RunTime: uint64(r.RunTime), + Memory: uint64(r.Memory), FileIDs: r.FileIDs, } if r.Files != nil { @@ -160,10 +161,10 @@ func convertCmd(c Cmd, srcPrefix string) (worker.Cmd, error) { Env: c.Env, Files: make([]worker.CmdFile, 0, len(c.Files)), TTY: c.TTY, - CPULimit: c.CPULimit, - RealCPULimit: c.RealCPULimit, - MemoryLimit: c.MemoryLimit, - StackLimit: c.StackLimit, + CPULimit: time.Duration(c.CPULimit), + RealCPULimit: time.Duration(c.RealCPULimit), + MemoryLimit: envexec.Size(c.MemoryLimit), + StackLimit: envexec.Size(c.StackLimit), ProcLimit: c.ProcLimit, CPURateLimit: c.CPURateLimit, CopyOut: c.CopyOut, diff --git a/cmd/ffi/cinterface.go b/cmd/ffi/cinterface.go index c239b5c..8d61a1b 100644 --- a/cmd/ffi/cinterface.go +++ b/cmd/ffi/cinterface.go @@ -12,8 +12,8 @@ import ( "github.com/criyle/go-judge/cmd/executorserver/model" "github.com/criyle/go-judge/env" + "github.com/criyle/go-judge/env/pool" "github.com/criyle/go-judge/filestore" - "github.com/criyle/go-judge/pkg/pool" "github.com/criyle/go-judge/worker" ) diff --git a/env/doc.go b/env/doc.go index 6260e78..0c1131f 100644 --- a/env/doc.go +++ b/env/doc.go @@ -1,2 +1,8 @@ -// Package env provides a unified method to create environment for envexec +// Package env provides a unified method to create environment for envexec. +// +// For linux, the env creates container & cgroup sandbox. +// +// For windows, the env creates low mandatory level sandbox. +// +// For macOS, the env creates sandbox_init sandbox. package env diff --git a/env/env_darwin.go b/env/env_darwin.go index eef2ebf..d2d2348 100644 --- a/env/env_darwin.go +++ b/env/env_darwin.go @@ -1,8 +1,8 @@ package env import ( - "github.com/criyle/go-judge/pkg/macsandbox" - "github.com/criyle/go-judge/pkg/pool" + "github.com/criyle/go-judge/env/macsandbox" + "github.com/criyle/go-judge/env/pool" ) var defaultRead = []string{ diff --git a/env/env_linux.go b/env/env_linux.go index 574dc66..42751d1 100644 --- a/env/env_linux.go +++ b/env/env_linux.go @@ -7,7 +7,8 @@ import ( "sync/atomic" "syscall" - "github.com/criyle/go-judge/pkg/pool" + "github.com/criyle/go-judge/env/linuxcontainer" + "github.com/criyle/go-judge/env/pool" "github.com/criyle/go-sandbox/container" "github.com/criyle/go-sandbox/pkg/cgroup" "github.com/criyle/go-sandbox/pkg/forkexec" @@ -116,11 +117,11 @@ func NewBuilder(c Config) (pool.EnvBuilder, error) { cg.Destroy() } - var cgroupPool pool.CgroupPool + var cgroupPool linuxcontainer.CgroupPool if cgb != nil { - cgroupPool = pool.NewFakeCgroupPool(cgb, c.CPUCfsPeriod) + cgroupPool = linuxcontainer.NewFakeCgroupPool(cgb, c.CPUCfsPeriod) } - return pool.NewEnvBuilder(pool.Config{ + return linuxcontainer.NewEnvBuilder(linuxcontainer.Config{ Builder: b, CgroupPool: cgroupPool, WorkDir: workDir, diff --git a/env/env_windows.go b/env/env_windows.go index 70df7c0..1370e43 100644 --- a/env/env_windows.go +++ b/env/env_windows.go @@ -1,8 +1,8 @@ package env import ( - "github.com/criyle/go-judge/pkg/pool" - "github.com/criyle/go-judge/pkg/winc" + "github.com/criyle/go-judge/env/pool" + "github.com/criyle/go-judge/env/winc" ) // NewBuilder build a environment builder diff --git a/pkg/pool/cgroup_fake_pool_linux.go b/env/linuxcontainer/cgroup_fake_pool_linux.go similarity index 96% rename from pkg/pool/cgroup_fake_pool_linux.go rename to env/linuxcontainer/cgroup_fake_pool_linux.go index 24b1ad6..d11f5b3 100644 --- a/pkg/pool/cgroup_fake_pool_linux.go +++ b/env/linuxcontainer/cgroup_fake_pool_linux.go @@ -1,4 +1,4 @@ -package pool +package linuxcontainer import "time" diff --git a/pkg/pool/cgroup_wrapper_linux.go b/env/linuxcontainer/cgroup_wrapper_linux.go similarity index 96% rename from pkg/pool/cgroup_wrapper_linux.go rename to env/linuxcontainer/cgroup_wrapper_linux.go index 0dbec80..1d2e881 100644 --- a/pkg/pool/cgroup_wrapper_linux.go +++ b/env/linuxcontainer/cgroup_wrapper_linux.go @@ -1,9 +1,9 @@ -package pool +package linuxcontainer import ( "time" - "github.com/criyle/go-judge/pkg/envexec" + "github.com/criyle/go-judge/envexec" "github.com/criyle/go-sandbox/pkg/cgroup" ) diff --git a/pkg/pool/cgrouppool_linux.go b/env/linuxcontainer/cgrouppool_linux.go similarity index 96% rename from pkg/pool/cgrouppool_linux.go rename to env/linuxcontainer/cgrouppool_linux.go index 2147c50..d7f79d2 100644 --- a/pkg/pool/cgrouppool_linux.go +++ b/env/linuxcontainer/cgrouppool_linux.go @@ -1,10 +1,10 @@ -package pool +package linuxcontainer import ( "sync" "time" - "github.com/criyle/go-judge/pkg/envexec" + "github.com/criyle/go-judge/envexec" ) // Cgroup defines interface to limit and monitor resources consumption of a process diff --git a/pkg/pool/envbuilder_linux.go b/env/linuxcontainer/envbuilder_linux.go similarity index 87% rename from pkg/pool/envbuilder_linux.go rename to env/linuxcontainer/envbuilder_linux.go index 00c6723..5ab3350 100644 --- a/pkg/pool/envbuilder_linux.go +++ b/env/linuxcontainer/envbuilder_linux.go @@ -1,9 +1,10 @@ -package pool +package linuxcontainer import ( "fmt" "syscall" + "github.com/criyle/go-judge/env/pool" "github.com/criyle/go-sandbox/container" ) @@ -27,7 +28,7 @@ type environmentBuilder struct { } // NewEnvBuilder creates builder for linux container pools -func NewEnvBuilder(c Config) EnvBuilder { +func NewEnvBuilder(c Config) pool.EnvBuilder { return &environmentBuilder{ builder: c.Builder, cgPool: c.CgroupPool, @@ -39,7 +40,7 @@ func NewEnvBuilder(c Config) EnvBuilder { } // Build creates linux container -func (b *environmentBuilder) Build() (Environment, error) { +func (b *environmentBuilder) Build() (pool.Environment, error) { m, err := b.builder.Build() if err != nil { return nil, err diff --git a/pkg/pool/environment_linux.go b/env/linuxcontainer/environment_linux.go similarity index 97% rename from pkg/pool/environment_linux.go rename to env/linuxcontainer/environment_linux.go index 134aa87..2d3ea8a 100644 --- a/pkg/pool/environment_linux.go +++ b/env/linuxcontainer/environment_linux.go @@ -1,4 +1,4 @@ -package pool +package linuxcontainer import ( "context" @@ -7,7 +7,7 @@ import ( "syscall" "time" - "github.com/criyle/go-judge/pkg/envexec" + "github.com/criyle/go-judge/envexec" "github.com/criyle/go-sandbox/container" "github.com/criyle/go-sandbox/pkg/rlimit" ) diff --git a/pkg/pool/envprocess_linux.go b/env/linuxcontainer/envprocess_linux.go similarity index 94% rename from pkg/pool/envprocess_linux.go rename to env/linuxcontainer/envprocess_linux.go index b1904a9..8cffbe9 100644 --- a/pkg/pool/envprocess_linux.go +++ b/env/linuxcontainer/envprocess_linux.go @@ -1,9 +1,9 @@ -package pool +package linuxcontainer import ( "time" - "github.com/criyle/go-judge/pkg/envexec" + "github.com/criyle/go-judge/envexec" "github.com/criyle/go-sandbox/runner" ) diff --git a/pkg/pool/interface_linux.go b/env/linuxcontainer/interface_linux.go similarity index 94% rename from pkg/pool/interface_linux.go rename to env/linuxcontainer/interface_linux.go index 74af9e0..81df570 100644 --- a/pkg/pool/interface_linux.go +++ b/env/linuxcontainer/interface_linux.go @@ -1,4 +1,4 @@ -package pool +package linuxcontainer import ( "github.com/criyle/go-sandbox/container" diff --git a/pkg/macsandbox/builder_darwin.go b/env/macsandbox/builder_darwin.go similarity index 96% rename from pkg/macsandbox/builder_darwin.go rename to env/macsandbox/builder_darwin.go index 19f4e4b..0e0cbbc 100644 --- a/pkg/macsandbox/builder_darwin.go +++ b/env/macsandbox/builder_darwin.go @@ -4,7 +4,7 @@ import ( "io/ioutil" "os" - "github.com/criyle/go-judge/pkg/pool" + "github.com/criyle/go-judge/env/pool" ) var _ pool.EnvBuilder = &Builder{} diff --git a/pkg/macsandbox/environment_darwin.go b/env/macsandbox/environment_darwin.go similarity index 97% rename from pkg/macsandbox/environment_darwin.go rename to env/macsandbox/environment_darwin.go index 17a2a72..f24a3cf 100644 --- a/pkg/macsandbox/environment_darwin.go +++ b/env/macsandbox/environment_darwin.go @@ -7,8 +7,8 @@ import ( "syscall" "time" - "github.com/criyle/go-judge/pkg/envexec" - "github.com/criyle/go-judge/pkg/pool" + "github.com/criyle/go-judge/env/pool" + "github.com/criyle/go-judge/envexec" "github.com/criyle/go-sandbox/pkg/forkexec" "github.com/criyle/go-sandbox/pkg/rlimit" "github.com/criyle/go-sandbox/runner" diff --git a/pkg/macsandbox/process_darwin.go b/env/macsandbox/process_darwin.go similarity index 90% rename from pkg/macsandbox/process_darwin.go rename to env/macsandbox/process_darwin.go index 53a6a70..573fec5 100644 --- a/pkg/macsandbox/process_darwin.go +++ b/env/macsandbox/process_darwin.go @@ -1,7 +1,7 @@ package macsandbox import ( - "github.com/criyle/go-judge/pkg/envexec" + "github.com/criyle/go-judge/envexec" "github.com/criyle/go-sandbox/runner" ) diff --git a/pkg/macsandbox/profile_darwin.go b/env/macsandbox/profile_darwin.go similarity index 100% rename from pkg/macsandbox/profile_darwin.go rename to env/macsandbox/profile_darwin.go diff --git a/env/pool/doc.go b/env/pool/doc.go new file mode 100644 index 0000000..3e8e6e5 --- /dev/null +++ b/env/pool/doc.go @@ -0,0 +1,2 @@ +// Package pool provides reference implementation for envexec.EnvironmentPool from EnvBuilder +package pool diff --git a/pkg/pool/interface.go b/env/pool/interface.go similarity index 95% rename from pkg/pool/interface.go rename to env/pool/interface.go index d05f314..656028f 100644 --- a/pkg/pool/interface.go +++ b/env/pool/interface.go @@ -3,7 +3,7 @@ package pool import ( "sync" - "github.com/criyle/go-judge/pkg/envexec" + "github.com/criyle/go-judge/envexec" ) // Environment defines envexec.Environment with destroy diff --git a/pkg/winc/builder_windows.go b/env/winc/builder_windows.go similarity index 97% rename from pkg/winc/builder_windows.go rename to env/winc/builder_windows.go index 60e5535..efd01a4 100644 --- a/pkg/winc/builder_windows.go +++ b/env/winc/builder_windows.go @@ -4,7 +4,7 @@ import ( "io/ioutil" "os" - "github.com/criyle/go-judge/pkg/pool" + "github.com/criyle/go-judge/env/pool" "golang.org/x/sys/windows" ) diff --git a/pkg/winc/environment_windows.go b/env/winc/environment_windows.go similarity index 99% rename from pkg/winc/environment_windows.go rename to env/winc/environment_windows.go index 18f9f13..4a64319 100644 --- a/pkg/winc/environment_windows.go +++ b/env/winc/environment_windows.go @@ -12,8 +12,8 @@ import ( "unicode/utf16" "unsafe" - "github.com/criyle/go-judge/pkg/envexec" - "github.com/criyle/go-judge/pkg/pool" + "github.com/criyle/go-judge/env/pool" + "github.com/criyle/go-judge/envexec" "github.com/criyle/go-sandbox/runner" "golang.org/x/sys/windows" ) diff --git a/pkg/winc/job_object_windows.go b/env/winc/job_object_windows.go similarity index 98% rename from pkg/winc/job_object_windows.go rename to env/winc/job_object_windows.go index c4ce451..7de7ab7 100644 --- a/pkg/winc/job_object_windows.go +++ b/env/winc/job_object_windows.go @@ -4,7 +4,7 @@ import ( "time" "unsafe" - "github.com/criyle/go-judge/pkg/envexec" + "github.com/criyle/go-judge/envexec" "golang.org/x/sys/windows" ) diff --git a/pkg/winc/mksyscall.go b/env/winc/mksyscall.go similarity index 100% rename from pkg/winc/mksyscall.go rename to env/winc/mksyscall.go diff --git a/pkg/winc/process_windows.go b/env/winc/process_windows.go similarity index 96% rename from pkg/winc/process_windows.go rename to env/winc/process_windows.go index 8a3819d..65a7d53 100644 --- a/pkg/winc/process_windows.go +++ b/env/winc/process_windows.go @@ -4,7 +4,7 @@ import ( "time" "unsafe" - "github.com/criyle/go-judge/pkg/envexec" + "github.com/criyle/go-judge/envexec" "golang.org/x/sys/windows" ) diff --git a/pkg/winc/syscall_windows.go b/env/winc/syscall_windows.go similarity index 100% rename from pkg/winc/syscall_windows.go rename to env/winc/syscall_windows.go diff --git a/pkg/winc/token_windows.go b/env/winc/token_windows.go similarity index 100% rename from pkg/winc/token_windows.go rename to env/winc/token_windows.go diff --git a/pkg/winc/zsyscall_windows.go b/env/winc/zsyscall_windows.go similarity index 100% rename from pkg/winc/zsyscall_windows.go rename to env/winc/zsyscall_windows.go diff --git a/pkg/envexec/cmd.go b/envexec/cmd.go similarity index 100% rename from pkg/envexec/cmd.go rename to envexec/cmd.go diff --git a/pkg/envexec/const.go b/envexec/const.go similarity index 100% rename from pkg/envexec/const.go rename to envexec/const.go diff --git a/pkg/envexec/doc.go b/envexec/doc.go similarity index 100% rename from pkg/envexec/doc.go rename to envexec/doc.go diff --git a/pkg/envexec/file_collect.go b/envexec/file_collect.go similarity index 83% rename from pkg/envexec/file_collect.go rename to envexec/file_collect.go index 4509f61..f3dbc1f 100644 --- a/pkg/envexec/file_collect.go +++ b/envexec/file_collect.go @@ -4,6 +4,7 @@ import ( "fmt" "io/ioutil" "os" + "sync" "github.com/criyle/go-judge/file" "github.com/criyle/go-sandbox/runner" @@ -12,8 +13,16 @@ import ( // copyOutAndCollect reads file and pipes in parallel from container func copyOutAndCollect(m Environment, c *Cmd, ptc []pipeCollector) (map[string]file.File, error) { - var g errgroup.Group - fc := make(chan file.File, len(ptc)+len(c.CopyOut)) + var ( + g errgroup.Group + l sync.Mutex + ) + rt := make(map[string]file.File) + put := func(f file.File) { + l.Lock() + defer l.Unlock() + rt[f.Name()] = f + } // copy out for _, n := range c.CopyOut { @@ -40,7 +49,7 @@ func copyOutAndCollect(m Environment, c *Cmd, ptc []pipeCollector) (map[string]f if err != nil { return err } - fc <- file.NewMemFile(n, c) + put(file.NewMemFile(n, c)) return nil }) } @@ -53,7 +62,7 @@ func copyOutAndCollect(m Environment, c *Cmd, ptc []pipeCollector) (map[string]f if int64(p.buff.Buffer.Len()) > p.buff.Max { return runner.StatusOutputLimitExceeded } - fc <- file.NewMemFile(p.name, p.buff.Buffer.Bytes()) + put(file.NewMemFile(p.name, p.buff.Buffer.Bytes())) return nil }) } @@ -65,15 +74,6 @@ func copyOutAndCollect(m Environment, c *Cmd, ptc []pipeCollector) (map[string]f }) } - var err error - go func() { - err = g.Wait() - close(fc) - }() - - rt := make(map[string]file.File) - for f := range fc { - rt[f.Name()] = f - } + err := g.Wait() return rt, err } diff --git a/pkg/envexec/file_copyin.go b/envexec/file_copyin.go similarity index 100% rename from pkg/envexec/file_copyin.go rename to envexec/file_copyin.go diff --git a/pkg/envexec/file_prepare.go b/envexec/file_prepare.go similarity index 100% rename from pkg/envexec/file_prepare.go rename to envexec/file_prepare.go diff --git a/pkg/envexec/file_util_linux.go b/envexec/file_util_linux.go similarity index 100% rename from pkg/envexec/file_util_linux.go rename to envexec/file_util_linux.go diff --git a/pkg/envexec/file_util_others.go b/envexec/file_util_others.go similarity index 100% rename from pkg/envexec/file_util_others.go rename to envexec/file_util_others.go diff --git a/pkg/envexec/group.go b/envexec/group.go similarity index 100% rename from pkg/envexec/group.go rename to envexec/group.go diff --git a/pkg/envexec/interface.go b/envexec/interface.go similarity index 100% rename from pkg/envexec/interface.go rename to envexec/interface.go diff --git a/pkg/envexec/run_single.go b/envexec/run_single.go similarity index 100% rename from pkg/envexec/run_single.go rename to envexec/run_single.go diff --git a/pkg/envexec/run_status.go b/envexec/run_status.go similarity index 100% rename from pkg/envexec/run_status.go rename to envexec/run_status.go diff --git a/pkg/envexec/single.go b/envexec/single.go similarity index 100% rename from pkg/envexec/single.go rename to envexec/single.go diff --git a/pkg/envexec/util.go b/envexec/util.go similarity index 100% rename from pkg/envexec/util.go rename to envexec/util.go diff --git a/go.mod b/go.mod index 0925936..26a4d6a 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.14 require ( cloud.google.com/go v0.74.0 // indirect github.com/creack/pty v1.1.11 - github.com/criyle/go-sandbox v0.5.10 + github.com/criyle/go-sandbox v0.6.1 github.com/elastic/go-seccomp-bpf v1.1.0 github.com/elastic/go-ucfg v0.7.0 github.com/fatih/camelcase v1.0.0 // indirect @@ -28,9 +28,9 @@ require ( go.uber.org/multierr v1.6.0 // indirect go.uber.org/zap v1.16.0 golang.org/x/crypto v0.0.0-20201208171446-5f87f3452ae9 - golang.org/x/net v0.0.0-20201209123823-ac852fbbde11 + golang.org/x/net v0.0.0-20201216054612-986b41b23924 golang.org/x/sync v0.0.0-20201207232520-09787c993a3a - golang.org/x/sys v0.0.0-20201211090839-8ad439b19e0f + golang.org/x/sys v0.0.0-20201223074533-0d417f636930 golang.org/x/term v0.0.0-20201210144234-2321bbc49cbf // indirect golang.org/x/tools v0.0.0-20201211185031-d93e913c1a58 // indirect google.golang.org/genproto v0.0.0-20201211151036-40ec1c210f7a // indirect diff --git a/go.sum b/go.sum index 520fbae..9e4c26c 100644 --- a/go.sum +++ b/go.sum @@ -85,8 +85,8 @@ github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7Do github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/creack/pty v1.1.11 h1:07n33Z8lZxZ2qwegKbObQohDhXDQxiMMz1NOUGYlesw= github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/criyle/go-sandbox v0.5.10 h1:T+HUg5UVzIqIjKp+H3hBIkPpODUEChUsdwVch0LgQhs= -github.com/criyle/go-sandbox v0.5.10/go.mod h1:6iDDyseyHVugTM2ixPleIBuRx3pVuMKh2qTm6IRmvd8= +github.com/criyle/go-sandbox v0.6.1 h1:CQi/6hqWJt9ELtbt3rvqIfAKt3YjJ+h3BASlPWCA8ok= +github.com/criyle/go-sandbox v0.6.1/go.mod h1:3IAbhJBgsSoyLixQELE82uibbZJ3dK0XNaqAy949hvk= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -564,10 +564,10 @@ golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81R golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb h1:eBmm0M9fYhWpKZLjQUUKka/LtIxf46G4fxeEz5KJr9U= -golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201209123823-ac852fbbde11 h1:lwlPPsmjDKK0J6eG6xDWd5XPehI0R024zxjDnw3esPA= golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20201216054612-986b41b23924 h1:QsnDpLLOKwHBBDa8nDws4DYNc/ryVW2vCpxCs09d4PY= +golang.org/x/net v0.0.0-20201216054612-986b41b23924/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be h1:vEDujvNQGv4jgYKudGeI/+DAX4Jffq6hpD55MmoEvKs= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -643,10 +643,8 @@ golang.org/x/sys v0.0.0-20201015000850-e3ed0017c211/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201119102817-f84b799fce68 h1:nxC68pudNYkKU6jWhgrqdreuFiOQWj1Fs7T3VrH4Pjw= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201204225414-ed752295db88 h1:KmZPnMocC93w341XZp26yTJg8Za7lhb2KhkYmixoeso= -golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201211090839-8ad439b19e0f h1:QdHQnPce6K4XQewki9WNbG5KOROuDzqO3NaYjI1cXJ0= -golang.org/x/sys v0.0.0-20201211090839-8ad439b19e0f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201223074533-0d417f636930 h1:vRgIt+nup/B/BwIS0g2oC0haq0iqbV3ZA+u6+0TlNCo= +golang.org/x/sys v0.0.0-20201223074533-0d417f636930/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221 h1:/ZHdbVpdR/jk3g30/d4yUL0JU9kksj8+F/bnQUVLGDM= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E= diff --git a/pkg/pool/doc.go b/pkg/pool/doc.go deleted file mode 100644 index d6dfa13..0000000 --- a/pkg/pool/doc.go +++ /dev/null @@ -1,2 +0,0 @@ -// Package pool provides reference implements of environent pool and cgroup pool -package pool diff --git a/worker/file.go b/worker/file.go index 7d841b0..0493b84 100644 --- a/worker/file.go +++ b/worker/file.go @@ -3,9 +3,9 @@ package worker import ( "fmt" + "github.com/criyle/go-judge/envexec" "github.com/criyle/go-judge/file" "github.com/criyle/go-judge/filestore" - "github.com/criyle/go-judge/pkg/envexec" ) // CmdFile defines file used in the cmd diff --git a/worker/model.go b/worker/model.go index af92561..446c665 100644 --- a/worker/model.go +++ b/worker/model.go @@ -1,6 +1,11 @@ package worker -import "github.com/criyle/go-judge/pkg/envexec" +import ( + "fmt" + "time" + + "github.com/criyle/go-judge/envexec" +) // Cmd defines command and limits to start a program using in envexec type Cmd struct { @@ -9,10 +14,10 @@ type Cmd struct { Files []CmdFile TTY bool - CPULimit uint64 - RealCPULimit uint64 - MemoryLimit uint64 - StackLimit uint64 + CPULimit time.Duration + RealCPULimit time.Duration + MemoryLimit envexec.Size + StackLimit envexec.Size ProcLimit uint64 CPURateLimit float64 @@ -48,9 +53,9 @@ type Result struct { Status envexec.Status ExitStatus int Error string - Time uint64 - RunTime uint64 - Memory uint64 + Time time.Duration + RunTime time.Duration + Memory envexec.Size Files map[string][]byte FileIDs map[string]string } @@ -61,3 +66,30 @@ type Response struct { Results []Result Error error } + +func (r Result) String() string { + type Result struct { + Status envexec.Status + ExitStatus int + Error string + Time time.Duration + RunTime time.Duration + Memory envexec.Size + Files map[string]string + FileIDs map[string]string + } + d := Result{ + Status: r.Status, + ExitStatus: r.ExitStatus, + Error: r.Error, + Time: r.Time, + RunTime: r.RunTime, + Memory: r.Memory, + Files: make(map[string]string), + FileIDs: r.FileIDs, + } + for k, v := range r.Files { + d.Files[k] = fmt.Sprintf("(len:%d)", len(v)) + } + return fmt.Sprintf("%+v", d) +} diff --git a/worker/waiter.go b/worker/waiter.go index 1ee8b69..131b6a0 100644 --- a/worker/waiter.go +++ b/worker/waiter.go @@ -4,7 +4,7 @@ import ( "context" "time" - "github.com/criyle/go-judge/pkg/envexec" + "github.com/criyle/go-judge/envexec" ) // default tick interval 100 ms diff --git a/worker/worker.go b/worker/worker.go index a276a78..f0f5ce0 100644 --- a/worker/worker.go +++ b/worker/worker.go @@ -7,9 +7,9 @@ import ( "sync" "time" + "github.com/criyle/go-judge/envexec" "github.com/criyle/go-judge/file" "github.com/criyle/go-judge/filestore" - "github.com/criyle/go-judge/pkg/envexec" ) const maxWaiting = 512 @@ -201,9 +201,9 @@ func (w *worker) convertResult(result envexec.Result, copyOutSet map[string]bool res.Status = result.Status res.ExitStatus = result.ExitStatus res.Error = result.Error - res.Time = uint64(result.Time) - res.RunTime = uint64(result.RunTime) - res.Memory = uint64(result.Memory) + res.Time = result.Time + res.RunTime = result.RunTime + res.Memory = result.Memory res.Files = make(map[string][]byte) res.FileIDs = make(map[string]string)