filestore: use /dev/shm instead of memory as file storage

- increase default tmpfs size to 128M
- add /config to get file store path
- remove memory only file store

close #20
This commit is contained in:
criyle 2021-09-19 15:47:24 -07:00
parent 9414057b8b
commit 78bb4d85de
40 changed files with 463 additions and 250 deletions

View File

@ -5,3 +5,7 @@ cmd = "go build -o ./tmp/executorserver ./cmd/executorserver"
full_bin = "tmp/executorserver -enable-grpc -enable-debug"
include_ext = ["go"]
delay = 1000
send_interrupt = true
kill_delay = 3000

View File

@ -8,6 +8,11 @@
快速,简单,安全
### 运行要求
- Linux 内核版本 >= 3.10
- 系统 Cgroup V1 文件系统挂载于 `/sys/fs/cgroup`Systemd 默认。Cgroup V2 还未支持
### 系统架构
```text
@ -35,6 +40,7 @@
- /metrics 提供 prometheus 版监控 (使用 `ES_ENABLE_METRICS=1` 环境变量开启)
- /debug 提供 go 语言调试接口 (使用 `ES_ENABLE_DEBUG=1` 环境变量开启)
- /version 得到本程序编译版本和 go 语言运行时版本
- /config 得到本程序部分运行参数
### 命令行参数
@ -81,7 +87,7 @@
或者使用 docker
```bash
docker run -it --rm --privileged -p 5050:5050 criyle/executorserver
docker run -it --rm --privileged --shm-size=256m -p 5050:5050 criyle/executorserver
```
#### 编译 docker
@ -126,6 +132,8 @@ docker run -it --rm --privileged -p 5050:5050 criyle/executorserver
使用 `mount.yaml` 定制容器文件系统。
`/w``/tmp` 挂载 `tmpfs` 大小通过 `-tmp-fs-param` 指定,默认值为 `size=128m,nr_inodes=4k`
### 包
- envexec: 核心逻辑包,在提供的环境中运行一个或多个程序

View File

@ -8,18 +8,23 @@
Fast, Simple, Secure
### Prerequisite
- Linux Kernel Version >= 3.10
- Cgroup file system V1 mounted at /sys/fs/cgroup. Cgroup V2 is not supported
### Architecture
```text
+---------------------------------------------------------------------------+
| Transport Layer (HTTP / WebSocket / FFI / ...) |
+---------------------------------------------------------------------------+
| Executor Worker (Environment Pool w/ Environment Builder ) |
+-----------------------------------------------------------+---------------+
| EnvExec | File Store |
+--------------------+----------------+---------------------+--------+------+
| Linux (go-sandbox) | Windows (winc) | macOS (app sandbox) | Memory | Disk |
+--------------------+----------------+---------------------+--------+------+
+----------------------------------------------------------------------------------+
| Transport Layer (HTTP / WebSocket / FFI / ...) |
+----------------------------------------------------------------------------------+
| Executor Worker (Environment Pool w/ Environment Builder ) |
+-----------------------------------------------------------+----------------------+
| EnvExec | File Store |
+--------------------+----------------+---------------------+---------------+------+
| Linux (go-sandbox) | Windows (winc) | macOS (app sandbox) | Shared Memory | Disk |
+--------------------+----------------+---------------------+---------------+------+
```
### REST API
@ -35,6 +40,7 @@ A REST service to run program in restricted environment and it is basically a wr
- /metrics prometheus metrics (specifies `ES_ENABLE_METRICS=1` environment variable to enable metrics)
- /debug (specifies `ES_ENABLE_DEBUG=1` environment variable to enable go runtime debug endpoint)
- /version gets build git version (e.g. `v0.9.4`) together with runtime information (go version, os, platform)
- /config gets some configuration (e.g. `fileStorePath`) together with some supported features
### Command Line Arguments
@ -84,7 +90,7 @@ Download compiled executable from [Release](https://github.com/criyle/go-judge/r
Or, by docker
```bash
docker run -it --rm --privileged -p 5050:5050 criyle/executorserver
docker run -it --rm --privileged --shm-size=256m -p 5050:5050 criyle/executorserver
```
#### Build Executor Server
@ -145,6 +151,8 @@ For linux platform, the default mounts points are bind mounting host's `/lib`, `
To customize mount points, please look at example `mount.yaml` file.
`tmpfs` size for `/w` and `/tmp` is configured through `-tmp-fs-param` with default value `size=128m,nr_inodes=4k`
### Packages
- envexec: run single / group of programs in parallel within restricted environment and resource constraints
@ -185,9 +193,9 @@ sysctl -p
#### Memory Usage
The controller will consume `60M` memory and each container will consume `20M` + size of tmpfs `2 * 16M`. For each request, it consumes as much as user program limit + extra limit (`16k`) + total copy out max * 2.
The controller will consume `60M` memory and each container will consume `20M` + size of tmpfs `2 * 128M`. For each request, it consumes as much as user program limit + extra limit (`16k`) + total copy out max.
For example, when concurrency = 4, the executor itself can consume as much as `60 + (20+32) * 4M = 268M` + 8 * total copy out + total max memory of requests.
For example, when concurrency = 4, the executor itself can consume as much as `60 + (20+32) * 4M = 268M` + 4 * total copy out + total max memory of requests.
Due to limitation of GO runtime, the memory will not return to OS automatically, which could lead to OOM killer. The background worker was introduced to checks heap usage and invokes GC when necessary.

View File

@ -14,7 +14,7 @@ type Config struct {
// container
ContainerInitPath string `flagUsage:"container init path"`
PreFork int `flagUsage:"control # of the prefork workers" default:"1"`
TmpFsParam string `flagUsage:"tmpfs mount data (only for default mount with no mount.yaml)" default:"size=16m,nr_inodes=4k"`
TmpFsParam string `flagUsage:"tmpfs mount data (only for default mount with no mount.yaml)" default:"size=128m,nr_inodes=4k"`
NetShare bool `flagUsage:"share net namespace with host"`
MountConf string `flagUsage:"specifies mount configuration file" default:"mount.yaml"`
SeccompConf string `flagUsage:"specifies seccomp filter" default:"seccomp.yaml"`
@ -49,7 +49,7 @@ type Config struct {
Silent bool `flagUsage:"do not print logs"`
// fix for high memory usage
ForceGCTarget *envexec.Size `flagUsage:"specifies force GC trigger heap size" default:"10m"`
ForceGCTarget *envexec.Size `flagUsage:"specifies force GC trigger heap size" default:"20m"`
ForceGCInterval time.Duration `flagUsage:"specifies force GC trigger interval" default:"5s"`
}

View File

@ -10,6 +10,7 @@ import (
"sync"
"time"
"github.com/criyle/go-judge/cmd/executorserver/model"
"github.com/criyle/go-judge/envexec"
"github.com/criyle/go-judge/filestore"
"github.com/criyle/go-judge/pb"
@ -60,7 +61,13 @@ func (e *execServer) Exec(ctx context.Context, req *pb.Request) (*pb.Response, e
if rt.Error != nil {
return nil, rt.Error
}
return convertPBResponse(rt), nil
ret, err := model.ConvertResponse(rt)
if err != nil {
return nil, err
}
defer ret.Close()
return convertPBResponse(ret)
}
func (e *execServer) FileList(c context.Context, n *emptypb.Empty) (*pb.FileListType, error) {
@ -88,9 +95,18 @@ func (e *execServer) FileGet(c context.Context, f *pb.FileID) (*pb.FileContent,
}
func (e *execServer) FileAdd(c context.Context, fc *pb.FileContent) (*pb.FileID, error) {
fid, err := e.fs.Add(fc.GetName(), fc.GetContent())
f, err := e.fs.New()
if err != nil {
return nil, status.Error(codes.InvalidArgument, err.Error())
return nil, status.Error(codes.Internal, err.Error())
}
defer f.Close()
if _, err := f.Write(fc.GetContent()); err != nil {
return nil, status.Error(codes.Internal, err.Error())
}
fid, err := e.fs.Add(fc.GetName(), f.Name())
if err != nil {
return nil, status.Error(codes.Internal, err.Error())
}
return &pb.FileID{
FileID: fid,
@ -105,21 +121,23 @@ func (e *execServer) FileDelete(c context.Context, f *pb.FileID) (*emptypb.Empty
return &emptypb.Empty{}, nil
}
func convertPBResponse(r worker.Response) *pb.Response {
func convertPBResponse(r model.Response) (*pb.Response, error) {
res := &pb.Response{
RequestID: r.RequestID,
Results: make([]*pb.Response_Result, 0, len(r.Results)),
Error: r.ErrorMsg,
}
for _, c := range r.Results {
res.Results = append(res.Results, convertPBResult(c))
rt, err := convertPBResult(c)
if err != nil {
return nil, err
}
res.Results = append(res.Results, rt)
}
if r.Error != nil {
res.Error = r.Error.Error()
}
return res
return res, nil
}
func convertPBResult(r worker.Result) *pb.Response_Result {
func convertPBResult(r model.Result) (*pb.Response_Result, error) {
return &pb.Response_Result{
Status: pb.Response_Result_StatusType(r.Status),
ExitStatus: int32(r.ExitStatus),
@ -127,9 +145,9 @@ func convertPBResult(r worker.Result) *pb.Response_Result {
Time: uint64(r.Time),
RunTime: uint64(r.RunTime),
Memory: uint64(r.Memory),
Files: r.Files,
Files: r.Buffs,
FileIDs: r.FileIDs,
}
}, nil
}
func convertPBRequest(r *pb.Request, srcPrefix string) (req *worker.Request, streamIn []*fileStreamIn, streamOut []*fileStreamOut, err error) {

View File

@ -1,3 +1,4 @@
//go:build !linux && !darwin
// +build !linux,!darwin
package grpcexecutor

View File

@ -6,6 +6,7 @@ import (
"io"
"sync"
"github.com/criyle/go-judge/cmd/executorserver/model"
"github.com/criyle/go-judge/pb"
"github.com/criyle/go-judge/worker"
"go.uber.org/zap"
@ -99,9 +100,19 @@ func execStreamLoop(es pb.Executor_ExecStreamServer, errCh chan error, outCh cha
case rt := <-rtCh:
logger.Sugar().Debugf("response: %+v", rt)
ret, err := model.ConvertResponse(rt)
if err != nil {
return status.Errorf(codes.Aborted, "response: %v", err)
}
defer ret.Close()
resp, err := convertPBResponse(ret)
if err != nil {
return status.Errorf(codes.Aborted, "response: %v", err)
}
return es.Send(&pb.StreamResponse{
Response: &pb.StreamResponse_ExecResponse{
ExecResponse: convertPBResponse(rt),
ExecResponse: resp,
},
})
}

View File

@ -1,3 +1,4 @@
//go:build linux || darwin
// +build linux darwin
package grpcexecutor

View File

@ -51,7 +51,11 @@ func main() {
logger.Sugar().Infof("config loaded: %+v", conf)
// Init environment pool
fs := newFilsStore(conf.Dir, conf.FileTimeout, conf.EnableMetrics)
fs, fsDir, fsCleanUp, err := newFilsStore(conf.Dir, conf.FileTimeout, conf.EnableMetrics)
if err != nil {
logger.Sugar().Fatalf("Create temp dir failed %v", err)
}
conf.Dir = fsDir
b := newEnvBuilder(conf)
envPool := newEnvPool(b, conf.EnableMetrics)
prefork(envPool, conf.PreFork)
@ -67,9 +71,13 @@ func main() {
Handler: r,
}
// Gracefully shutdown, with signal / HTTP server / gRPC server
sig := make(chan os.Signal, 3)
go func() {
logger.Sugar().Info("Starting http server at ", conf.HTTPAddr)
logger.Sugar().Info("Http serve finished: ", srv.ListenAndServe())
logger.Sugar().Info("Http server stopped: ", srv.ListenAndServe())
sig <- os.Interrupt
}()
// Init gRPC server
@ -84,7 +92,8 @@ func main() {
}
go func() {
logger.Sugar().Info("Starting gRPC server at ", conf.GRPCAddr)
logger.Sugar().Info("gRPC serve finished: ", grpcServer.Serve(lis))
logger.Sugar().Info("gRPC server stopped: ", grpcServer.Serve(lis))
sig <- os.Interrupt
}()
}
@ -92,9 +101,9 @@ func main() {
newForceGCWorker(conf)
// Graceful shutdown...
sig := make(chan os.Signal, 1)
signal.Notify(sig, os.Interrupt)
<-sig
signal.Reset(os.Interrupt)
logger.Sugar().Info("Shutting Down...")
@ -121,6 +130,14 @@ func main() {
})
}
if fsCleanUp != nil {
eg.Go(func() error {
err := fsCleanUp()
logger.Sugar().Info("FileStore clean up")
return err
})
}
go func() {
logger.Sugar().Info("Shutdown Finished: ", eg.Wait())
cancel()
@ -197,6 +214,9 @@ func initHTTPMux(conf *config.Config, work worker.Worker, fs filestore.FileStore
// Version handle
r.GET("/version", handleVersion)
// Config handle
r.GET("/config", generateHandleConfig(conf))
// Add auth token
if conf.AuthToken != "" {
r.Use(tokenAuth(conf.AuthToken))
@ -279,23 +299,35 @@ func grpcTokenAuth(token string) func(context.Context) (context.Context, error)
}
}
func newFilsStore(dir string, fileTimeout time.Duration, enableMetrics bool) filestore.FileStore {
func newFilsStore(dir string, fileTimeout time.Duration, enableMetrics bool) (filestore.FileStore, string, func() error, error) {
const timeoutCheckInterval = 15 * time.Second
var cleanUp func() error
var fs filestore.FileStore
if dir == "" {
fs = filestore.NewFileMemoryStore()
} else {
os.MkdirAll(dir, 0755)
fs = filestore.NewFileLocalStore(dir)
if runtime.GOOS == "linux" {
dir = "/dev/shm"
} else {
dir = os.TempDir()
}
var err error
dir, err = os.MkdirTemp(dir, "executorserver")
if err != nil {
return nil, "", cleanUp, err
}
cleanUp = func() error {
return os.RemoveAll(dir)
}
}
os.MkdirAll(dir, 0755)
fs = filestore.NewFileLocalStore(dir)
if enableMetrics {
fs = newMetricsFileStore(fs)
}
if fileTimeout > 0 {
fs = filestore.NewTimeout(fs, fileTimeout, timeoutCheckInterval)
}
return fs
return fs, dir, cleanUp, nil
}
func newEnvBuilder(conf *config.Config) pool.EnvBuilder {
@ -370,3 +402,13 @@ func handleVersion(c *gin.Context) {
"pipeProxy": true,
})
}
func generateHandleConfig(conf *config.Config) func(*gin.Context) {
return func(c *gin.Context) {
c.JSON(http.StatusOK, gin.H{
"copyOutOptional": true,
"pipeProxy": true,
"fileStorePath": conf.Dir,
})
}
}

View File

@ -1,6 +1,7 @@
package main
import (
"os"
"sync"
"time"
@ -131,18 +132,18 @@ var _ filestore.FileStore = &metricsFileStore{}
type metricsFileStore struct {
mu sync.Mutex
filestore.FileStore
fileSize map[string]int
fileSize map[string]int64
}
func newMetricsFileStore(fs filestore.FileStore) filestore.FileStore {
return &metricsFileStore{
FileStore: fs,
fileSize: make(map[string]int),
fileSize: make(map[string]int64),
}
}
func (m *metricsFileStore) Add(name string, content []byte) (string, error) {
id, err := m.FileStore.Add(name, content)
func (m *metricsFileStore) Add(name, path string) (string, error) {
id, err := m.FileStore.Add(name, path)
if err != nil {
return "", err
}
@ -150,7 +151,12 @@ func (m *metricsFileStore) Add(name string, content []byte) (string, error) {
m.mu.Lock()
defer m.mu.Unlock()
s := len(content)
fi, err := os.Stat(path)
if err != nil {
return id, nil
}
s := fi.Size()
m.fileSize[id] = s
sf := float64(s)

View File

@ -95,6 +95,9 @@ type Result struct {
RunTime uint64 `json:"runTime"`
Files map[string]string `json:"files,omitempty"`
FileIDs map[string]string `json:"fileIds,omitempty"`
files []string
Buffs map[string][]byte `json:"-"`
}
// Response defines worker response for single request
@ -104,19 +107,55 @@ type Response struct {
ErrorMsg string `json:"error,omitempty"`
}
func (r *Response) Close() {
for _, res := range r.Results {
res.Close()
}
}
func (r *Result) Close() {
// remove temporary files
for _, f := range r.files {
os.Remove(f)
}
// remove potential mmap
for _, b := range r.Buffs {
releaseByte(b)
}
}
// ConvertResponse converts
func ConvertResponse(r worker.Response) Response {
ret := Response{
func ConvertResponse(r worker.Response) (ret Response, err error) {
// in error case, release all resources
defer func() {
if err != nil {
for _, r := range ret.Results {
r.Close()
}
for _, r := range r.Results {
for _, f := range r.Files {
f.Close()
os.Remove(f.Name())
}
}
}
}()
ret = Response{
RequestID: r.RequestID,
Results: make([]Result, 0, len(r.Results)),
}
for _, r := range r.Results {
ret.Results = append(ret.Results, convertResult(r))
res, err := convertResult(r)
if err != nil {
return ret, err
}
ret.Results = append(ret.Results, res)
}
if r.Error != nil {
ret.ErrorMsg = r.Error.Error()
}
return ret
return ret, nil
}
// ConvertRequest converts json request into worker request
@ -139,7 +178,7 @@ func ConvertRequest(r *Request, srcPrefix string) (*worker.Request, error) {
return req, nil
}
func convertResult(r worker.Result) Result {
func convertResult(r worker.Result) (Result, error) {
res := Result{
Status: Status(r.Status),
ExitStatus: r.ExitStatus,
@ -151,11 +190,19 @@ func convertResult(r worker.Result) Result {
}
if r.Files != nil {
res.Files = make(map[string]string)
for k, v := range r.Files {
res.Files[k] = byteArrayToString(v)
res.Buffs = make(map[string][]byte)
for k, f := range r.Files {
b, err := fileToByte(f)
if err != nil {
return res, err
}
res.Files[k] = byteArrayToString(b)
res.files = append(res.files, f.Name())
res.Buffs[k] = b
}
}
return res
return res, nil
}
func convertPipe(p PipeMap) worker.PipeMap {

View File

@ -0,0 +1,32 @@
package model
import (
"os"
"syscall"
)
func fileToByte(f *os.File) ([]byte, error) {
defer f.Close()
var s int64
if fi, err := f.Stat(); err != nil {
return nil, err
} else {
s = fi.Size()
}
if s == 0 {
return []byte{}, nil
}
b, err := syscall.Mmap(int(f.Fd()), 0, int(s), syscall.PROT_READ, syscall.MAP_SHARED)
if err != nil {
return nil, err
}
return b, nil
}
func releaseByte(b []byte) {
if len(b) > 0 {
_ = syscall.Munmap(b)
}
}

View File

@ -0,0 +1,30 @@
//go:build !linux
// +build !linux
package model
import (
"io"
"os"
)
func fileToByte(f *os.File) ([]byte, error) {
if _, err := f.Seek(0, 0); err != nil {
return nil, err
}
var s int64
if fi, err := f.Stat(); err != nil {
return nil, err
} else {
s = fi.Size()
}
c := make([]byte, s)
if _, err := io.ReadFull(f, c); err != nil {
return nil, err
}
f.Close()
return c, nil
}
func releaseByte(b []byte) {
}

View File

@ -74,12 +74,18 @@ func (h *handle) handleRun(c *gin.Context) {
}
// encode json directly to avoid allocation
// c.JSON(http.StatusOK, model.ConvertResponse(rt).Results)
c.Status(http.StatusOK)
c.Header("Content-Type", "application/json; charset=utf-8")
if err := json.NewEncoder(c.Writer).Encode(model.ConvertResponse(rt).Results); err != nil {
res, err := model.ConvertResponse(rt)
if err != nil {
c.Error(err)
c.AbortWithStatusJSON(http.StatusInternalServerError, err.Error())
return
}
defer res.Close()
if err := json.NewEncoder(c.Writer).Encode(res.Results); err != nil {
c.Error(err)
}
}

View File

@ -33,13 +33,18 @@ func (f *fileHandle) filePost(c *gin.Context) {
c.AbortWithError(http.StatusInternalServerError, err)
return
}
b, err := io.ReadAll(fi)
sf, err := f.fs.New()
if err != nil {
c.AbortWithError(http.StatusInternalServerError, err)
return
}
defer sf.Close()
id, err := f.fs.Add(fh.Filename, b)
if _, err := sf.ReadFrom(fi); err != nil {
c.AbortWithError(http.StatusInternalServerError, err)
return
}
id, err := f.fs.Add(fh.Filename, sf.Name())
if err != nil {
c.AbortWithError(http.StatusInternalServerError, err)
return

View File

@ -1,3 +1,4 @@
//go:build ignore
// +build ignore
package main

View File

@ -2,4 +2,4 @@
package version
const Version = `v1.1.7`
const Version = `v1.2.7`

View File

@ -81,7 +81,9 @@ func (h *wsHandle) handleWS(c *gin.Context) {
}
go func() {
ret := <-h.worker.Submit(ctx, r)
resultCh <- model.ConvertResponse(ret)
resp, _ := model.ConvertResponse(ret)
resp.Close()
resultCh <- resp
}()
}
}()

View File

@ -1,3 +1,4 @@
//go:build !linux
// +build !linux
package main

View File

@ -9,6 +9,7 @@ import (
"io"
"log"
"os"
"runtime"
"time"
"github.com/criyle/go-judge/cmd/executorserver/model"
@ -40,14 +41,16 @@ var (
)
func newFilsStore(dir string) filestore.FileStore {
var fs filestore.FileStore
if dir == "" {
fs = filestore.NewFileMemoryStore()
} else {
os.MkdirAll(dir, 0755)
fs = filestore.NewFileLocalStore(dir)
if runtime.GOOS == "linux" {
dir = "/dev/shm"
} else {
dir = os.TempDir()
}
dir, _ = os.MkdirTemp(dir, "executorserver")
}
return fs
os.MkdirAll(dir, 0755)
return filestore.NewFileLocalStore(dir)
}
// Init initialize the sandbox environment
@ -114,11 +117,13 @@ func Exec(e *C.char) *C.char {
return nil
}
rt := <-work.Submit(context.TODO(), r)
ret := model.ConvertResponse(rt)
ret, _ := model.ConvertResponse(rt)
var buf bytes.Buffer
if err := json.NewEncoder(&buf).Encode(ret); err != nil {
return nil
}
defer ret.Close()
return C.CString(buf.String())
}
@ -143,12 +148,21 @@ func FileAdd(e *C.char) *C.char {
}
es := C.GoString(e)
var f fileAdd
if err := json.NewDecoder(bytes.NewBufferString(es)).Decode(&f); err != nil {
var fa fileAdd
if err := json.NewDecoder(bytes.NewBufferString(es)).Decode(&fa); err != nil {
return nil
}
id, err := fs.Add(f.Name, []byte(f.Content))
f, err := fs.New()
if err != nil {
return nil
}
defer f.Close()
if _, err := f.Write([]byte(fa.Content)); err != nil {
return nil
}
id, err := fs.Add(fa.Name, f.Name())
if err != nil {
return nil
}

1
env/env_others.go vendored
View File

@ -1,3 +1,4 @@
//go:build !windows && !linux && !darwin
// +build !windows,!linux,!darwin
package env

View File

@ -1,3 +1,4 @@
//go:build !seccomp
// +build !seccomp
package env

View File

@ -1,3 +1,4 @@
//go:build seccomp
// +build seccomp
package env

View File

@ -2,6 +2,7 @@ package envexec
import (
"context"
"os"
"time"
"github.com/criyle/go-sandbox/runner"
@ -70,5 +71,5 @@ type Result struct {
Memory Size // byte
// Files stores copy out files
Files map[string][]byte
Files map[string]*os.File
}

View File

@ -1,7 +1,6 @@
package envexec
import (
"bytes"
"errors"
"fmt"
"io"
@ -13,13 +12,13 @@ import (
)
// copyOutAndCollect reads file and pipes in parallel from container
func copyOutAndCollect(m Environment, c *Cmd, ptc []pipeCollector) (map[string][]byte, error) {
func copyOutAndCollect(m Environment, c *Cmd, ptc []pipeCollector, newStoreFile NewStoreFile) (map[string]*os.File, error) {
var (
g errgroup.Group
l sync.Mutex
)
rt := make(map[string][]byte)
put := func(f []byte, n string) {
rt := make(map[string]*os.File)
put := func(f *os.File, n string) {
l.Lock()
defer l.Unlock()
rt[n] = f
@ -51,15 +50,18 @@ func copyOutAndCollect(m Environment, c *Cmd, ptc []pipeCollector) (map[string][
if c.CopyOutMax > 0 && s > int64(c.CopyOutMax) {
return fmt.Errorf("%s: size (%d) exceeded the limit (%d)", n.Name, s, c.CopyOutMax)
}
var buf bytes.Buffer
buf.Grow(int(s))
// create store file
buf, err := newStoreFile()
if err != nil {
return fmt.Errorf("%s: failed to create store file %v", n.Name, err)
}
// Ensure not copy over file size
_, err = buf.ReadFrom(io.LimitReader(cf, s))
if err != nil {
return err
}
put(buf.Bytes(), n.Name)
put(buf, n.Name)
return nil
})
}
@ -69,8 +71,8 @@ func copyOutAndCollect(m Environment, c *Cmd, ptc []pipeCollector) (map[string][
p := p
g.Go(func() error {
<-p.done
put(p.buffer.Bytes(), p.name)
if int64(p.buffer.Len()) > int64(p.limit) {
put(p.buffer, p.name)
if fi, err := p.buffer.Stat(); err == nil && fi.Size() > int64(p.limit) {
return runner.StatusOutputLimitExceeded
}
return nil

View File

@ -1,21 +1,20 @@
package envexec
import (
"bytes"
"io"
"os"
)
type pipeBuffer struct {
W *os.File
Buffer *bytes.Buffer
Buffer *os.File
Done <-chan struct{}
Limit Size
}
type pipeCollector struct {
done <-chan struct{}
buffer *bytes.Buffer
buffer *os.File
limit Size
name string
}
@ -36,10 +35,14 @@ func newPipe(writer io.Writer, limit Size) (<-chan struct{}, *os.File, error) {
return done, w, nil
}
func newPipeBuffer(limit Size) (*pipeBuffer, error) {
buffer := new(bytes.Buffer)
func newPipeBuffer(limit Size, newFile NewStoreFile) (*pipeBuffer, error) {
buffer, err := newFile()
if err != nil {
return nil, err
}
done, w, err := newPipe(buffer, limit+1)
if err != nil {
buffer.Close()
return nil, err
}
return &pipeBuffer{

View File

@ -1,7 +1,6 @@
package envexec
import (
"bytes"
"fmt"
"io"
"os"
@ -11,7 +10,7 @@ import (
)
// prepare Files for tty input / output
func prepareCmdFdTTY(c *Cmd, count int) (f []*os.File, p []pipeCollector, err error) {
func prepareCmdFdTTY(c *Cmd, count int, newStoreFile NewStoreFile) (f []*os.File, p []pipeCollector, err error) {
var wg sync.WaitGroup
var hasInput, hasOutput bool
@ -74,7 +73,10 @@ func prepareCmdFdTTY(c *Cmd, count int) (f []*os.File, p []pipeCollector, err er
hasOutput = true
done := make(chan struct{})
buf := new(bytes.Buffer)
buf, err := newStoreFile()
if err != nil {
return nil, nil, fmt.Errorf("filed to create store file %v", err)
}
pipeToCollect = append(pipeToCollect, pipeCollector{done, buf, t.Limit, t.Name})
wg.Add(1)
@ -110,9 +112,9 @@ func prepareCmdFdTTY(c *Cmd, count int) (f []*os.File, p []pipeCollector, err er
return files, pipeToCollect, nil
}
func prepareCmdFd(c *Cmd, count int) (f []*os.File, p []pipeCollector, err error) {
func prepareCmdFd(c *Cmd, count int, newFileStore NewStoreFile) (f []*os.File, p []pipeCollector, err error) {
if c.TTY {
return prepareCmdFdTTY(c, count)
return prepareCmdFdTTY(c, count, newFileStore)
}
files := make([]*os.File, count)
pipeToCollect := make([]pipeCollector, 0)
@ -160,7 +162,7 @@ func prepareCmdFd(c *Cmd, count int) (f []*os.File, p []pipeCollector, err error
break
}
b, err := newPipeBuffer(t.Limit)
b, err := newPipeBuffer(t.Limit, newFileStore)
if err != nil {
return nil, nil, fmt.Errorf("failed to create pipe %v", err)
}
@ -184,7 +186,7 @@ func prepareCmdFd(c *Cmd, count int) (f []*os.File, p []pipeCollector, err error
}
// prepareFd returns fds, pipeToCollect fileToClose, error
func prepareFds(r *Group) (f [][]*os.File, p [][]pipeCollector, err error) {
func prepareFds(r *Group, newStoreFile NewStoreFile) (f [][]*os.File, p [][]pipeCollector, err error) {
// prepare fd count
fdCount, err := countFd(r)
if err != nil {
@ -206,7 +208,7 @@ func prepareFds(r *Group) (f [][]*os.File, p [][]pipeCollector, err error) {
// prepare cmd fd
for i, c := range r.Cmd {
files[i], pipeToCollect[i], err = prepareCmdFd(c, fdCount[i])
files[i], pipeToCollect[i], err = prepareCmdFd(c, fdCount[i], newStoreFile)
if err != nil {
return nil, nil, err
}
@ -214,7 +216,7 @@ func prepareFds(r *Group) (f [][]*os.File, p [][]pipeCollector, err error) {
// prepare pipes
for _, p := range r.Pipes {
out, in, pc, err := pipe(p)
out, in, pc, err := pipe(p, newStoreFile)
if err != nil {
return nil, nil, err
}
@ -249,14 +251,19 @@ func countFd(r *Group) ([]int, error) {
return fdCount, nil
}
func pipe(p Pipe) (out *os.File, in *os.File, pc *pipeCollector, err error) {
func pipe(p Pipe, newStoreFile NewStoreFile) (out *os.File, in *os.File, pc *pipeCollector, err error) {
if p.Proxy {
out1, in1, out2, in2, err := pipe2()
buffer, err := newStoreFile()
if err != nil {
return nil, nil, nil, err
}
out1, in1, out2, in2, err := pipe2()
if err != nil {
buffer.Close()
return nil, nil, nil, err
}
pc := pipeProxy(p, out1, in2)
pc := pipeProxy(p, out1, in2, buffer)
return out2, in1, pc, nil
}
@ -281,7 +288,7 @@ func pipe2() (out1 *os.File, in1 *os.File, out2 *os.File, in2 *os.File, err erro
return
}
func pipeProxy(p Pipe, out1 *os.File, in2 *os.File) *pipeCollector {
func pipeProxy(p Pipe, out1 *os.File, in2 *os.File, buffer *os.File) *pipeCollector {
copyAndClose := func() {
io.Copy(in2, out1)
in2.Close()
@ -296,7 +303,6 @@ func pipeProxy(p Pipe, out1 *os.File, in2 *os.File) *pipeCollector {
}
done := make(chan struct{})
buffer := new(bytes.Buffer)
limit := p.Limit
// out1 -> in2

View File

@ -1,3 +1,4 @@
//go:build !linux
// +build !linux
package envexec

View File

@ -15,6 +15,9 @@ type Group struct {
// Pipes defines the potential mapping between Cmd.
// ensure nil is used as placeholder in correspond cmd
Pipes []Pipe
// NewStoreFile defines interface to create stored file
NewStoreFile NewStoreFile
}
// PipeIndex defines the index of cmd and the fd of the that cmd
@ -42,7 +45,7 @@ type Pipe struct {
// Run starts the cmd and returns exec results
func (r *Group) Run(ctx context.Context) ([]Result, error) {
// prepare files
fds, pipeToCollect, err := prepareFds(r)
fds, pipeToCollect, err := prepareFds(r, r.NewStoreFile)
if err != nil {
return nil, err
}
@ -53,7 +56,7 @@ func (r *Group) Run(ctx context.Context) ([]Result, error) {
for i, c := range r.Cmd {
i, c := i, c
g.Go(func() error {
r, err := runSingle(ctx, c, fds[i], pipeToCollect[i])
r, err := runSingle(ctx, c, fds[i], pipeToCollect[i], r.NewStoreFile)
result[i] = r
if err != nil {
result[i].Status = StatusInternalError

View File

@ -58,3 +58,6 @@ type Environment interface {
// Open open file at work dir with given relative path and flags
Open(path string, flags int, perm os.FileMode) (*os.File, error)
}
// NewStoreFile creates a new file in storage
type NewStoreFile func() (*os.File, error)

View File

@ -8,7 +8,7 @@ import (
)
// runSingle runs Cmd inside the given environment and cgroup
func runSingle(pc context.Context, c *Cmd, fds []*os.File, ptc []pipeCollector) (result Result, err error) {
func runSingle(pc context.Context, c *Cmd, fds []*os.File, ptc []pipeCollector, newStoreFile NewStoreFile) (result Result, err error) {
m := c.Environment
// copyin
if err := runSingleCopyIn(m, c.CopyIn); err != nil {
@ -22,7 +22,7 @@ func runSingle(pc context.Context, c *Cmd, fds []*os.File, ptc []pipeCollector)
rt := runSingleWait(pc, m, c, fds)
// collect result
files, err := copyOutAndCollect(m, c, ptc)
files, err := copyOutAndCollect(m, c, ptc, newStoreFile)
result = Result{
Status: convertStatus(rt.Status),
ExitStatus: rt.ExitStatus,

View File

@ -9,17 +9,20 @@ import (
type Single struct {
// Cmd defines Cmd running in parallel in multiple environments
Cmd *Cmd
// NewStoreFile defines interface to create stored file
NewStoreFile NewStoreFile
}
// Run starts the cmd and returns exec results
func (s *Single) Run(ctx context.Context) (result Result, err error) {
// prepare files
fd, pipeToCollect, err := prepareCmdFd(s.Cmd, len(s.Cmd.Files))
fd, pipeToCollect, err := prepareCmdFd(s.Cmd, len(s.Cmd.Files), s.NewStoreFile)
if err != nil {
return result, err
}
result, err = runSingle(ctx, s.Cmd, fd, pipeToCollect)
result, err = runSingle(ctx, s.Cmd, fd, pipeToCollect, s.NewStoreFile)
if err != nil {
result.Status = StatusInternalError
result.Error = err.Error()

View File

@ -2,8 +2,10 @@ package filestore
import (
"errors"
"fmt"
"os"
"path"
"path/filepath"
"sync"
"github.com/criyle/go-judge/envexec"
@ -18,35 +20,21 @@ type fileLocalStore struct {
// NewFileLocalStore create new local file store
func NewFileLocalStore(dir string) FileStore {
return &fileLocalStore{
dir: dir,
dir: filepath.Clean(dir),
name: make(map[string]string),
}
}
func (s *fileLocalStore) Add(name string, content []byte) (string, error) {
func (s *fileLocalStore) Add(name, path string) (string, error) {
s.mu.Lock()
defer s.mu.Unlock()
id, err := generateUniqueID(func(id string) (bool, error) {
_, err := os.Stat(path.Join(s.dir, id))
switch {
case errors.Is(err, os.ErrNotExist):
return false, nil
case err == nil:
return true, nil
default:
return false, err
}
})
if err != nil {
return "", err
if s.dir == filepath.Dir(path) {
id := filepath.Base(path)
s.name[id] = name
return id, nil
}
err = os.WriteFile(path.Join(s.dir, id), content, 0644)
if err != nil {
return "", err
}
s.name[id] = name
return id, err
return "", fmt.Errorf("add: %s does not have prefix %s", path, s.dir)
}
func (s *fileLocalStore) Get(id string) (string, envexec.File) {
@ -92,3 +80,25 @@ func (s *fileLocalStore) List() map[string]string {
}
return names
}
func (s *fileLocalStore) New() (*os.File, error) {
id, err := generateUniqueID(func(id string) (bool, error) {
_, err := os.Stat(path.Join(s.dir, id))
switch {
case errors.Is(err, os.ErrNotExist):
return false, nil
case err == nil:
return true, nil
default:
return false, err
}
})
if err != nil {
return nil, err
}
f, err := os.OpenFile(path.Join(s.dir, id), os.O_CREATE|os.O_RDWR, 0644)
if err != nil {
return nil, err
}
return f, nil
}

View File

@ -1,72 +0,0 @@
package filestore
import (
"bytes"
"sync"
"github.com/criyle/go-judge/envexec"
)
type fileMemoryStore struct {
store map[string]fileMemory
mu sync.RWMutex
}
type fileMemory struct {
name string
content []byte
}
// NewFileMemoryStore create new memory file store
func NewFileMemoryStore() FileStore {
return &fileMemoryStore{
store: make(map[string]fileMemory),
}
}
func (s *fileMemoryStore) Add(name string, content []byte) (string, error) {
s.mu.Lock()
defer s.mu.Unlock()
id, err := generateUniqueID(func(id string) (bool, error) {
_, ok := s.store[id]
return ok, nil
})
if err != nil {
return "", err
}
s.store[id] = fileMemory{name: name, content: content}
return id, err
}
func (s *fileMemoryStore) Remove(fileID string) bool {
s.mu.Lock()
defer s.mu.Unlock()
_, ok := s.store[fileID]
delete(s.store, fileID)
return ok
}
func (s *fileMemoryStore) Get(fileID string) (string, envexec.File) {
s.mu.RLock()
defer s.mu.RUnlock()
f, ok := s.store[fileID]
if !ok {
return "", nil
}
return f.name, envexec.NewFileReader(bytes.NewReader(f.content), false)
}
func (s *fileMemoryStore) List() map[string]string {
s.mu.RLock()
defer s.mu.RUnlock()
b := make(map[string]string, len(s.store))
for n, v := range s.store {
b[n] = v.name
}
return b
}

View File

@ -5,6 +5,7 @@ import (
"crypto/rand"
"encoding/base32"
"errors"
"os"
"github.com/criyle/go-judge/envexec"
)
@ -15,10 +16,11 @@ var errUniqueIDNotGenerated = errors.New("unique id does not exists after tried
// FileStore defines interface to store file
type FileStore interface {
Add(name string, content []byte) (string, error) // Add creates a file with name & content to the storage, returns id
Remove(string) bool // Remove deletes a file by id
Get(string) (string, envexec.File) // Get file by id, nil if not exists
List() map[string]string // List return all file ids to original name
Add(name, path string) (string, error) // Add creates a file with path to the storage, returns id
Remove(string) bool // Remove deletes a file by id
Get(string) (string, envexec.File) // Get file by id, nil if not exists
List() map[string]string // List return all file ids to original name
New() (*os.File, error) // Create a temporary file to the file store, can be added through Add to save it
}
func generateID() (string, error) {

View File

@ -2,6 +2,7 @@ package filestore
import (
"container/heap"
"os"
"sync"
"time"
@ -86,9 +87,9 @@ func (t *Timeout) Pop() interface{} {
return e
}
func (t *Timeout) Add(name string, content []byte) (string, error) {
func (t *Timeout) Add(name, path string) (string, error) {
// try add to file store underlying
id, err := t.FileStore.Add(name, content)
id, err := t.FileStore.Add(name, path)
if err != nil {
return "", err
}
@ -131,3 +132,7 @@ func (t *Timeout) Get(id string) (string, envexec.File) {
return name, file
}
func (t *Timeout) New() (*os.File, error) {
return t.FileStore.New()
}

28
go.mod
View File

@ -3,9 +3,9 @@ module github.com/criyle/go-judge
go 1.17
require (
github.com/creack/pty v1.1.14
github.com/criyle/go-sandbox v0.8.2
github.com/elastic/go-seccomp-bpf v1.1.0
github.com/creack/pty v1.1.15
github.com/criyle/go-sandbox v0.8.5
github.com/elastic/go-seccomp-bpf v1.2.0
github.com/elastic/go-ucfg v0.8.3
github.com/gin-contrib/pprof v1.3.0
github.com/gin-contrib/zap v0.0.1
@ -17,11 +17,11 @@ require (
github.com/koding/multiconfig v0.0.0-20171124222453-69c27309b2d7
github.com/prometheus/client_golang v1.11.0
github.com/zsais/go-gin-prometheus v0.1.0
go.uber.org/zap v1.19.0
golang.org/x/crypto v0.0.0-20210817164053-32db794688a5
golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d
go.uber.org/zap v1.19.1
golang.org/x/crypto v0.0.0-20210915214749-c084706c2272
golang.org/x/net v0.0.0-20210917221730-978cfadd31cf
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
golang.org/x/sys v0.0.0-20210818153620-00dd8d7831e7
golang.org/x/sys v0.0.0-20210917161153-d61c044b1678
google.golang.org/grpc v1.40.0
google.golang.org/protobuf v1.27.1
gopkg.in/yaml.v2 v2.4.0
@ -30,19 +30,19 @@ require (
require (
github.com/BurntSushi/toml v0.4.1 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/cespare/xxhash/v2 v2.1.1 // indirect
github.com/cespare/xxhash/v2 v2.1.2 // indirect
github.com/fatih/camelcase v1.0.0 // indirect
github.com/fatih/structs v1.1.0 // indirect
github.com/gin-contrib/sse v0.1.0 // indirect
github.com/go-playground/locales v0.14.0 // indirect
github.com/go-playground/universal-translator v0.18.0 // indirect
github.com/go-playground/validator/v10 v10.9.0 // indirect
github.com/json-iterator/go v1.1.11 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/leodido/go-urn v1.2.1 // indirect
github.com/mattn/go-isatty v0.0.13 // indirect
github.com/mattn/go-isatty v0.0.14 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.1 // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/prometheus/client_model v0.2.0 // indirect
github.com/prometheus/common v0.30.0 // indirect
@ -51,9 +51,9 @@ require (
github.com/ugorji/go/codec v1.2.6 // indirect
go.uber.org/atomic v1.9.0 // indirect
go.uber.org/multierr v1.7.0 // indirect
golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b // indirect
golang.org/x/term v0.0.0-20210916214954-140adaaadfaf // indirect
golang.org/x/text v0.3.7 // indirect
google.golang.org/genproto v0.0.0-20210818220304-27ea9cc85d9f // indirect
google.golang.org/genproto v0.0.0-20210917145530-b395a37504d4 // indirect
)
retract (
@ -62,3 +62,5 @@ retract (
// Old version, don't use
[v0.0.1, v0.9.4]
)
replace github.com/criyle/go-sandbox => ../go-sandbox

55
go.sum
View File

@ -52,8 +52,9 @@ github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6r
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY=
github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE=
github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
@ -62,15 +63,13 @@ github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGX
github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
github.com/creack/pty v1.1.14 h1:55VbUWoBxE1iTAh3B6JztD6xyQ06CvW/31oD6rYwrtY=
github.com/creack/pty v1.1.14/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4=
github.com/criyle/go-sandbox v0.8.2 h1:dnNUiM0UttklthcbeggvBXH9c3qunOV5facbYyuI1Mo=
github.com/criyle/go-sandbox v0.8.2/go.mod h1:zgKlZdlXm/ifmEKt7PvIG47IEj5dy3mSBuUIu3yUb6U=
github.com/creack/pty v1.1.15 h1:cKRCLMj3Ddm54bKSpemfQ8AtYFBhAI2MPmdys22fBdc=
github.com/creack/pty v1.1.15/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/elastic/go-seccomp-bpf v1.1.0 h1:jUzzDc6LyCtdolZdvL/26dad6rZ9vsc7xZ2eadKECAU=
github.com/elastic/go-seccomp-bpf v1.1.0/go.mod h1:l+89Vy5BzjVcaX8USZRMOwmwwDScE+vxCFzzvQwN7T8=
github.com/elastic/go-seccomp-bpf v1.2.0 h1:K5fToUAMzm0pmdlYORmw0FP0DloRa1SfqRYkum647Yk=
github.com/elastic/go-seccomp-bpf v1.2.0/go.mod h1:l+89Vy5BzjVcaX8USZRMOwmwwDScE+vxCFzzvQwN7T8=
github.com/elastic/go-ucfg v0.7.0/go.mod h1:iaiY0NBIYeasNgycLyTvhJftQlQEUO2hpF+FX0JKxzo=
github.com/elastic/go-ucfg v0.8.3 h1:leywnFjzr2QneZZWhE6uWd+QN/UpP0sdJRHYyuFvkeo=
github.com/elastic/go-ucfg v0.8.3/go.mod h1:iaiY0NBIYeasNgycLyTvhJftQlQEUO2hpF+FX0JKxzo=
@ -191,8 +190,9 @@ github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCV
github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
github.com/json-iterator/go v1.1.11 h1:uVUAXhF2To8cbw/3xN3pxj6kk7TYKs98NIrTqPlMWAQ=
github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
@ -218,16 +218,17 @@ github.com/leodido/go-urn v1.2.1 h1:BqpAaACuzVSgi/VLzGZIobT2z4v53pjosyNd9Yv6n/w=
github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY=
github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ=
github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
github.com/mattn/go-isatty v0.0.13 h1:qdl+GuBjcsKKDco5BsxPJlId98mSWNKqYA+Co0SC1yA=
github.com/mattn/go-isatty v0.0.13/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y=
github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI=
github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
@ -303,15 +304,15 @@ go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE=
go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
go.uber.org/goleak v1.1.10 h1:z+mqJhf6ss6BSfSM671tgKyZBFPTTJM+HLxnhPC3wu0=
go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A=
go.uber.org/goleak v1.1.11-0.20210813005559-691160354723 h1:sHOAIxRGBp443oHZIPB+HsUGaksVCXVQENPxwTfQdH4=
go.uber.org/goleak v1.1.11-0.20210813005559-691160354723/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ=
go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
go.uber.org/multierr v1.7.0 h1:zaiO/rmgFjbmCXdSYJWQcdvOCsthmdaHfr3Gm2Kx4Ec=
go.uber.org/multierr v1.7.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak=
go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
go.uber.org/zap v1.19.0 h1:mZQZefskPPCMIBCSEH0v2/iUqqLrYtaeqwD6FUGUnFE=
go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI=
go.uber.org/zap v1.19.1 h1:ue41HOKd1vGURxrmeKIgELGb3jPW9DMUDGtsinblHwI=
go.uber.org/zap v1.19.1/go.mod h1:j3DNczoxDZroyBnOT1L/Q79cfUMGZxlv/9dzN7SM1rI=
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
@ -319,8 +320,8 @@ golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8U
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.0.0-20210817164053-32db794688a5 h1:HWj/xjIHfjYU5nVXpTM0s39J9CbLn7Cc5a7IC5rwsMQ=
golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.0.0-20210915214749-c084706c2272 h1:3erb+vDS8lU1sxfDHF4/hhWyaXnhIaO+7RgL4fDZORA=
golang.org/x/crypto v0.0.0-20210915214749-c084706c2272/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@ -343,7 +344,6 @@ golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHl
golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs=
golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 h1:VLliZ0d+/avPrXXH+OakdXhpJuEoBZuwh1m2j7U6Iug=
golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
@ -386,8 +386,8 @@ golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwY
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d h1:LO7XpTYMwTqxjLcGWPijK3vRXg1aWdlNOVOHRq45d7c=
golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20210917221730-978cfadd31cf h1:R150MpwJIv1MpS0N/pc+NhTM8ajzvlmxlY5OYsrevXQ=
golang.org/x/net v0.0.0-20210917221730-978cfadd31cf/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@ -450,12 +450,13 @@ golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210818153620-00dd8d7831e7 h1:/bmDWM82ZX7TawqxuI8kVjKI0TXHdSY6pHJArewwHtU=
golang.org/x/sys v0.0.0-20210818153620-00dd8d7831e7/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210917161153-d61c044b1678 h1:J27LZFQBFoihqXoegpscI10HpjZ7B5WQLLKL2FZXQKw=
golang.org/x/sys v0.0.0-20210917161153-d61c044b1678/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b h1:9zKuko04nR4gjZ4+DNjHqRlAJqbJETHwiNKDqTfOjfE=
golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.0.0-20210916214954-140adaaadfaf h1:Ihq/mm/suC88gF8WFcVwk+OV6Tq+wyA1O0E5UEvDglI=
golang.org/x/term v0.0.0-20210916214954-140adaaadfaf/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@ -483,7 +484,6 @@ golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgw
golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
@ -511,7 +511,6 @@ golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc
golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.1.5 h1:ouewzE6p+/VEB31YYnTbEJdi8pFqKp4P4n85vwo3DHA=
golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@ -572,8 +571,8 @@ google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7Fc
google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
google.golang.org/genproto v0.0.0-20210818220304-27ea9cc85d9f h1:enWPderunHptc5pzJkSYGx0olpF8goXzG0rY3kL0eSg=
google.golang.org/genproto v0.0.0-20210818220304-27ea9cc85d9f/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY=
google.golang.org/genproto v0.0.0-20210917145530-b395a37504d4 h1:ysnBoUyeL/H6RCvNRhWHjKoDEmguI+mPU+qHgK8qv/w=
google.golang.org/genproto v0.0.0-20210917145530-b395a37504d4/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY=
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=

View File

@ -2,6 +2,7 @@ package worker
import (
"fmt"
"os"
"time"
"github.com/criyle/go-judge/envexec"
@ -50,7 +51,7 @@ type Result struct {
Time time.Duration
RunTime time.Duration
Memory envexec.Size
Files map[string][]byte
Files map[string]*os.File
FileIDs map[string]string
}
@ -83,7 +84,7 @@ func (r Result) String() string {
FileIDs: r.FileIDs,
}
for k, v := range r.Files {
d.Files[k] = fmt.Sprintf("(len:%d)", len(v))
d.Files[k] = fmt.Sprintf("(path:%s)", v.Name())
}
return fmt.Sprintf("%+v", d)
}

View File

@ -3,6 +3,7 @@ package worker
import (
"context"
"fmt"
"os"
"path"
"sync"
"time"
@ -176,7 +177,8 @@ func (w *worker) workDoSingle(ctx context.Context, rc Cmd) (rt Response) {
c.Environment = env
s := &envexec.Single{
Cmd: c,
Cmd: c,
NewStoreFile: w.fs.New,
}
result, err := s.Run(ctx)
if err != nil {
@ -215,8 +217,9 @@ func (w *worker) workDoGroup(ctx context.Context, rc []Cmd, pm []PipeMap) (rt Re
cs[i].Environment = env
}
g := envexec.Group{
Cmd: cs,
Pipes: pm,
Cmd: cs,
Pipes: pm,
NewStoreFile: w.fs.New,
}
results, err := g.Run(ctx)
if err != nil {
@ -239,7 +242,7 @@ func (w *worker) convertResult(result envexec.Result, cmd Cmd) (res Result) {
res.Time = result.Time
res.RunTime = result.RunTime
res.Memory = result.Memory
res.Files = make(map[string][]byte)
res.Files = make(map[string]*os.File)
res.FileIDs = make(map[string]string)
copyOutCachedSet := make(map[string]bool, len(cmd.CopyOutCached))
@ -252,13 +255,14 @@ func (w *worker) convertResult(result envexec.Result, cmd Cmd) (res Result) {
res.Files[name] = b
continue
}
id, err := w.fs.Add(name, b)
id, err := w.fs.Add(name, b.Name())
if err != nil {
res.Status = envexec.StatusFileError
res.Error = err.Error()
return
}
res.FileIDs[name] = id
b.Close()
}
return res
}