mirror of
https://github.com/criyle/go-judge.git
synced 2025-09-26 22:39:12 +08:00
filestore: use /dev/shm instead of memory as file storage
- increase default tmpfs size to 128M - add /config to get file store path - remove memory only file store close #20
This commit is contained in:
parent
9414057b8b
commit
78bb4d85de
@ -5,3 +5,7 @@ cmd = "go build -o ./tmp/executorserver ./cmd/executorserver"
|
||||
full_bin = "tmp/executorserver -enable-grpc -enable-debug"
|
||||
|
||||
include_ext = ["go"]
|
||||
|
||||
delay = 1000
|
||||
send_interrupt = true
|
||||
kill_delay = 3000
|
10
README.cn.md
10
README.cn.md
@ -8,6 +8,11 @@
|
||||
|
||||
快速,简单,安全
|
||||
|
||||
### 运行要求
|
||||
|
||||
- Linux 内核版本 >= 3.10
|
||||
- 系统 Cgroup V1 文件系统挂载于 `/sys/fs/cgroup`(Systemd 默认)。Cgroup V2 还未支持
|
||||
|
||||
### 系统架构
|
||||
|
||||
```text
|
||||
@ -35,6 +40,7 @@
|
||||
- /metrics 提供 prometheus 版监控 (使用 `ES_ENABLE_METRICS=1` 环境变量开启)
|
||||
- /debug 提供 go 语言调试接口 (使用 `ES_ENABLE_DEBUG=1` 环境变量开启)
|
||||
- /version 得到本程序编译版本和 go 语言运行时版本
|
||||
- /config 得到本程序部分运行参数
|
||||
|
||||
### 命令行参数
|
||||
|
||||
@ -81,7 +87,7 @@
|
||||
或者使用 docker
|
||||
|
||||
```bash
|
||||
docker run -it --rm --privileged -p 5050:5050 criyle/executorserver
|
||||
docker run -it --rm --privileged --shm-size=256m -p 5050:5050 criyle/executorserver
|
||||
```
|
||||
|
||||
#### 编译 docker
|
||||
@ -126,6 +132,8 @@ docker run -it --rm --privileged -p 5050:5050 criyle/executorserver
|
||||
|
||||
使用 `mount.yaml` 定制容器文件系统。
|
||||
|
||||
`/w` 的 `/tmp` 挂载 `tmpfs` 大小通过 `-tmp-fs-param` 指定,默认值为 `size=128m,nr_inodes=4k`
|
||||
|
||||
### 包
|
||||
|
||||
- envexec: 核心逻辑包,在提供的环境中运行一个或多个程序
|
||||
|
32
README.md
32
README.md
@ -8,18 +8,23 @@
|
||||
|
||||
Fast, Simple, Secure
|
||||
|
||||
### Prerequisite
|
||||
|
||||
- Linux Kernel Version >= 3.10
|
||||
- Cgroup file system V1 mounted at /sys/fs/cgroup. Cgroup V2 is not supported
|
||||
|
||||
### Architecture
|
||||
|
||||
```text
|
||||
+---------------------------------------------------------------------------+
|
||||
| Transport Layer (HTTP / WebSocket / FFI / ...) |
|
||||
+---------------------------------------------------------------------------+
|
||||
| Executor Worker (Environment Pool w/ Environment Builder ) |
|
||||
+-----------------------------------------------------------+---------------+
|
||||
| EnvExec | File Store |
|
||||
+--------------------+----------------+---------------------+--------+------+
|
||||
| Linux (go-sandbox) | Windows (winc) | macOS (app sandbox) | Memory | Disk |
|
||||
+--------------------+----------------+---------------------+--------+------+
|
||||
+----------------------------------------------------------------------------------+
|
||||
| Transport Layer (HTTP / WebSocket / FFI / ...) |
|
||||
+----------------------------------------------------------------------------------+
|
||||
| Executor Worker (Environment Pool w/ Environment Builder ) |
|
||||
+-----------------------------------------------------------+----------------------+
|
||||
| EnvExec | File Store |
|
||||
+--------------------+----------------+---------------------+---------------+------+
|
||||
| Linux (go-sandbox) | Windows (winc) | macOS (app sandbox) | Shared Memory | Disk |
|
||||
+--------------------+----------------+---------------------+---------------+------+
|
||||
```
|
||||
|
||||
### REST API
|
||||
@ -35,6 +40,7 @@ A REST service to run program in restricted environment and it is basically a wr
|
||||
- /metrics prometheus metrics (specifies `ES_ENABLE_METRICS=1` environment variable to enable metrics)
|
||||
- /debug (specifies `ES_ENABLE_DEBUG=1` environment variable to enable go runtime debug endpoint)
|
||||
- /version gets build git version (e.g. `v0.9.4`) together with runtime information (go version, os, platform)
|
||||
- /config gets some configuration (e.g. `fileStorePath`) together with some supported features
|
||||
|
||||
### Command Line Arguments
|
||||
|
||||
@ -84,7 +90,7 @@ Download compiled executable from [Release](https://github.com/criyle/go-judge/r
|
||||
Or, by docker
|
||||
|
||||
```bash
|
||||
docker run -it --rm --privileged -p 5050:5050 criyle/executorserver
|
||||
docker run -it --rm --privileged --shm-size=256m -p 5050:5050 criyle/executorserver
|
||||
```
|
||||
|
||||
#### Build Executor Server
|
||||
@ -145,6 +151,8 @@ For linux platform, the default mounts points are bind mounting host's `/lib`, `
|
||||
|
||||
To customize mount points, please look at example `mount.yaml` file.
|
||||
|
||||
`tmpfs` size for `/w` and `/tmp` is configured through `-tmp-fs-param` with default value `size=128m,nr_inodes=4k`
|
||||
|
||||
### Packages
|
||||
|
||||
- envexec: run single / group of programs in parallel within restricted environment and resource constraints
|
||||
@ -185,9 +193,9 @@ sysctl -p
|
||||
|
||||
#### Memory Usage
|
||||
|
||||
The controller will consume `60M` memory and each container will consume `20M` + size of tmpfs `2 * 16M`. For each request, it consumes as much as user program limit + extra limit (`16k`) + total copy out max * 2.
|
||||
The controller will consume `60M` memory and each container will consume `20M` + size of tmpfs `2 * 128M`. For each request, it consumes as much as user program limit + extra limit (`16k`) + total copy out max.
|
||||
|
||||
For example, when concurrency = 4, the executor itself can consume as much as `60 + (20+32) * 4M = 268M` + 8 * total copy out + total max memory of requests.
|
||||
For example, when concurrency = 4, the executor itself can consume as much as `60 + (20+32) * 4M = 268M` + 4 * total copy out + total max memory of requests.
|
||||
|
||||
Due to limitation of GO runtime, the memory will not return to OS automatically, which could lead to OOM killer. The background worker was introduced to checks heap usage and invokes GC when necessary.
|
||||
|
||||
|
@ -14,7 +14,7 @@ type Config struct {
|
||||
// container
|
||||
ContainerInitPath string `flagUsage:"container init path"`
|
||||
PreFork int `flagUsage:"control # of the prefork workers" default:"1"`
|
||||
TmpFsParam string `flagUsage:"tmpfs mount data (only for default mount with no mount.yaml)" default:"size=16m,nr_inodes=4k"`
|
||||
TmpFsParam string `flagUsage:"tmpfs mount data (only for default mount with no mount.yaml)" default:"size=128m,nr_inodes=4k"`
|
||||
NetShare bool `flagUsage:"share net namespace with host"`
|
||||
MountConf string `flagUsage:"specifies mount configuration file" default:"mount.yaml"`
|
||||
SeccompConf string `flagUsage:"specifies seccomp filter" default:"seccomp.yaml"`
|
||||
@ -49,7 +49,7 @@ type Config struct {
|
||||
Silent bool `flagUsage:"do not print logs"`
|
||||
|
||||
// fix for high memory usage
|
||||
ForceGCTarget *envexec.Size `flagUsage:"specifies force GC trigger heap size" default:"10m"`
|
||||
ForceGCTarget *envexec.Size `flagUsage:"specifies force GC trigger heap size" default:"20m"`
|
||||
ForceGCInterval time.Duration `flagUsage:"specifies force GC trigger interval" default:"5s"`
|
||||
}
|
||||
|
||||
|
@ -10,6 +10,7 @@ import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/criyle/go-judge/cmd/executorserver/model"
|
||||
"github.com/criyle/go-judge/envexec"
|
||||
"github.com/criyle/go-judge/filestore"
|
||||
"github.com/criyle/go-judge/pb"
|
||||
@ -60,7 +61,13 @@ func (e *execServer) Exec(ctx context.Context, req *pb.Request) (*pb.Response, e
|
||||
if rt.Error != nil {
|
||||
return nil, rt.Error
|
||||
}
|
||||
return convertPBResponse(rt), nil
|
||||
ret, err := model.ConvertResponse(rt)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer ret.Close()
|
||||
|
||||
return convertPBResponse(ret)
|
||||
}
|
||||
|
||||
func (e *execServer) FileList(c context.Context, n *emptypb.Empty) (*pb.FileListType, error) {
|
||||
@ -88,9 +95,18 @@ func (e *execServer) FileGet(c context.Context, f *pb.FileID) (*pb.FileContent,
|
||||
}
|
||||
|
||||
func (e *execServer) FileAdd(c context.Context, fc *pb.FileContent) (*pb.FileID, error) {
|
||||
fid, err := e.fs.Add(fc.GetName(), fc.GetContent())
|
||||
f, err := e.fs.New()
|
||||
if err != nil {
|
||||
return nil, status.Error(codes.InvalidArgument, err.Error())
|
||||
return nil, status.Error(codes.Internal, err.Error())
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
if _, err := f.Write(fc.GetContent()); err != nil {
|
||||
return nil, status.Error(codes.Internal, err.Error())
|
||||
}
|
||||
fid, err := e.fs.Add(fc.GetName(), f.Name())
|
||||
if err != nil {
|
||||
return nil, status.Error(codes.Internal, err.Error())
|
||||
}
|
||||
return &pb.FileID{
|
||||
FileID: fid,
|
||||
@ -105,21 +121,23 @@ func (e *execServer) FileDelete(c context.Context, f *pb.FileID) (*emptypb.Empty
|
||||
return &emptypb.Empty{}, nil
|
||||
}
|
||||
|
||||
func convertPBResponse(r worker.Response) *pb.Response {
|
||||
func convertPBResponse(r model.Response) (*pb.Response, error) {
|
||||
res := &pb.Response{
|
||||
RequestID: r.RequestID,
|
||||
Results: make([]*pb.Response_Result, 0, len(r.Results)),
|
||||
Error: r.ErrorMsg,
|
||||
}
|
||||
for _, c := range r.Results {
|
||||
res.Results = append(res.Results, convertPBResult(c))
|
||||
rt, err := convertPBResult(c)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
res.Results = append(res.Results, rt)
|
||||
}
|
||||
if r.Error != nil {
|
||||
res.Error = r.Error.Error()
|
||||
}
|
||||
return res
|
||||
return res, nil
|
||||
}
|
||||
|
||||
func convertPBResult(r worker.Result) *pb.Response_Result {
|
||||
func convertPBResult(r model.Result) (*pb.Response_Result, error) {
|
||||
return &pb.Response_Result{
|
||||
Status: pb.Response_Result_StatusType(r.Status),
|
||||
ExitStatus: int32(r.ExitStatus),
|
||||
@ -127,9 +145,9 @@ func convertPBResult(r worker.Result) *pb.Response_Result {
|
||||
Time: uint64(r.Time),
|
||||
RunTime: uint64(r.RunTime),
|
||||
Memory: uint64(r.Memory),
|
||||
Files: r.Files,
|
||||
Files: r.Buffs,
|
||||
FileIDs: r.FileIDs,
|
||||
}
|
||||
}, nil
|
||||
}
|
||||
|
||||
func convertPBRequest(r *pb.Request, srcPrefix string) (req *worker.Request, streamIn []*fileStreamIn, streamOut []*fileStreamOut, err error) {
|
||||
|
@ -1,3 +1,4 @@
|
||||
//go:build !linux && !darwin
|
||||
// +build !linux,!darwin
|
||||
|
||||
package grpcexecutor
|
||||
|
@ -6,6 +6,7 @@ import (
|
||||
"io"
|
||||
"sync"
|
||||
|
||||
"github.com/criyle/go-judge/cmd/executorserver/model"
|
||||
"github.com/criyle/go-judge/pb"
|
||||
"github.com/criyle/go-judge/worker"
|
||||
"go.uber.org/zap"
|
||||
@ -99,9 +100,19 @@ func execStreamLoop(es pb.Executor_ExecStreamServer, errCh chan error, outCh cha
|
||||
|
||||
case rt := <-rtCh:
|
||||
logger.Sugar().Debugf("response: %+v", rt)
|
||||
ret, err := model.ConvertResponse(rt)
|
||||
if err != nil {
|
||||
return status.Errorf(codes.Aborted, "response: %v", err)
|
||||
}
|
||||
defer ret.Close()
|
||||
|
||||
resp, err := convertPBResponse(ret)
|
||||
if err != nil {
|
||||
return status.Errorf(codes.Aborted, "response: %v", err)
|
||||
}
|
||||
return es.Send(&pb.StreamResponse{
|
||||
Response: &pb.StreamResponse_ExecResponse{
|
||||
ExecResponse: convertPBResponse(rt),
|
||||
ExecResponse: resp,
|
||||
},
|
||||
})
|
||||
}
|
||||
|
@ -1,3 +1,4 @@
|
||||
//go:build linux || darwin
|
||||
// +build linux darwin
|
||||
|
||||
package grpcexecutor
|
||||
|
@ -51,7 +51,11 @@ func main() {
|
||||
logger.Sugar().Infof("config loaded: %+v", conf)
|
||||
|
||||
// Init environment pool
|
||||
fs := newFilsStore(conf.Dir, conf.FileTimeout, conf.EnableMetrics)
|
||||
fs, fsDir, fsCleanUp, err := newFilsStore(conf.Dir, conf.FileTimeout, conf.EnableMetrics)
|
||||
if err != nil {
|
||||
logger.Sugar().Fatalf("Create temp dir failed %v", err)
|
||||
}
|
||||
conf.Dir = fsDir
|
||||
b := newEnvBuilder(conf)
|
||||
envPool := newEnvPool(b, conf.EnableMetrics)
|
||||
prefork(envPool, conf.PreFork)
|
||||
@ -67,9 +71,13 @@ func main() {
|
||||
Handler: r,
|
||||
}
|
||||
|
||||
// Gracefully shutdown, with signal / HTTP server / gRPC server
|
||||
sig := make(chan os.Signal, 3)
|
||||
|
||||
go func() {
|
||||
logger.Sugar().Info("Starting http server at ", conf.HTTPAddr)
|
||||
logger.Sugar().Info("Http serve finished: ", srv.ListenAndServe())
|
||||
logger.Sugar().Info("Http server stopped: ", srv.ListenAndServe())
|
||||
sig <- os.Interrupt
|
||||
}()
|
||||
|
||||
// Init gRPC server
|
||||
@ -84,7 +92,8 @@ func main() {
|
||||
}
|
||||
go func() {
|
||||
logger.Sugar().Info("Starting gRPC server at ", conf.GRPCAddr)
|
||||
logger.Sugar().Info("gRPC serve finished: ", grpcServer.Serve(lis))
|
||||
logger.Sugar().Info("gRPC server stopped: ", grpcServer.Serve(lis))
|
||||
sig <- os.Interrupt
|
||||
}()
|
||||
}
|
||||
|
||||
@ -92,9 +101,9 @@ func main() {
|
||||
newForceGCWorker(conf)
|
||||
|
||||
// Graceful shutdown...
|
||||
sig := make(chan os.Signal, 1)
|
||||
signal.Notify(sig, os.Interrupt)
|
||||
<-sig
|
||||
signal.Reset(os.Interrupt)
|
||||
|
||||
logger.Sugar().Info("Shutting Down...")
|
||||
|
||||
@ -121,6 +130,14 @@ func main() {
|
||||
})
|
||||
}
|
||||
|
||||
if fsCleanUp != nil {
|
||||
eg.Go(func() error {
|
||||
err := fsCleanUp()
|
||||
logger.Sugar().Info("FileStore clean up")
|
||||
return err
|
||||
})
|
||||
}
|
||||
|
||||
go func() {
|
||||
logger.Sugar().Info("Shutdown Finished: ", eg.Wait())
|
||||
cancel()
|
||||
@ -197,6 +214,9 @@ func initHTTPMux(conf *config.Config, work worker.Worker, fs filestore.FileStore
|
||||
// Version handle
|
||||
r.GET("/version", handleVersion)
|
||||
|
||||
// Config handle
|
||||
r.GET("/config", generateHandleConfig(conf))
|
||||
|
||||
// Add auth token
|
||||
if conf.AuthToken != "" {
|
||||
r.Use(tokenAuth(conf.AuthToken))
|
||||
@ -279,23 +299,35 @@ func grpcTokenAuth(token string) func(context.Context) (context.Context, error)
|
||||
}
|
||||
}
|
||||
|
||||
func newFilsStore(dir string, fileTimeout time.Duration, enableMetrics bool) filestore.FileStore {
|
||||
func newFilsStore(dir string, fileTimeout time.Duration, enableMetrics bool) (filestore.FileStore, string, func() error, error) {
|
||||
const timeoutCheckInterval = 15 * time.Second
|
||||
var cleanUp func() error
|
||||
|
||||
var fs filestore.FileStore
|
||||
if dir == "" {
|
||||
fs = filestore.NewFileMemoryStore()
|
||||
} else {
|
||||
os.MkdirAll(dir, 0755)
|
||||
fs = filestore.NewFileLocalStore(dir)
|
||||
if runtime.GOOS == "linux" {
|
||||
dir = "/dev/shm"
|
||||
} else {
|
||||
dir = os.TempDir()
|
||||
}
|
||||
var err error
|
||||
dir, err = os.MkdirTemp(dir, "executorserver")
|
||||
if err != nil {
|
||||
return nil, "", cleanUp, err
|
||||
}
|
||||
cleanUp = func() error {
|
||||
return os.RemoveAll(dir)
|
||||
}
|
||||
}
|
||||
os.MkdirAll(dir, 0755)
|
||||
fs = filestore.NewFileLocalStore(dir)
|
||||
if enableMetrics {
|
||||
fs = newMetricsFileStore(fs)
|
||||
}
|
||||
if fileTimeout > 0 {
|
||||
fs = filestore.NewTimeout(fs, fileTimeout, timeoutCheckInterval)
|
||||
}
|
||||
return fs
|
||||
return fs, dir, cleanUp, nil
|
||||
}
|
||||
|
||||
func newEnvBuilder(conf *config.Config) pool.EnvBuilder {
|
||||
@ -370,3 +402,13 @@ func handleVersion(c *gin.Context) {
|
||||
"pipeProxy": true,
|
||||
})
|
||||
}
|
||||
|
||||
func generateHandleConfig(conf *config.Config) func(*gin.Context) {
|
||||
return func(c *gin.Context) {
|
||||
c.JSON(http.StatusOK, gin.H{
|
||||
"copyOutOptional": true,
|
||||
"pipeProxy": true,
|
||||
"fileStorePath": conf.Dir,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
@ -1,6 +1,7 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
@ -131,18 +132,18 @@ var _ filestore.FileStore = &metricsFileStore{}
|
||||
type metricsFileStore struct {
|
||||
mu sync.Mutex
|
||||
filestore.FileStore
|
||||
fileSize map[string]int
|
||||
fileSize map[string]int64
|
||||
}
|
||||
|
||||
func newMetricsFileStore(fs filestore.FileStore) filestore.FileStore {
|
||||
return &metricsFileStore{
|
||||
FileStore: fs,
|
||||
fileSize: make(map[string]int),
|
||||
fileSize: make(map[string]int64),
|
||||
}
|
||||
}
|
||||
|
||||
func (m *metricsFileStore) Add(name string, content []byte) (string, error) {
|
||||
id, err := m.FileStore.Add(name, content)
|
||||
func (m *metricsFileStore) Add(name, path string) (string, error) {
|
||||
id, err := m.FileStore.Add(name, path)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
@ -150,7 +151,12 @@ func (m *metricsFileStore) Add(name string, content []byte) (string, error) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
|
||||
s := len(content)
|
||||
fi, err := os.Stat(path)
|
||||
if err != nil {
|
||||
return id, nil
|
||||
}
|
||||
|
||||
s := fi.Size()
|
||||
m.fileSize[id] = s
|
||||
|
||||
sf := float64(s)
|
||||
|
@ -95,6 +95,9 @@ type Result struct {
|
||||
RunTime uint64 `json:"runTime"`
|
||||
Files map[string]string `json:"files,omitempty"`
|
||||
FileIDs map[string]string `json:"fileIds,omitempty"`
|
||||
|
||||
files []string
|
||||
Buffs map[string][]byte `json:"-"`
|
||||
}
|
||||
|
||||
// Response defines worker response for single request
|
||||
@ -104,19 +107,55 @@ type Response struct {
|
||||
ErrorMsg string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
func (r *Response) Close() {
|
||||
for _, res := range r.Results {
|
||||
res.Close()
|
||||
}
|
||||
}
|
||||
|
||||
func (r *Result) Close() {
|
||||
// remove temporary files
|
||||
for _, f := range r.files {
|
||||
os.Remove(f)
|
||||
}
|
||||
// remove potential mmap
|
||||
for _, b := range r.Buffs {
|
||||
releaseByte(b)
|
||||
}
|
||||
}
|
||||
|
||||
// ConvertResponse converts
|
||||
func ConvertResponse(r worker.Response) Response {
|
||||
ret := Response{
|
||||
func ConvertResponse(r worker.Response) (ret Response, err error) {
|
||||
// in error case, release all resources
|
||||
defer func() {
|
||||
if err != nil {
|
||||
for _, r := range ret.Results {
|
||||
r.Close()
|
||||
}
|
||||
for _, r := range r.Results {
|
||||
for _, f := range r.Files {
|
||||
f.Close()
|
||||
os.Remove(f.Name())
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
ret = Response{
|
||||
RequestID: r.RequestID,
|
||||
Results: make([]Result, 0, len(r.Results)),
|
||||
}
|
||||
for _, r := range r.Results {
|
||||
ret.Results = append(ret.Results, convertResult(r))
|
||||
res, err := convertResult(r)
|
||||
if err != nil {
|
||||
return ret, err
|
||||
}
|
||||
ret.Results = append(ret.Results, res)
|
||||
}
|
||||
if r.Error != nil {
|
||||
ret.ErrorMsg = r.Error.Error()
|
||||
}
|
||||
return ret
|
||||
return ret, nil
|
||||
}
|
||||
|
||||
// ConvertRequest converts json request into worker request
|
||||
@ -139,7 +178,7 @@ func ConvertRequest(r *Request, srcPrefix string) (*worker.Request, error) {
|
||||
return req, nil
|
||||
}
|
||||
|
||||
func convertResult(r worker.Result) Result {
|
||||
func convertResult(r worker.Result) (Result, error) {
|
||||
res := Result{
|
||||
Status: Status(r.Status),
|
||||
ExitStatus: r.ExitStatus,
|
||||
@ -151,11 +190,19 @@ func convertResult(r worker.Result) Result {
|
||||
}
|
||||
if r.Files != nil {
|
||||
res.Files = make(map[string]string)
|
||||
for k, v := range r.Files {
|
||||
res.Files[k] = byteArrayToString(v)
|
||||
res.Buffs = make(map[string][]byte)
|
||||
for k, f := range r.Files {
|
||||
b, err := fileToByte(f)
|
||||
if err != nil {
|
||||
return res, err
|
||||
}
|
||||
res.Files[k] = byteArrayToString(b)
|
||||
|
||||
res.files = append(res.files, f.Name())
|
||||
res.Buffs[k] = b
|
||||
}
|
||||
}
|
||||
return res
|
||||
return res, nil
|
||||
}
|
||||
|
||||
func convertPipe(p PipeMap) worker.PipeMap {
|
||||
|
32
cmd/executorserver/model/util_linux.go
Normal file
32
cmd/executorserver/model/util_linux.go
Normal file
@ -0,0 +1,32 @@
|
||||
package model
|
||||
|
||||
import (
|
||||
"os"
|
||||
"syscall"
|
||||
)
|
||||
|
||||
func fileToByte(f *os.File) ([]byte, error) {
|
||||
defer f.Close()
|
||||
|
||||
var s int64
|
||||
if fi, err := f.Stat(); err != nil {
|
||||
return nil, err
|
||||
} else {
|
||||
s = fi.Size()
|
||||
}
|
||||
if s == 0 {
|
||||
return []byte{}, nil
|
||||
}
|
||||
|
||||
b, err := syscall.Mmap(int(f.Fd()), 0, int(s), syscall.PROT_READ, syscall.MAP_SHARED)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return b, nil
|
||||
}
|
||||
|
||||
func releaseByte(b []byte) {
|
||||
if len(b) > 0 {
|
||||
_ = syscall.Munmap(b)
|
||||
}
|
||||
}
|
30
cmd/executorserver/model/util_others.go
Normal file
30
cmd/executorserver/model/util_others.go
Normal file
@ -0,0 +1,30 @@
|
||||
//go:build !linux
|
||||
// +build !linux
|
||||
|
||||
package model
|
||||
|
||||
import (
|
||||
"io"
|
||||
"os"
|
||||
)
|
||||
|
||||
func fileToByte(f *os.File) ([]byte, error) {
|
||||
if _, err := f.Seek(0, 0); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var s int64
|
||||
if fi, err := f.Stat(); err != nil {
|
||||
return nil, err
|
||||
} else {
|
||||
s = fi.Size()
|
||||
}
|
||||
c := make([]byte, s)
|
||||
if _, err := io.ReadFull(f, c); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
f.Close()
|
||||
return c, nil
|
||||
}
|
||||
|
||||
func releaseByte(b []byte) {
|
||||
}
|
@ -74,12 +74,18 @@ func (h *handle) handleRun(c *gin.Context) {
|
||||
}
|
||||
|
||||
// encode json directly to avoid allocation
|
||||
// c.JSON(http.StatusOK, model.ConvertResponse(rt).Results)
|
||||
|
||||
c.Status(http.StatusOK)
|
||||
c.Header("Content-Type", "application/json; charset=utf-8")
|
||||
|
||||
if err := json.NewEncoder(c.Writer).Encode(model.ConvertResponse(rt).Results); err != nil {
|
||||
res, err := model.ConvertResponse(rt)
|
||||
if err != nil {
|
||||
c.Error(err)
|
||||
c.AbortWithStatusJSON(http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
defer res.Close()
|
||||
|
||||
if err := json.NewEncoder(c.Writer).Encode(res.Results); err != nil {
|
||||
c.Error(err)
|
||||
}
|
||||
}
|
||||
|
@ -33,13 +33,18 @@ func (f *fileHandle) filePost(c *gin.Context) {
|
||||
c.AbortWithError(http.StatusInternalServerError, err)
|
||||
return
|
||||
}
|
||||
b, err := io.ReadAll(fi)
|
||||
sf, err := f.fs.New()
|
||||
if err != nil {
|
||||
c.AbortWithError(http.StatusInternalServerError, err)
|
||||
return
|
||||
}
|
||||
defer sf.Close()
|
||||
|
||||
id, err := f.fs.Add(fh.Filename, b)
|
||||
if _, err := sf.ReadFrom(fi); err != nil {
|
||||
c.AbortWithError(http.StatusInternalServerError, err)
|
||||
return
|
||||
}
|
||||
id, err := f.fs.Add(fh.Filename, sf.Name())
|
||||
if err != nil {
|
||||
c.AbortWithError(http.StatusInternalServerError, err)
|
||||
return
|
||||
|
@ -1,3 +1,4 @@
|
||||
//go:build ignore
|
||||
// +build ignore
|
||||
|
||||
package main
|
||||
|
@ -2,4 +2,4 @@
|
||||
|
||||
package version
|
||||
|
||||
const Version = `v1.1.7`
|
||||
const Version = `v1.2.7`
|
@ -81,7 +81,9 @@ func (h *wsHandle) handleWS(c *gin.Context) {
|
||||
}
|
||||
go func() {
|
||||
ret := <-h.worker.Submit(ctx, r)
|
||||
resultCh <- model.ConvertResponse(ret)
|
||||
resp, _ := model.ConvertResponse(ret)
|
||||
resp.Close()
|
||||
resultCh <- resp
|
||||
}()
|
||||
}
|
||||
}()
|
||||
|
@ -1,3 +1,4 @@
|
||||
//go:build !linux
|
||||
// +build !linux
|
||||
|
||||
package main
|
||||
|
@ -9,6 +9,7 @@ import (
|
||||
"io"
|
||||
"log"
|
||||
"os"
|
||||
"runtime"
|
||||
"time"
|
||||
|
||||
"github.com/criyle/go-judge/cmd/executorserver/model"
|
||||
@ -40,14 +41,16 @@ var (
|
||||
)
|
||||
|
||||
func newFilsStore(dir string) filestore.FileStore {
|
||||
var fs filestore.FileStore
|
||||
if dir == "" {
|
||||
fs = filestore.NewFileMemoryStore()
|
||||
} else {
|
||||
os.MkdirAll(dir, 0755)
|
||||
fs = filestore.NewFileLocalStore(dir)
|
||||
if runtime.GOOS == "linux" {
|
||||
dir = "/dev/shm"
|
||||
} else {
|
||||
dir = os.TempDir()
|
||||
}
|
||||
dir, _ = os.MkdirTemp(dir, "executorserver")
|
||||
}
|
||||
return fs
|
||||
os.MkdirAll(dir, 0755)
|
||||
return filestore.NewFileLocalStore(dir)
|
||||
}
|
||||
|
||||
// Init initialize the sandbox environment
|
||||
@ -114,11 +117,13 @@ func Exec(e *C.char) *C.char {
|
||||
return nil
|
||||
}
|
||||
rt := <-work.Submit(context.TODO(), r)
|
||||
ret := model.ConvertResponse(rt)
|
||||
ret, _ := model.ConvertResponse(rt)
|
||||
var buf bytes.Buffer
|
||||
if err := json.NewEncoder(&buf).Encode(ret); err != nil {
|
||||
return nil
|
||||
}
|
||||
defer ret.Close()
|
||||
|
||||
return C.CString(buf.String())
|
||||
}
|
||||
|
||||
@ -143,12 +148,21 @@ func FileAdd(e *C.char) *C.char {
|
||||
}
|
||||
es := C.GoString(e)
|
||||
|
||||
var f fileAdd
|
||||
if err := json.NewDecoder(bytes.NewBufferString(es)).Decode(&f); err != nil {
|
||||
var fa fileAdd
|
||||
if err := json.NewDecoder(bytes.NewBufferString(es)).Decode(&fa); err != nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
id, err := fs.Add(f.Name, []byte(f.Content))
|
||||
f, err := fs.New()
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
if _, err := f.Write([]byte(fa.Content)); err != nil {
|
||||
return nil
|
||||
}
|
||||
id, err := fs.Add(fa.Name, f.Name())
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
|
1
env/env_others.go
vendored
1
env/env_others.go
vendored
@ -1,3 +1,4 @@
|
||||
//go:build !windows && !linux && !darwin
|
||||
// +build !windows,!linux,!darwin
|
||||
|
||||
package env
|
||||
|
1
env/noseccomp_linux.go
vendored
1
env/noseccomp_linux.go
vendored
@ -1,3 +1,4 @@
|
||||
//go:build !seccomp
|
||||
// +build !seccomp
|
||||
|
||||
package env
|
||||
|
1
env/seccomp_linux.go
vendored
1
env/seccomp_linux.go
vendored
@ -1,3 +1,4 @@
|
||||
//go:build seccomp
|
||||
// +build seccomp
|
||||
|
||||
package env
|
||||
|
@ -2,6 +2,7 @@ package envexec
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/criyle/go-sandbox/runner"
|
||||
@ -70,5 +71,5 @@ type Result struct {
|
||||
Memory Size // byte
|
||||
|
||||
// Files stores copy out files
|
||||
Files map[string][]byte
|
||||
Files map[string]*os.File
|
||||
}
|
||||
|
@ -1,7 +1,6 @@
|
||||
package envexec
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
@ -13,13 +12,13 @@ import (
|
||||
)
|
||||
|
||||
// copyOutAndCollect reads file and pipes in parallel from container
|
||||
func copyOutAndCollect(m Environment, c *Cmd, ptc []pipeCollector) (map[string][]byte, error) {
|
||||
func copyOutAndCollect(m Environment, c *Cmd, ptc []pipeCollector, newStoreFile NewStoreFile) (map[string]*os.File, error) {
|
||||
var (
|
||||
g errgroup.Group
|
||||
l sync.Mutex
|
||||
)
|
||||
rt := make(map[string][]byte)
|
||||
put := func(f []byte, n string) {
|
||||
rt := make(map[string]*os.File)
|
||||
put := func(f *os.File, n string) {
|
||||
l.Lock()
|
||||
defer l.Unlock()
|
||||
rt[n] = f
|
||||
@ -51,15 +50,18 @@ func copyOutAndCollect(m Environment, c *Cmd, ptc []pipeCollector) (map[string][
|
||||
if c.CopyOutMax > 0 && s > int64(c.CopyOutMax) {
|
||||
return fmt.Errorf("%s: size (%d) exceeded the limit (%d)", n.Name, s, c.CopyOutMax)
|
||||
}
|
||||
var buf bytes.Buffer
|
||||
buf.Grow(int(s))
|
||||
// create store file
|
||||
buf, err := newStoreFile()
|
||||
if err != nil {
|
||||
return fmt.Errorf("%s: failed to create store file %v", n.Name, err)
|
||||
}
|
||||
|
||||
// Ensure not copy over file size
|
||||
_, err = buf.ReadFrom(io.LimitReader(cf, s))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
put(buf.Bytes(), n.Name)
|
||||
put(buf, n.Name)
|
||||
return nil
|
||||
})
|
||||
}
|
||||
@ -69,8 +71,8 @@ func copyOutAndCollect(m Environment, c *Cmd, ptc []pipeCollector) (map[string][
|
||||
p := p
|
||||
g.Go(func() error {
|
||||
<-p.done
|
||||
put(p.buffer.Bytes(), p.name)
|
||||
if int64(p.buffer.Len()) > int64(p.limit) {
|
||||
put(p.buffer, p.name)
|
||||
if fi, err := p.buffer.Stat(); err == nil && fi.Size() > int64(p.limit) {
|
||||
return runner.StatusOutputLimitExceeded
|
||||
}
|
||||
return nil
|
||||
|
@ -1,21 +1,20 @@
|
||||
package envexec
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"io"
|
||||
"os"
|
||||
)
|
||||
|
||||
type pipeBuffer struct {
|
||||
W *os.File
|
||||
Buffer *bytes.Buffer
|
||||
Buffer *os.File
|
||||
Done <-chan struct{}
|
||||
Limit Size
|
||||
}
|
||||
|
||||
type pipeCollector struct {
|
||||
done <-chan struct{}
|
||||
buffer *bytes.Buffer
|
||||
buffer *os.File
|
||||
limit Size
|
||||
name string
|
||||
}
|
||||
@ -36,10 +35,14 @@ func newPipe(writer io.Writer, limit Size) (<-chan struct{}, *os.File, error) {
|
||||
return done, w, nil
|
||||
}
|
||||
|
||||
func newPipeBuffer(limit Size) (*pipeBuffer, error) {
|
||||
buffer := new(bytes.Buffer)
|
||||
func newPipeBuffer(limit Size, newFile NewStoreFile) (*pipeBuffer, error) {
|
||||
buffer, err := newFile()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
done, w, err := newPipe(buffer, limit+1)
|
||||
if err != nil {
|
||||
buffer.Close()
|
||||
return nil, err
|
||||
}
|
||||
return &pipeBuffer{
|
||||
|
@ -1,7 +1,6 @@
|
||||
package envexec
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
@ -11,7 +10,7 @@ import (
|
||||
)
|
||||
|
||||
// prepare Files for tty input / output
|
||||
func prepareCmdFdTTY(c *Cmd, count int) (f []*os.File, p []pipeCollector, err error) {
|
||||
func prepareCmdFdTTY(c *Cmd, count int, newStoreFile NewStoreFile) (f []*os.File, p []pipeCollector, err error) {
|
||||
var wg sync.WaitGroup
|
||||
var hasInput, hasOutput bool
|
||||
|
||||
@ -74,7 +73,10 @@ func prepareCmdFdTTY(c *Cmd, count int) (f []*os.File, p []pipeCollector, err er
|
||||
hasOutput = true
|
||||
|
||||
done := make(chan struct{})
|
||||
buf := new(bytes.Buffer)
|
||||
buf, err := newStoreFile()
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("filed to create store file %v", err)
|
||||
}
|
||||
pipeToCollect = append(pipeToCollect, pipeCollector{done, buf, t.Limit, t.Name})
|
||||
|
||||
wg.Add(1)
|
||||
@ -110,9 +112,9 @@ func prepareCmdFdTTY(c *Cmd, count int) (f []*os.File, p []pipeCollector, err er
|
||||
return files, pipeToCollect, nil
|
||||
}
|
||||
|
||||
func prepareCmdFd(c *Cmd, count int) (f []*os.File, p []pipeCollector, err error) {
|
||||
func prepareCmdFd(c *Cmd, count int, newFileStore NewStoreFile) (f []*os.File, p []pipeCollector, err error) {
|
||||
if c.TTY {
|
||||
return prepareCmdFdTTY(c, count)
|
||||
return prepareCmdFdTTY(c, count, newFileStore)
|
||||
}
|
||||
files := make([]*os.File, count)
|
||||
pipeToCollect := make([]pipeCollector, 0)
|
||||
@ -160,7 +162,7 @@ func prepareCmdFd(c *Cmd, count int) (f []*os.File, p []pipeCollector, err error
|
||||
break
|
||||
}
|
||||
|
||||
b, err := newPipeBuffer(t.Limit)
|
||||
b, err := newPipeBuffer(t.Limit, newFileStore)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("failed to create pipe %v", err)
|
||||
}
|
||||
@ -184,7 +186,7 @@ func prepareCmdFd(c *Cmd, count int) (f []*os.File, p []pipeCollector, err error
|
||||
}
|
||||
|
||||
// prepareFd returns fds, pipeToCollect fileToClose, error
|
||||
func prepareFds(r *Group) (f [][]*os.File, p [][]pipeCollector, err error) {
|
||||
func prepareFds(r *Group, newStoreFile NewStoreFile) (f [][]*os.File, p [][]pipeCollector, err error) {
|
||||
// prepare fd count
|
||||
fdCount, err := countFd(r)
|
||||
if err != nil {
|
||||
@ -206,7 +208,7 @@ func prepareFds(r *Group) (f [][]*os.File, p [][]pipeCollector, err error) {
|
||||
|
||||
// prepare cmd fd
|
||||
for i, c := range r.Cmd {
|
||||
files[i], pipeToCollect[i], err = prepareCmdFd(c, fdCount[i])
|
||||
files[i], pipeToCollect[i], err = prepareCmdFd(c, fdCount[i], newStoreFile)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
@ -214,7 +216,7 @@ func prepareFds(r *Group) (f [][]*os.File, p [][]pipeCollector, err error) {
|
||||
|
||||
// prepare pipes
|
||||
for _, p := range r.Pipes {
|
||||
out, in, pc, err := pipe(p)
|
||||
out, in, pc, err := pipe(p, newStoreFile)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
@ -249,14 +251,19 @@ func countFd(r *Group) ([]int, error) {
|
||||
return fdCount, nil
|
||||
}
|
||||
|
||||
func pipe(p Pipe) (out *os.File, in *os.File, pc *pipeCollector, err error) {
|
||||
func pipe(p Pipe, newStoreFile NewStoreFile) (out *os.File, in *os.File, pc *pipeCollector, err error) {
|
||||
if p.Proxy {
|
||||
out1, in1, out2, in2, err := pipe2()
|
||||
buffer, err := newStoreFile()
|
||||
if err != nil {
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
out1, in1, out2, in2, err := pipe2()
|
||||
if err != nil {
|
||||
buffer.Close()
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
|
||||
pc := pipeProxy(p, out1, in2)
|
||||
pc := pipeProxy(p, out1, in2, buffer)
|
||||
return out2, in1, pc, nil
|
||||
}
|
||||
|
||||
@ -281,7 +288,7 @@ func pipe2() (out1 *os.File, in1 *os.File, out2 *os.File, in2 *os.File, err erro
|
||||
return
|
||||
}
|
||||
|
||||
func pipeProxy(p Pipe, out1 *os.File, in2 *os.File) *pipeCollector {
|
||||
func pipeProxy(p Pipe, out1 *os.File, in2 *os.File, buffer *os.File) *pipeCollector {
|
||||
copyAndClose := func() {
|
||||
io.Copy(in2, out1)
|
||||
in2.Close()
|
||||
@ -296,7 +303,6 @@ func pipeProxy(p Pipe, out1 *os.File, in2 *os.File) *pipeCollector {
|
||||
}
|
||||
|
||||
done := make(chan struct{})
|
||||
buffer := new(bytes.Buffer)
|
||||
limit := p.Limit
|
||||
|
||||
// out1 -> in2
|
||||
|
@ -1,3 +1,4 @@
|
||||
//go:build !linux
|
||||
// +build !linux
|
||||
|
||||
package envexec
|
||||
|
@ -15,6 +15,9 @@ type Group struct {
|
||||
// Pipes defines the potential mapping between Cmd.
|
||||
// ensure nil is used as placeholder in correspond cmd
|
||||
Pipes []Pipe
|
||||
|
||||
// NewStoreFile defines interface to create stored file
|
||||
NewStoreFile NewStoreFile
|
||||
}
|
||||
|
||||
// PipeIndex defines the index of cmd and the fd of the that cmd
|
||||
@ -42,7 +45,7 @@ type Pipe struct {
|
||||
// Run starts the cmd and returns exec results
|
||||
func (r *Group) Run(ctx context.Context) ([]Result, error) {
|
||||
// prepare files
|
||||
fds, pipeToCollect, err := prepareFds(r)
|
||||
fds, pipeToCollect, err := prepareFds(r, r.NewStoreFile)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@ -53,7 +56,7 @@ func (r *Group) Run(ctx context.Context) ([]Result, error) {
|
||||
for i, c := range r.Cmd {
|
||||
i, c := i, c
|
||||
g.Go(func() error {
|
||||
r, err := runSingle(ctx, c, fds[i], pipeToCollect[i])
|
||||
r, err := runSingle(ctx, c, fds[i], pipeToCollect[i], r.NewStoreFile)
|
||||
result[i] = r
|
||||
if err != nil {
|
||||
result[i].Status = StatusInternalError
|
||||
|
@ -58,3 +58,6 @@ type Environment interface {
|
||||
// Open open file at work dir with given relative path and flags
|
||||
Open(path string, flags int, perm os.FileMode) (*os.File, error)
|
||||
}
|
||||
|
||||
// NewStoreFile creates a new file in storage
|
||||
type NewStoreFile func() (*os.File, error)
|
||||
|
@ -8,7 +8,7 @@ import (
|
||||
)
|
||||
|
||||
// runSingle runs Cmd inside the given environment and cgroup
|
||||
func runSingle(pc context.Context, c *Cmd, fds []*os.File, ptc []pipeCollector) (result Result, err error) {
|
||||
func runSingle(pc context.Context, c *Cmd, fds []*os.File, ptc []pipeCollector, newStoreFile NewStoreFile) (result Result, err error) {
|
||||
m := c.Environment
|
||||
// copyin
|
||||
if err := runSingleCopyIn(m, c.CopyIn); err != nil {
|
||||
@ -22,7 +22,7 @@ func runSingle(pc context.Context, c *Cmd, fds []*os.File, ptc []pipeCollector)
|
||||
rt := runSingleWait(pc, m, c, fds)
|
||||
|
||||
// collect result
|
||||
files, err := copyOutAndCollect(m, c, ptc)
|
||||
files, err := copyOutAndCollect(m, c, ptc, newStoreFile)
|
||||
result = Result{
|
||||
Status: convertStatus(rt.Status),
|
||||
ExitStatus: rt.ExitStatus,
|
||||
|
@ -9,17 +9,20 @@ import (
|
||||
type Single struct {
|
||||
// Cmd defines Cmd running in parallel in multiple environments
|
||||
Cmd *Cmd
|
||||
|
||||
// NewStoreFile defines interface to create stored file
|
||||
NewStoreFile NewStoreFile
|
||||
}
|
||||
|
||||
// Run starts the cmd and returns exec results
|
||||
func (s *Single) Run(ctx context.Context) (result Result, err error) {
|
||||
// prepare files
|
||||
fd, pipeToCollect, err := prepareCmdFd(s.Cmd, len(s.Cmd.Files))
|
||||
fd, pipeToCollect, err := prepareCmdFd(s.Cmd, len(s.Cmd.Files), s.NewStoreFile)
|
||||
if err != nil {
|
||||
return result, err
|
||||
}
|
||||
|
||||
result, err = runSingle(ctx, s.Cmd, fd, pipeToCollect)
|
||||
result, err = runSingle(ctx, s.Cmd, fd, pipeToCollect, s.NewStoreFile)
|
||||
if err != nil {
|
||||
result.Status = StatusInternalError
|
||||
result.Error = err.Error()
|
||||
|
@ -2,8 +2,10 @@ package filestore
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
|
||||
"github.com/criyle/go-judge/envexec"
|
||||
@ -18,35 +20,21 @@ type fileLocalStore struct {
|
||||
// NewFileLocalStore create new local file store
|
||||
func NewFileLocalStore(dir string) FileStore {
|
||||
return &fileLocalStore{
|
||||
dir: dir,
|
||||
dir: filepath.Clean(dir),
|
||||
name: make(map[string]string),
|
||||
}
|
||||
}
|
||||
|
||||
func (s *fileLocalStore) Add(name string, content []byte) (string, error) {
|
||||
func (s *fileLocalStore) Add(name, path string) (string, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
|
||||
id, err := generateUniqueID(func(id string) (bool, error) {
|
||||
_, err := os.Stat(path.Join(s.dir, id))
|
||||
switch {
|
||||
case errors.Is(err, os.ErrNotExist):
|
||||
return false, nil
|
||||
case err == nil:
|
||||
return true, nil
|
||||
default:
|
||||
return false, err
|
||||
}
|
||||
})
|
||||
if err != nil {
|
||||
return "", err
|
||||
if s.dir == filepath.Dir(path) {
|
||||
id := filepath.Base(path)
|
||||
s.name[id] = name
|
||||
return id, nil
|
||||
}
|
||||
err = os.WriteFile(path.Join(s.dir, id), content, 0644)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
s.name[id] = name
|
||||
return id, err
|
||||
return "", fmt.Errorf("add: %s does not have prefix %s", path, s.dir)
|
||||
}
|
||||
|
||||
func (s *fileLocalStore) Get(id string) (string, envexec.File) {
|
||||
@ -92,3 +80,25 @@ func (s *fileLocalStore) List() map[string]string {
|
||||
}
|
||||
return names
|
||||
}
|
||||
|
||||
func (s *fileLocalStore) New() (*os.File, error) {
|
||||
id, err := generateUniqueID(func(id string) (bool, error) {
|
||||
_, err := os.Stat(path.Join(s.dir, id))
|
||||
switch {
|
||||
case errors.Is(err, os.ErrNotExist):
|
||||
return false, nil
|
||||
case err == nil:
|
||||
return true, nil
|
||||
default:
|
||||
return false, err
|
||||
}
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
f, err := os.OpenFile(path.Join(s.dir, id), os.O_CREATE|os.O_RDWR, 0644)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return f, nil
|
||||
}
|
||||
|
@ -1,72 +0,0 @@
|
||||
package filestore
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"sync"
|
||||
|
||||
"github.com/criyle/go-judge/envexec"
|
||||
)
|
||||
|
||||
type fileMemoryStore struct {
|
||||
store map[string]fileMemory
|
||||
mu sync.RWMutex
|
||||
}
|
||||
|
||||
type fileMemory struct {
|
||||
name string
|
||||
content []byte
|
||||
}
|
||||
|
||||
// NewFileMemoryStore create new memory file store
|
||||
func NewFileMemoryStore() FileStore {
|
||||
return &fileMemoryStore{
|
||||
store: make(map[string]fileMemory),
|
||||
}
|
||||
}
|
||||
|
||||
func (s *fileMemoryStore) Add(name string, content []byte) (string, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
|
||||
id, err := generateUniqueID(func(id string) (bool, error) {
|
||||
_, ok := s.store[id]
|
||||
return ok, nil
|
||||
})
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
s.store[id] = fileMemory{name: name, content: content}
|
||||
return id, err
|
||||
}
|
||||
|
||||
func (s *fileMemoryStore) Remove(fileID string) bool {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
|
||||
_, ok := s.store[fileID]
|
||||
delete(s.store, fileID)
|
||||
return ok
|
||||
}
|
||||
|
||||
func (s *fileMemoryStore) Get(fileID string) (string, envexec.File) {
|
||||
s.mu.RLock()
|
||||
defer s.mu.RUnlock()
|
||||
|
||||
f, ok := s.store[fileID]
|
||||
if !ok {
|
||||
return "", nil
|
||||
}
|
||||
return f.name, envexec.NewFileReader(bytes.NewReader(f.content), false)
|
||||
}
|
||||
|
||||
func (s *fileMemoryStore) List() map[string]string {
|
||||
s.mu.RLock()
|
||||
defer s.mu.RUnlock()
|
||||
|
||||
b := make(map[string]string, len(s.store))
|
||||
for n, v := range s.store {
|
||||
b[n] = v.name
|
||||
}
|
||||
return b
|
||||
}
|
@ -5,6 +5,7 @@ import (
|
||||
"crypto/rand"
|
||||
"encoding/base32"
|
||||
"errors"
|
||||
"os"
|
||||
|
||||
"github.com/criyle/go-judge/envexec"
|
||||
)
|
||||
@ -15,10 +16,11 @@ var errUniqueIDNotGenerated = errors.New("unique id does not exists after tried
|
||||
|
||||
// FileStore defines interface to store file
|
||||
type FileStore interface {
|
||||
Add(name string, content []byte) (string, error) // Add creates a file with name & content to the storage, returns id
|
||||
Remove(string) bool // Remove deletes a file by id
|
||||
Get(string) (string, envexec.File) // Get file by id, nil if not exists
|
||||
List() map[string]string // List return all file ids to original name
|
||||
Add(name, path string) (string, error) // Add creates a file with path to the storage, returns id
|
||||
Remove(string) bool // Remove deletes a file by id
|
||||
Get(string) (string, envexec.File) // Get file by id, nil if not exists
|
||||
List() map[string]string // List return all file ids to original name
|
||||
New() (*os.File, error) // Create a temporary file to the file store, can be added through Add to save it
|
||||
}
|
||||
|
||||
func generateID() (string, error) {
|
||||
|
@ -2,6 +2,7 @@ package filestore
|
||||
|
||||
import (
|
||||
"container/heap"
|
||||
"os"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
@ -86,9 +87,9 @@ func (t *Timeout) Pop() interface{} {
|
||||
return e
|
||||
}
|
||||
|
||||
func (t *Timeout) Add(name string, content []byte) (string, error) {
|
||||
func (t *Timeout) Add(name, path string) (string, error) {
|
||||
// try add to file store underlying
|
||||
id, err := t.FileStore.Add(name, content)
|
||||
id, err := t.FileStore.Add(name, path)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
@ -131,3 +132,7 @@ func (t *Timeout) Get(id string) (string, envexec.File) {
|
||||
|
||||
return name, file
|
||||
}
|
||||
|
||||
func (t *Timeout) New() (*os.File, error) {
|
||||
return t.FileStore.New()
|
||||
}
|
||||
|
28
go.mod
28
go.mod
@ -3,9 +3,9 @@ module github.com/criyle/go-judge
|
||||
go 1.17
|
||||
|
||||
require (
|
||||
github.com/creack/pty v1.1.14
|
||||
github.com/criyle/go-sandbox v0.8.2
|
||||
github.com/elastic/go-seccomp-bpf v1.1.0
|
||||
github.com/creack/pty v1.1.15
|
||||
github.com/criyle/go-sandbox v0.8.5
|
||||
github.com/elastic/go-seccomp-bpf v1.2.0
|
||||
github.com/elastic/go-ucfg v0.8.3
|
||||
github.com/gin-contrib/pprof v1.3.0
|
||||
github.com/gin-contrib/zap v0.0.1
|
||||
@ -17,11 +17,11 @@ require (
|
||||
github.com/koding/multiconfig v0.0.0-20171124222453-69c27309b2d7
|
||||
github.com/prometheus/client_golang v1.11.0
|
||||
github.com/zsais/go-gin-prometheus v0.1.0
|
||||
go.uber.org/zap v1.19.0
|
||||
golang.org/x/crypto v0.0.0-20210817164053-32db794688a5
|
||||
golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d
|
||||
go.uber.org/zap v1.19.1
|
||||
golang.org/x/crypto v0.0.0-20210915214749-c084706c2272
|
||||
golang.org/x/net v0.0.0-20210917221730-978cfadd31cf
|
||||
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
|
||||
golang.org/x/sys v0.0.0-20210818153620-00dd8d7831e7
|
||||
golang.org/x/sys v0.0.0-20210917161153-d61c044b1678
|
||||
google.golang.org/grpc v1.40.0
|
||||
google.golang.org/protobuf v1.27.1
|
||||
gopkg.in/yaml.v2 v2.4.0
|
||||
@ -30,19 +30,19 @@ require (
|
||||
require (
|
||||
github.com/BurntSushi/toml v0.4.1 // indirect
|
||||
github.com/beorn7/perks v1.0.1 // indirect
|
||||
github.com/cespare/xxhash/v2 v2.1.1 // indirect
|
||||
github.com/cespare/xxhash/v2 v2.1.2 // indirect
|
||||
github.com/fatih/camelcase v1.0.0 // indirect
|
||||
github.com/fatih/structs v1.1.0 // indirect
|
||||
github.com/gin-contrib/sse v0.1.0 // indirect
|
||||
github.com/go-playground/locales v0.14.0 // indirect
|
||||
github.com/go-playground/universal-translator v0.18.0 // indirect
|
||||
github.com/go-playground/validator/v10 v10.9.0 // indirect
|
||||
github.com/json-iterator/go v1.1.11 // indirect
|
||||
github.com/json-iterator/go v1.1.12 // indirect
|
||||
github.com/leodido/go-urn v1.2.1 // indirect
|
||||
github.com/mattn/go-isatty v0.0.13 // indirect
|
||||
github.com/mattn/go-isatty v0.0.14 // indirect
|
||||
github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
|
||||
github.com/modern-go/reflect2 v1.0.1 // indirect
|
||||
github.com/modern-go/reflect2 v1.0.2 // indirect
|
||||
github.com/pkg/errors v0.9.1 // indirect
|
||||
github.com/prometheus/client_model v0.2.0 // indirect
|
||||
github.com/prometheus/common v0.30.0 // indirect
|
||||
@ -51,9 +51,9 @@ require (
|
||||
github.com/ugorji/go/codec v1.2.6 // indirect
|
||||
go.uber.org/atomic v1.9.0 // indirect
|
||||
go.uber.org/multierr v1.7.0 // indirect
|
||||
golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b // indirect
|
||||
golang.org/x/term v0.0.0-20210916214954-140adaaadfaf // indirect
|
||||
golang.org/x/text v0.3.7 // indirect
|
||||
google.golang.org/genproto v0.0.0-20210818220304-27ea9cc85d9f // indirect
|
||||
google.golang.org/genproto v0.0.0-20210917145530-b395a37504d4 // indirect
|
||||
)
|
||||
|
||||
retract (
|
||||
@ -62,3 +62,5 @@ retract (
|
||||
// Old version, don't use
|
||||
[v0.0.1, v0.9.4]
|
||||
)
|
||||
|
||||
replace github.com/criyle/go-sandbox => ../go-sandbox
|
||||
|
55
go.sum
55
go.sum
@ -52,8 +52,9 @@ github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6r
|
||||
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
|
||||
github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
|
||||
github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
|
||||
github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY=
|
||||
github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE=
|
||||
github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
|
||||
github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
|
||||
github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
|
||||
@ -62,15 +63,13 @@ github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGX
|
||||
github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
|
||||
github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
|
||||
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
|
||||
github.com/creack/pty v1.1.14 h1:55VbUWoBxE1iTAh3B6JztD6xyQ06CvW/31oD6rYwrtY=
|
||||
github.com/creack/pty v1.1.14/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4=
|
||||
github.com/criyle/go-sandbox v0.8.2 h1:dnNUiM0UttklthcbeggvBXH9c3qunOV5facbYyuI1Mo=
|
||||
github.com/criyle/go-sandbox v0.8.2/go.mod h1:zgKlZdlXm/ifmEKt7PvIG47IEj5dy3mSBuUIu3yUb6U=
|
||||
github.com/creack/pty v1.1.15 h1:cKRCLMj3Ddm54bKSpemfQ8AtYFBhAI2MPmdys22fBdc=
|
||||
github.com/creack/pty v1.1.15/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/elastic/go-seccomp-bpf v1.1.0 h1:jUzzDc6LyCtdolZdvL/26dad6rZ9vsc7xZ2eadKECAU=
|
||||
github.com/elastic/go-seccomp-bpf v1.1.0/go.mod h1:l+89Vy5BzjVcaX8USZRMOwmwwDScE+vxCFzzvQwN7T8=
|
||||
github.com/elastic/go-seccomp-bpf v1.2.0 h1:K5fToUAMzm0pmdlYORmw0FP0DloRa1SfqRYkum647Yk=
|
||||
github.com/elastic/go-seccomp-bpf v1.2.0/go.mod h1:l+89Vy5BzjVcaX8USZRMOwmwwDScE+vxCFzzvQwN7T8=
|
||||
github.com/elastic/go-ucfg v0.7.0/go.mod h1:iaiY0NBIYeasNgycLyTvhJftQlQEUO2hpF+FX0JKxzo=
|
||||
github.com/elastic/go-ucfg v0.8.3 h1:leywnFjzr2QneZZWhE6uWd+QN/UpP0sdJRHYyuFvkeo=
|
||||
github.com/elastic/go-ucfg v0.8.3/go.mod h1:iaiY0NBIYeasNgycLyTvhJftQlQEUO2hpF+FX0JKxzo=
|
||||
@ -191,8 +190,9 @@ github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCV
|
||||
github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
|
||||
github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
|
||||
github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
|
||||
github.com/json-iterator/go v1.1.11 h1:uVUAXhF2To8cbw/3xN3pxj6kk7TYKs98NIrTqPlMWAQ=
|
||||
github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
|
||||
github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
|
||||
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
|
||||
github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
|
||||
github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
|
||||
github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
|
||||
@ -218,16 +218,17 @@ github.com/leodido/go-urn v1.2.1 h1:BqpAaACuzVSgi/VLzGZIobT2z4v53pjosyNd9Yv6n/w=
|
||||
github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY=
|
||||
github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ=
|
||||
github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
|
||||
github.com/mattn/go-isatty v0.0.13 h1:qdl+GuBjcsKKDco5BsxPJlId98mSWNKqYA+Co0SC1yA=
|
||||
github.com/mattn/go-isatty v0.0.13/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
|
||||
github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y=
|
||||
github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
|
||||
github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
|
||||
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
|
||||
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
|
||||
github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
|
||||
github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI=
|
||||
github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
|
||||
github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
|
||||
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
|
||||
github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
|
||||
github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
|
||||
github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
|
||||
@ -303,15 +304,15 @@ go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
|
||||
go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
|
||||
go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE=
|
||||
go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
|
||||
go.uber.org/goleak v1.1.10 h1:z+mqJhf6ss6BSfSM671tgKyZBFPTTJM+HLxnhPC3wu0=
|
||||
go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A=
|
||||
go.uber.org/goleak v1.1.11-0.20210813005559-691160354723 h1:sHOAIxRGBp443oHZIPB+HsUGaksVCXVQENPxwTfQdH4=
|
||||
go.uber.org/goleak v1.1.11-0.20210813005559-691160354723/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ=
|
||||
go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
|
||||
go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
|
||||
go.uber.org/multierr v1.7.0 h1:zaiO/rmgFjbmCXdSYJWQcdvOCsthmdaHfr3Gm2Kx4Ec=
|
||||
go.uber.org/multierr v1.7.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak=
|
||||
go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
|
||||
go.uber.org/zap v1.19.0 h1:mZQZefskPPCMIBCSEH0v2/iUqqLrYtaeqwD6FUGUnFE=
|
||||
go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI=
|
||||
go.uber.org/zap v1.19.1 h1:ue41HOKd1vGURxrmeKIgELGb3jPW9DMUDGtsinblHwI=
|
||||
go.uber.org/zap v1.19.1/go.mod h1:j3DNczoxDZroyBnOT1L/Q79cfUMGZxlv/9dzN7SM1rI=
|
||||
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
@ -319,8 +320,8 @@ golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8U
|
||||
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
|
||||
golang.org/x/crypto v0.0.0-20210817164053-32db794688a5 h1:HWj/xjIHfjYU5nVXpTM0s39J9CbLn7Cc5a7IC5rwsMQ=
|
||||
golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
|
||||
golang.org/x/crypto v0.0.0-20210915214749-c084706c2272 h1:3erb+vDS8lU1sxfDHF4/hhWyaXnhIaO+7RgL4fDZORA=
|
||||
golang.org/x/crypto v0.0.0-20210915214749-c084706c2272/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
|
||||
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
|
||||
@ -343,7 +344,6 @@ golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHl
|
||||
golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs=
|
||||
golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
|
||||
golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
|
||||
golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 h1:VLliZ0d+/avPrXXH+OakdXhpJuEoBZuwh1m2j7U6Iug=
|
||||
golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
|
||||
golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
|
||||
golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
|
||||
@ -386,8 +386,8 @@ golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwY
|
||||
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
|
||||
golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
|
||||
golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d h1:LO7XpTYMwTqxjLcGWPijK3vRXg1aWdlNOVOHRq45d7c=
|
||||
golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
golang.org/x/net v0.0.0-20210917221730-978cfadd31cf h1:R150MpwJIv1MpS0N/pc+NhTM8ajzvlmxlY5OYsrevXQ=
|
||||
golang.org/x/net v0.0.0-20210917221730-978cfadd31cf/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
@ -450,12 +450,13 @@ golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7w
|
||||
golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20210818153620-00dd8d7831e7 h1:/bmDWM82ZX7TawqxuI8kVjKI0TXHdSY6pHJArewwHtU=
|
||||
golang.org/x/sys v0.0.0-20210818153620-00dd8d7831e7/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20210917161153-d61c044b1678 h1:J27LZFQBFoihqXoegpscI10HpjZ7B5WQLLKL2FZXQKw=
|
||||
golang.org/x/sys v0.0.0-20210917161153-d61c044b1678/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b h1:9zKuko04nR4gjZ4+DNjHqRlAJqbJETHwiNKDqTfOjfE=
|
||||
golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
|
||||
golang.org/x/term v0.0.0-20210916214954-140adaaadfaf h1:Ihq/mm/suC88gF8WFcVwk+OV6Tq+wyA1O0E5UEvDglI=
|
||||
golang.org/x/term v0.0.0-20210916214954-140adaaadfaf/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
|
||||
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
@ -483,7 +484,6 @@ golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgw
|
||||
golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
@ -511,7 +511,6 @@ golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc
|
||||
golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
|
||||
golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
|
||||
golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
|
||||
golang.org/x/tools v0.1.5 h1:ouewzE6p+/VEB31YYnTbEJdi8pFqKp4P4n85vwo3DHA=
|
||||
golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
|
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
@ -572,8 +571,8 @@ google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7Fc
|
||||
google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
|
||||
google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
|
||||
google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
|
||||
google.golang.org/genproto v0.0.0-20210818220304-27ea9cc85d9f h1:enWPderunHptc5pzJkSYGx0olpF8goXzG0rY3kL0eSg=
|
||||
google.golang.org/genproto v0.0.0-20210818220304-27ea9cc85d9f/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY=
|
||||
google.golang.org/genproto v0.0.0-20210917145530-b395a37504d4 h1:ysnBoUyeL/H6RCvNRhWHjKoDEmguI+mPU+qHgK8qv/w=
|
||||
google.golang.org/genproto v0.0.0-20210917145530-b395a37504d4/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY=
|
||||
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
|
||||
google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
|
||||
google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
|
||||
|
@ -2,6 +2,7 @@ package worker
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/criyle/go-judge/envexec"
|
||||
@ -50,7 +51,7 @@ type Result struct {
|
||||
Time time.Duration
|
||||
RunTime time.Duration
|
||||
Memory envexec.Size
|
||||
Files map[string][]byte
|
||||
Files map[string]*os.File
|
||||
FileIDs map[string]string
|
||||
}
|
||||
|
||||
@ -83,7 +84,7 @@ func (r Result) String() string {
|
||||
FileIDs: r.FileIDs,
|
||||
}
|
||||
for k, v := range r.Files {
|
||||
d.Files[k] = fmt.Sprintf("(len:%d)", len(v))
|
||||
d.Files[k] = fmt.Sprintf("(path:%s)", v.Name())
|
||||
}
|
||||
return fmt.Sprintf("%+v", d)
|
||||
}
|
||||
|
@ -3,6 +3,7 @@ package worker
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"path"
|
||||
"sync"
|
||||
"time"
|
||||
@ -176,7 +177,8 @@ func (w *worker) workDoSingle(ctx context.Context, rc Cmd) (rt Response) {
|
||||
c.Environment = env
|
||||
|
||||
s := &envexec.Single{
|
||||
Cmd: c,
|
||||
Cmd: c,
|
||||
NewStoreFile: w.fs.New,
|
||||
}
|
||||
result, err := s.Run(ctx)
|
||||
if err != nil {
|
||||
@ -215,8 +217,9 @@ func (w *worker) workDoGroup(ctx context.Context, rc []Cmd, pm []PipeMap) (rt Re
|
||||
cs[i].Environment = env
|
||||
}
|
||||
g := envexec.Group{
|
||||
Cmd: cs,
|
||||
Pipes: pm,
|
||||
Cmd: cs,
|
||||
Pipes: pm,
|
||||
NewStoreFile: w.fs.New,
|
||||
}
|
||||
results, err := g.Run(ctx)
|
||||
if err != nil {
|
||||
@ -239,7 +242,7 @@ func (w *worker) convertResult(result envexec.Result, cmd Cmd) (res Result) {
|
||||
res.Time = result.Time
|
||||
res.RunTime = result.RunTime
|
||||
res.Memory = result.Memory
|
||||
res.Files = make(map[string][]byte)
|
||||
res.Files = make(map[string]*os.File)
|
||||
res.FileIDs = make(map[string]string)
|
||||
|
||||
copyOutCachedSet := make(map[string]bool, len(cmd.CopyOutCached))
|
||||
@ -252,13 +255,14 @@ func (w *worker) convertResult(result envexec.Result, cmd Cmd) (res Result) {
|
||||
res.Files[name] = b
|
||||
continue
|
||||
}
|
||||
id, err := w.fs.Add(name, b)
|
||||
id, err := w.fs.Add(name, b.Name())
|
||||
if err != nil {
|
||||
res.Status = envexec.StatusFileError
|
||||
res.Error = err.Error()
|
||||
return
|
||||
}
|
||||
res.FileIDs[name] = id
|
||||
b.Close()
|
||||
}
|
||||
return res
|
||||
}
|
||||
|
Loading…
Reference in New Issue
Block a user