envexec: refactor to not use empty interface

This commit is contained in:
criyle 2021-03-13 16:08:51 -08:00
parent ba9ddc610a
commit 2acca7d71c
35 changed files with 660 additions and 578 deletions

View File

@ -3,13 +3,13 @@ package grpcexecutor
import (
"context"
"fmt"
"io"
"os"
"path/filepath"
"strings"
"sync"
"time"
"github.com/creack/pty"
"github.com/criyle/go-judge/envexec"
"github.com/criyle/go-judge/filestore"
"github.com/criyle/go-judge/pb"
@ -70,13 +70,19 @@ func (e *execServer) FileList(c context.Context, n *emptypb.Empty) (*pb.FileList
}
func (e *execServer) FileGet(c context.Context, f *pb.FileID) (*pb.FileContent, error) {
file := e.fs.Get(f.GetFileID())
content, err := file.Content()
name, file := e.fs.Get(f.GetFileID())
r, err := envexec.FileToReader(file)
if err != nil {
return nil, err
}
defer r.Close()
content, err := io.ReadAll(r)
if err != nil {
return nil, err
}
return &pb.FileContent{
Name: file.Name(),
Name: name,
Content: content,
}, nil
}
@ -202,52 +208,17 @@ func convertPBCmd(c *pb.Request_CmdType, srcPrefix string) (cm worker.Cmd, strea
CopyOutMax: c.GetCopyOutMax(),
CopyOutDir: c.GetCopyOutDir(),
}
var (
fPty, fTty *os.File
ttyOut *fileStreamOut
)
for _, f := range c.GetFiles() {
var cf worker.CmdFile
switch fi := f.File.(type) {
case *pb.Request_File_StreamIn:
var si *fileStreamIn
if c.Tty {
fPty, fTty, err = pty.Open()
if err != nil {
return cm, streamIn, streamOut, err
}
si = &fileStreamIn{
name: fi.StreamIn.GetName(),
w: fPty,
r: fTty,
}
streamIn = append(streamIn, si)
} else {
si, err = newFileStreamIn(fi.StreamIn.GetName())
if err == nil {
streamIn = append(streamIn, si)
}
}
si := newFileStreamIn(fi.StreamIn.GetName(), c.GetTty())
streamIn = append(streamIn, si)
cf = si
case *pb.Request_File_StreamOut:
var so *fileStreamOut
if fPty != nil {
if ttyOut == nil {
ttyOut = &fileStreamOut{
name: fi.StreamOut.GetName(),
w: fTty,
r: fPty,
}
streamOut = append(streamOut, ttyOut)
}
so = ttyOut
} else {
so, err = newFileStreamOut(fi.StreamOut.GetName())
if err == nil {
streamOut = append(streamOut, so)
}
}
so := newFileStreamOut(fi.StreamOut.GetName())
streamOut = append(streamOut, so)
cf = so
default:
@ -291,7 +262,7 @@ func convertPBFile(c *pb.Request_File, srcPrefix string) (worker.CmdFile, error)
case *pb.Request_File_Cached:
return &worker.CachedFile{FileID: c.Cached.GetFileID()}, nil
case *pb.Request_File_Pipe:
return &worker.PipeCollector{Name: c.Pipe.GetName(), Max: c.Pipe.GetMax()}, nil
return &worker.PipeCollector{Name: c.Pipe.GetName(), Max: envexec.Size(c.Pipe.GetMax())}, nil
}
return nil, fmt.Errorf("request file type not supported yet %v", c)
}

View File

@ -2,22 +2,52 @@ package grpcexecutor
import (
"fmt"
"io"
"math"
"os"
"github.com/criyle/go-judge/envexec"
"github.com/criyle/go-judge/filestore"
"github.com/criyle/go-judge/worker"
)
var (
_ worker.CmdFile = &fileStreamIn{}
_ worker.CmdFile = &fileStreamOut{}
)
type fileStreamIn struct {
name string
r, w *os.File
name string
r io.ReadCloser
w *io.PipeWriter
tty *os.File
done chan struct{}
hasTTY bool
}
func newFileStreamIn(name string) (*fileStreamIn, error) {
r, w, err := os.Pipe()
if err != nil {
return nil, err
type fileStreamInReader struct {
*io.PipeReader
fi *fileStreamIn
}
func (f *fileStreamInReader) TTY(tty *os.File) {
f.fi.tty = tty
close(f.fi.done)
}
func newFileStreamIn(name string, hasTTY bool) *fileStreamIn {
r, w := io.Pipe()
fi := &fileStreamIn{name: name, w: w, done: make(chan struct{}), hasTTY: hasTTY}
fi.r = &fileStreamInReader{r, fi}
return fi
}
func (f *fileStreamIn) GetTTY() *os.File {
if !f.hasTTY {
return nil
}
return &fileStreamIn{name: name, r: r, w: w}, nil
<-f.done
return f.tty
}
func (f *fileStreamIn) Name() string {
@ -28,8 +58,8 @@ func (f *fileStreamIn) Write(b []byte) (int, error) {
return f.w.Write(b)
}
func (f *fileStreamIn) EnvFile(fs filestore.FileStore) (interface{}, error) {
return f.r, nil
func (f *fileStreamIn) EnvFile(fs filestore.FileStore) (envexec.File, error) {
return envexec.NewFileReader(f.r, true), nil
}
func (f *fileStreamIn) String() string {
@ -43,15 +73,13 @@ func (f *fileStreamIn) Close() error {
type fileStreamOut struct {
name string
r, w *os.File
r *io.PipeReader
w *io.PipeWriter
}
func newFileStreamOut(name string) (*fileStreamOut, error) {
r, w, err := os.Pipe()
if err != nil {
return nil, err
}
return &fileStreamOut{name: name, r: r, w: w}, nil
func newFileStreamOut(name string) *fileStreamOut {
r, w := io.Pipe()
return &fileStreamOut{name: name, r: r, w: w}
}
func (f *fileStreamOut) Name() string {
@ -62,8 +90,8 @@ func (f *fileStreamOut) Read(b []byte) (int, error) {
return f.r.Read(b)
}
func (f *fileStreamOut) EnvFile(fs filestore.FileStore) (interface{}, error) {
return f.w, nil
func (f *fileStreamOut) EnvFile(fs filestore.FileStore) (envexec.File, error) {
return envexec.NewFileWriter(f.w, envexec.Size(math.MaxInt32)), nil
}
func (f *fileStreamOut) String() string {

View File

@ -168,7 +168,11 @@ func streamInput(ctx context.Context, es pb.Executor_ExecStreamServer, streamIn
if !ok {
return fmt.Errorf("input %s not exists", i.ExecResize.GetName())
}
if err = setWinsize(f.w, i); err != nil {
tty := f.GetTTY()
if tty == nil {
return fmt.Errorf("input %s does not have TTY", i.ExecResize.GetName())
}
if err = setWinsize(tty, i); err != nil {
return fmt.Errorf("resize to input %s with err %w", i.ExecResize.GetName(), err)
}

View File

@ -154,7 +154,7 @@ func initLogger(conf *config.Config) {
}
}
func prefork(envPool envexec.EnvironmentPool, prefork int) {
func prefork(envPool worker.EnvironmentPool, prefork int) {
if prefork <= 0 {
return
}
@ -306,7 +306,7 @@ func newEnvBuilder(conf *config.Config) pool.EnvBuilder {
return b
}
func newWorker(conf *config.Config, envPool envexec.EnvironmentPool, fs filestore.FileStore) worker.Worker {
func newWorker(conf *config.Config, envPool worker.EnvironmentPool, fs filestore.FileStore) worker.Worker {
return worker.New(worker.Config{
FileStore: fs,
EnvironmentPool: envPool,

View File

@ -219,7 +219,7 @@ func convertCmdFile(f *CmdFile, srcPrefix string) (worker.CmdFile, error) {
case f.FileID != nil:
return &worker.CachedFile{FileID: *f.FileID}, nil
case f.Max != nil && f.Name != nil:
return &worker.PipeCollector{Name: *f.Name, Max: *f.Max}, nil
return &worker.PipeCollector{Name: *f.Name, Max: envexec.Size(*f.Max)}, nil
default:
return nil, fmt.Errorf("file is not valid for cmd")
}

View File

@ -7,6 +7,7 @@ import (
"net/http"
"path"
"github.com/criyle/go-judge/envexec"
"github.com/criyle/go-judge/filestore"
"github.com/gin-gonic/gin"
)
@ -56,19 +57,26 @@ func (f *fileHandle) fileIDGet(c *gin.Context) {
return
}
file := f.fs.Get(uri.FileID)
name, file := f.fs.Get(uri.FileID)
if file == nil {
c.AbortWithStatus(http.StatusNotFound)
return
}
content, err := file.Content()
r, err := envexec.FileToReader(file)
if err != nil {
c.AbortWithError(http.StatusInternalServerError, err)
return
}
defer r.Close()
content, err := io.ReadAll(r)
if err != nil {
c.AbortWithError(http.StatusInternalServerError, err)
return
}
typ := mime.TypeByExtension(path.Ext(file.Name()))
c.Header("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", file.Name()))
typ := mime.TypeByExtension(path.Ext(name))
c.Header("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", name))
c.Data(http.StatusOK, typ, content)
}

View File

@ -6,6 +6,7 @@ import (
"bytes"
"context"
"encoding/json"
"io"
"log"
"os"
"time"
@ -13,6 +14,7 @@ import (
"github.com/criyle/go-judge/cmd/executorserver/model"
"github.com/criyle/go-judge/env"
"github.com/criyle/go-judge/env/pool"
"github.com/criyle/go-judge/envexec"
"github.com/criyle/go-judge/filestore"
"github.com/criyle/go-judge/worker"
)
@ -165,11 +167,17 @@ func FileGet(e *C.char) *C.char {
if err := json.NewDecoder(bytes.NewBufferString(es)).Decode(&f); err != nil {
return nil
}
file := fs.Get(f.ID)
_, file := fs.Get(f.ID)
if file == nil {
return nil
}
c, err := file.Content()
r, err := envexec.FileToReader(file)
if err != nil {
return nil
}
defer r.Close()
c, err := io.ReadAll(r)
if err != nil {
return nil
}

View File

@ -4,6 +4,7 @@ import (
"sync"
"github.com/criyle/go-judge/envexec"
"github.com/criyle/go-judge/worker"
)
// Environment defines envexec.Environment with destroy
@ -26,7 +27,7 @@ type pool struct {
}
// NewPool returns a pool for EnvBuilder
func NewPool(builder EnvBuilder) envexec.EnvironmentPool {
func NewPool(builder EnvBuilder) worker.EnvironmentPool {
return &pool{
builder: builder,
}
@ -45,7 +46,10 @@ func (p *pool) Get() (envexec.Environment, error) {
}
func (p *pool) Put(env envexec.Environment) {
e, _ := env.(Environment)
e, ok := env.(Environment)
if !ok {
panic("invalid environment put")
}
e.Reset()
p.mu.Lock()

View File

@ -4,7 +4,6 @@ import (
"context"
"time"
"github.com/criyle/go-judge/file"
"github.com/criyle/go-sandbox/runner"
)
@ -16,16 +15,17 @@ type RunnerResult = runner.Result
// Cmd defines instruction to run a program in container environment
type Cmd struct {
// argument, environment
Environment Environment
// file contents to copyin before exec
CopyIn map[string]File
// exec argument, environment
Args []string
Env []string
// fds for exec: can be nil, file.Opener, PipeCollector
// nil: undefined, will be closed
// *os.File: will be fd and passed to runner, file will be close after cmd starts
// file.Opener: will be opened and passed to runner
// PipeCollector: a pipe write end will be passed to runner and collected as a copyout file
Files []interface{}
// Files for the executing command
Files []File
TTY bool // use pty as input / output
// resource limits
@ -38,8 +38,10 @@ type Cmd struct {
CPURateLimit float64
StrictMemoryLimit bool
// file contents to copyin before exec
CopyIn map[string]file.File
// Waiter is called after cmd starts and it should return
// once time limit exceeded.
// return true to as TLE and false as normal exits (context finished)
Waiter func(context.Context, Process) bool
// file names to copyout after exec
CopyOut []string
@ -47,17 +49,6 @@ type Cmd struct {
// CopyOutDir specifies a dir to dump all /w contnet
CopyOutDir string
// Waiter is called after cmd starts and it should return
// once time limit exceeded.
// return true to as TLE and false as normal exits (context finished)
Waiter func(context.Context, Process) bool
}
// PipeCollector can be used in Cmd.Files paramenter
type PipeCollector struct {
Name string
SizeLimit int64
}
// Result defines the running result for single Cmd
@ -73,5 +64,5 @@ type Result struct {
Memory Size // byte
// Files stores copy out files
Files map[string]file.File
Files map[string][]byte
}

104
envexec/file.go Normal file
View File

@ -0,0 +1,104 @@
package envexec
import (
"fmt"
"io"
"os"
)
// File defines interface of envexec files
type File interface {
isFile()
}
// FileReader represent file input which can be fully read before exec
// or piped into exec
type FileReader struct {
Reader io.Reader
Stream bool
}
func (*FileReader) isFile() {}
// NewFileReader creates File input which can be fully read before exec
// or piped into exec
func NewFileReader(r io.Reader, s bool) File {
return &FileReader{Reader: r, Stream: s}
}
// ReaderTTY will be asserts when File Reader is provided and TTY is enabled
// and then TTY will be called with pty file
type ReaderTTY interface {
TTY(*os.File)
}
// FileInput represent file input which will be opened in read-only mode
type FileInput struct {
Path string
}
func (*FileInput) isFile() {}
// NewFileInput creates file input which will be opened in read-only mode
func NewFileInput(p string) File {
return &FileInput{Path: p}
}
// FilePipeCollector represent pipe output which will be collected through pipe
type FilePipeCollector struct {
Name string
Limit Size
}
func (*FilePipeCollector) isFile() {}
// NewFilePipeCollector creates file output which will be collected through pipe
func NewFilePipeCollector(name string, limit Size) File {
return &FilePipeCollector{Name: name, Limit: limit}
}
// FileWriter represent pipe output which will be piped out from exec
type FileWriter struct {
Writer io.Writer
Limit Size
}
func (*FileWriter) isFile() {}
// NewFileWriter create File which will be piped out from exec
func NewFileWriter(w io.Writer, limit Size) File {
return &FileWriter{Writer: w, Limit: limit}
}
// FileOpened represent file that is already opened
type FileOpened struct {
File *os.File
}
func (*FileOpened) isFile() {}
// NewFileOpened creates file that contains already opened file and it will be closed
func NewFileOpened(f *os.File) File {
return &FileOpened{File: f}
}
// FileToReader get a ReadCloser from underlying file
func FileToReader(f File) (io.ReadCloser, error) {
switch f := f.(type) {
case *FileOpened:
return f.File, nil
case *FileReader:
return io.NopCloser(f.Reader), nil
case *FileInput:
file, err := os.Open(f.Path)
if err != nil {
return nil, err
}
return file, nil
default:
return nil, fmt.Errorf("file cannot open as reader %v", f)
}
}

View File

@ -7,22 +7,21 @@ import (
"os"
"sync"
"github.com/criyle/go-judge/file"
"github.com/criyle/go-sandbox/runner"
"golang.org/x/sync/errgroup"
)
// copyOutAndCollect reads file and pipes in parallel from container
func copyOutAndCollect(m Environment, c *Cmd, ptc []pipeCollector) (map[string]file.File, error) {
func copyOutAndCollect(m Environment, c *Cmd, ptc []pipeCollector) (map[string][]byte, error) {
var (
g errgroup.Group
l sync.Mutex
)
rt := make(map[string]file.File)
put := func(f file.File) {
rt := make(map[string][]byte)
put := func(f []byte, n string) {
l.Lock()
defer l.Unlock()
rt[f.Name()] = f
rt[n] = f
}
// copy out
@ -56,7 +55,7 @@ func copyOutAndCollect(m Environment, c *Cmd, ptc []pipeCollector) (map[string]f
if err != nil {
return err
}
put(file.NewMemFile(n, buf.Bytes()))
put(buf.Bytes(), n)
return nil
})
}
@ -65,11 +64,11 @@ func copyOutAndCollect(m Environment, c *Cmd, ptc []pipeCollector) (map[string]f
for _, p := range ptc {
p := p
g.Go(func() error {
<-p.buff.Done
if int64(p.buff.Buffer.Len()) > p.buff.Max {
<-p.done
if int64(p.buffer.Len()) > int64(p.limit) {
return runner.StatusOutputLimitExceeded
}
put(file.NewMemFile(p.name, p.buff.Buffer.Bytes()))
put(p.buffer.Bytes(), p.name)
return nil
})
}

View File

@ -1,30 +1,30 @@
package envexec
import (
"fmt"
"os"
"github.com/criyle/go-judge/file"
"golang.org/x/sync/errgroup"
)
// copyIn copied file from host to container in parallel
func copyIn(m Environment, copyIn map[string]file.File) error {
func copyIn(m Environment, copyIn map[string]File) error {
var g errgroup.Group
for n, f := range copyIn {
n, f := n, f
g.Go(func() error {
hf, err := FileToReader(f)
if err != nil {
return fmt.Errorf("failed to copyIn %v", err)
}
defer hf.Close()
cf, err := m.Open(n, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0777)
if err != nil {
return err
}
defer cf.Close()
hf, err := f.Reader()
if err != nil {
return err
}
defer hf.Close()
_, err = cf.ReadFrom(hf)
if err != nil {
return err

51
envexec/file_pipe.go Normal file
View File

@ -0,0 +1,51 @@
package envexec
import (
"bytes"
"io"
"os"
)
type pipeBuffer struct {
W *os.File
Buffer *bytes.Buffer
Done <-chan struct{}
Limit Size
}
type pipeCollector struct {
done <-chan struct{}
buffer *bytes.Buffer
limit Size
name string
}
func newPipe(writer io.Writer, limit Size) (<-chan struct{}, *os.File, error) {
r, w, err := os.Pipe()
if err != nil {
return nil, nil, err
}
done := make(chan struct{})
go func() {
io.CopyN(writer, r, int64(limit))
close(done)
// ensure no blocking / SIGPIPE on the other end
io.Copy(io.Discard, r)
r.Close()
}()
return done, w, nil
}
func newPipeBuffer(limit Size) (*pipeBuffer, error) {
buffer := new(bytes.Buffer)
done, w, err := newPipe(buffer, limit+1)
if err != nil {
return nil, err
}
return &pipeBuffer{
W: w,
Buffer: buffer,
Done: done,
Limit: limit,
}, nil
}

View File

@ -8,85 +8,97 @@ import (
"sync"
"github.com/creack/pty"
"github.com/criyle/go-judge/file"
"github.com/criyle/go-sandbox/pkg/pipe"
)
type pipeCollector struct {
buff *pipe.Buffer
name string
}
// prepare Files for tty input / output
func prepareCmdFdTTY(c *Cmd, count int) (fd, ftc []*os.File, ptc []pipeCollector, err error) {
var fPty, fTty *os.File
if c.TTY {
fPty, fTty, err = pty.Open()
if err != nil {
err = fmt.Errorf("failed to open tty %v", err)
return nil, nil, nil, err
}
}
ftc = append(ftc, fTty)
fd = make([]*os.File, count)
hasInput := false
var output *pipe.Buffer
func prepareCmdFdTTY(c *Cmd, count int) (f []*os.File, p []pipeCollector, err error) {
var wg sync.WaitGroup
var hasInput, hasOutput bool
fPty, fTty, err := pty.Open()
if err != nil {
err = fmt.Errorf("failed to open tty %v", err)
return nil, nil, err
}
files := make([]*os.File, count)
pipeToCollect := make([]pipeCollector, 0)
defer func() {
if err != nil {
closeFiles(files...)
closeFiles(fTty, fPty)
wg.Wait()
}
}()
for j, t := range c.Files {
switch t := t.(type) {
case nil: // ignore
case *os.File:
fd[j] = t
ftc = append(ftc, t)
case *FileOpened:
files[j] = t.File
case file.ReaderOpener:
case *FileReader:
if hasInput {
err = fmt.Errorf("cannot have multiple input when tty enabled")
goto openError
return nil, nil, fmt.Errorf("cannot have multiple input when tty enabled")
}
hasInput = true
r, err := t.Reader()
if err != nil {
err = fmt.Errorf("failed to open file %v", t)
goto openError
}
fd[j] = fTty
files[j] = fTty
// copy input
wg.Add(1)
go func() {
defer wg.Done()
io.Copy(fPty, r)
io.Copy(fPty, t.Reader)
}()
case PipeCollector:
fd[j] = fTty
if output != nil {
break
// provide TTY
if tty, ok := t.Reader.(ReaderTTY); ok {
tty.TTY(fPty)
}
done := make(chan struct{})
output = &pipe.Buffer{
W: fTty,
Max: t.SizeLimit,
Buffer: new(bytes.Buffer),
Done: done,
case *FileInput:
var f *os.File
f, err = os.Open(t.Path)
if err != nil {
return nil, nil, fmt.Errorf("failed to open file %v", t.Path)
}
ptc = append(ptc, pipeCollector{output, t.Name})
files[j] = f
case *FilePipeCollector:
files[j] = fTty
if hasOutput {
break
}
hasOutput = true
done := make(chan struct{})
buf := new(bytes.Buffer)
pipeToCollect = append(pipeToCollect, pipeCollector{done, buf, t.Limit, t.Name})
wg.Add(1)
go func() {
defer close(done)
defer wg.Done()
io.CopyN(output.Buffer, fPty, output.Max+1)
io.CopyN(buf, fPty, int64(t.Limit)+1)
}()
case *FileWriter:
files[j] = fTty
if hasOutput {
break
}
hasOutput = true
wg.Add(1)
go func() {
defer wg.Done()
io.Copy(t.Writer, fPty)
}()
default:
err = fmt.Errorf("unknown file type %v %t", t, t)
goto openError
return nil, nil, fmt.Errorf("unknown file type %v %t", t, t)
}
}
@ -95,101 +107,121 @@ func prepareCmdFdTTY(c *Cmd, count int) (fd, ftc []*os.File, ptc []pipeCollector
wg.Wait()
fPty.Close()
}()
return
openError:
closeFiles(ftc)
return nil, nil, nil, err
return files, pipeToCollect, nil
}
func prepareCmdFd(c *Cmd, count int) (fd, ftc []*os.File, ptc []pipeCollector, err error) {
func prepareCmdFd(c *Cmd, count int) (f []*os.File, p []pipeCollector, err error) {
if c.TTY {
return prepareCmdFdTTY(c, count)
}
fd = make([]*os.File, count)
files := make([]*os.File, count)
pipeToCollect := make([]pipeCollector, 0)
defer func() {
if err != nil {
closeFiles(files...)
}
}()
// record same name buffer for one command to avoid multiple pipe creation
pb := make(map[string]*pipe.Buffer)
pb := make(map[string]*pipeBuffer)
for j, t := range c.Files {
switch t := t.(type) {
case nil: // ignore
case *os.File:
fd[j] = t
ftc = append(ftc, t)
case *FileOpened:
files[j] = t.File
case file.Opener:
f, err := t.Open()
if err != nil {
err = fmt.Errorf("failed to open file %v", t)
goto openError
case *FileReader:
if t.Stream {
r, w, err := os.Pipe()
if err != nil {
return nil, nil, fmt.Errorf("failed to create pipe %v", err)
}
go w.ReadFrom(t.Reader)
files[j] = r
} else {
f, err := readerToFile(t.Reader)
if err != nil {
return nil, nil, fmt.Errorf("failed to open reader %v", err)
}
files[j] = f
}
fd[j] = f
ftc = append(ftc, f)
case PipeCollector:
case *FileInput:
f, err := os.Open(t.Path)
if err != nil {
return nil, nil, fmt.Errorf("failed to open file %v", t.Path)
}
files[j] = f
case *FilePipeCollector:
if b, ok := pb[t.Name]; ok {
fd[j] = b.W
files[j] = b.W
break
}
b, err := pipe.NewBuffer(t.SizeLimit)
b, err := newPipeBuffer(t.Limit)
if err != nil {
err = fmt.Errorf("failed to create pipe %v", err)
goto openError
return nil, nil, fmt.Errorf("failed to create pipe %v", err)
}
fd[j] = b.W
pb[t.Name] = b
ptc = append(ptc, pipeCollector{b, t.Name})
ftc = append(ftc, b.W)
files[j] = b.W
pipeToCollect = append(pipeToCollect, pipeCollector{b.Done, b.Buffer, t.Limit, t.Name})
case *FileWriter:
_, w, err := newPipe(t.Writer, t.Limit)
if err != nil {
return nil, nil, fmt.Errorf("failed to create pipe %v", err)
}
files[j] = w
default:
err = fmt.Errorf("unknown file type %v %t", t, t)
goto openError
return nil, nil, fmt.Errorf("unknown file type %v %t", t, t)
}
}
return
openError:
closeFiles(ftc)
return nil, nil, nil, err
return files, pipeToCollect, nil
}
// prepareFd returns fds, pipeToCollect fileToClose, error
func prepareFds(r *Group) ([][]*os.File, [][]pipeCollector, []*os.File, error) {
func prepareFds(r *Group) (f [][]*os.File, p [][]pipeCollector, err error) {
// prepare fd count
fdCount, err := countFd(r)
if err != nil {
return nil, nil, nil, err
return nil, nil, err
}
// newly opened files need to be closed
var fileToClose []*os.File
// prepare files
fds := make([][]*os.File, len(fdCount))
files := make([][]*os.File, len(fdCount))
pipeToCollect := make([][]pipeCollector, len(fdCount))
// newly opened files need to be closed
defer func() {
if err != nil {
for _, fs := range files {
closeFiles(fs...)
}
}
}()
// prepare cmd fd
for i, c := range r.Cmd {
var ftc []*os.File
fds[i], ftc, pipeToCollect[i], err = prepareCmdFd(c, fdCount[i])
files[i], pipeToCollect[i], err = prepareCmdFd(c, fdCount[i])
if err != nil {
return nil, nil, nil, err
return nil, nil, err
}
fileToClose = append(fileToClose, ftc...)
}
// prepare pipes
for _, p := range r.Pipes {
out, in, err := os.Pipe()
if err != nil {
return nil, nil, nil, err
return nil, nil, err
}
fileToClose = append(fileToClose, out, in)
fds[p.Out.Index][p.Out.Fd] = out
fds[p.In.Index][p.In.Fd] = in
files[p.Out.Index][p.Out.Fd] = out
files[p.In.Index][p.In.Fd] = in
}
return fds, pipeToCollect, fileToClose, nil
return files, pipeToCollect, nil
}
func countFd(r *Group) ([]int, error) {

View File

@ -2,10 +2,37 @@ package envexec
import (
"fmt"
"io"
"os"
"sync/atomic"
"syscall"
"github.com/criyle/go-sandbox/pkg/memfd"
)
const memfdName = "input"
var enableMemFd int32
func readerToFile(reader io.Reader) (*os.File, error) {
if atomic.LoadInt32(&enableMemFd) == 0 {
f, err := memfd.DupToMemfd(memfdName, reader)
if err == nil {
return f, err
}
atomic.StoreInt32(&enableMemFd, 1)
}
r, w, err := os.Pipe()
if err != nil {
return nil, err
}
go func() {
defer w.Close()
w.ReadFrom(reader)
}()
return r, nil
}
func copyDir(src *os.File, dst string) error {
// make sure dir exists
os.MkdirAll(dst, 0777)

View File

@ -8,6 +8,18 @@ import (
"path"
)
func readerToFile(reader io.Reader) (*os.File, error) {
r, w, err := os.Pipe()
if err != nil {
return nil, err
}
go func() {
defer w.Close()
w.ReadFrom(reader)
}()
return r, nil
}
func copyDir(src *os.File, dst string) error {
// make sure dir exists
os.MkdirAll(dst, 0777)

View File

@ -2,7 +2,6 @@ package envexec
import (
"context"
"fmt"
"golang.org/x/sync/errgroup"
)
@ -10,9 +9,6 @@ import (
// Group defines the running instruction to run multiple
// exec in parallel restricted within cgroup
type Group struct {
// EnvironmentPool defines pool used for runner environment
EnvironmentPool EnvironmentPool
// Cmd defines Cmd running in parallel in multiple environments
Cmd []*Cmd
@ -35,31 +31,18 @@ type Pipe struct {
// Run starts the cmd and returns exec results
func (r *Group) Run(ctx context.Context) ([]Result, error) {
// prepare files
fds, pipeToCollect, fileToClose, err := prepareFds(r)
defer func() { closeFiles(fileToClose) }()
fds, pipeToCollect, err := prepareFds(r)
if err != nil {
return nil, err
}
// prepare environments
ms := make([]Environment, 0, len(r.Cmd))
for range r.Cmd {
m, err := r.EnvironmentPool.Get()
if err != nil {
return nil, fmt.Errorf("failed to get environment %v", err)
}
defer r.EnvironmentPool.Put(m)
ms = append(ms, m)
}
// wait all cmd to finish
var g errgroup.Group
result := make([]Result, len(r.Cmd))
for i, c := range r.Cmd {
i, c := i, c
g.Go(func() error {
r, err := runSingle(ctx, ms[i], c, fds[i], pipeToCollect[i])
r, err := runSingle(ctx, c, fds[i], pipeToCollect[i])
result[i] = r
if err != nil {
result[i].Status = StatusInternalError

View File

@ -47,7 +47,7 @@ type Usage struct {
// Process reference to the running process group
type Process interface {
Done() <-chan struct{} // Done returns a channel for wait process to exit
Result() RunnerResult // Result is available after done is closed
Result() RunnerResult // Result wait until done and returns RunnerResult
Usage() Usage // Usage retrieves the process usage during the run time
}
@ -58,9 +58,3 @@ type Environment interface {
// Open open file at work dir with given relative path and flags
Open(path string, flags int, perm os.FileMode) (*os.File, error)
}
// EnvironmentPool implements pool of environments
type EnvironmentPool interface {
Get() (Environment, error)
Put(Environment)
}

View File

@ -8,19 +8,83 @@ import (
)
// runSingle runs Cmd inside the given environment and cgroup
func runSingle(pc context.Context, m Environment, c *Cmd, fds []*os.File, ptc []pipeCollector) (result Result, err error) {
fdToClose := fds
defer func() { closeFiles(fdToClose) }()
func runSingle(pc context.Context, c *Cmd, fds []*os.File, ptc []pipeCollector) (result Result, err error) {
m := c.Environment
// copyin
if len(c.CopyIn) > 0 {
if err := copyIn(m, c.CopyIn); err != nil {
if err := runSingleCopyIn(m, c.CopyIn); err != nil {
result.Status = StatusFileError
result.Error = err.Error()
closeFiles(fds...)
return result, nil
}
// run cmd and wait for result
rt := runSingleWait(pc, m, c, fds)
// collect result
files, err := copyOutAndCollect(m, c, ptc)
result = Result{
Status: convertStatus(rt.Status),
ExitStatus: rt.ExitStatus,
Error: rt.Error,
Time: rt.Time,
RunTime: rt.RunningTime,
Memory: rt.Memory,
Files: files,
}
// collect error (only if the process exits normally)
if rt.Status == runner.StatusNormal && err != nil && result.Error == "" {
switch err := err.(type) {
case runner.Status:
result.Status = convertStatus(err)
default:
result.Status = StatusFileError
result.Error = err.Error()
return result, nil
}
result.Error = err.Error()
}
if result.Time > c.TimeLimit {
result.Status = StatusTimeLimitExceeded
}
if result.Memory > c.MemoryLimit {
result.Status = StatusMemoryLimitExceeded
}
return result, nil
}
func runSingleCopyIn(m Environment, copyInFiles map[string]File) error {
if len(copyInFiles) == 0 {
return nil
}
return copyIn(m, copyInFiles)
}
func runSingleWait(pc context.Context, m Environment, c *Cmd, fds []*os.File) RunnerResult {
// start the cmd (they will be canceled in other goroutines)
ctx, cancel := context.WithCancel(pc)
defer cancel()
process, err := runSingleExecve(ctx, m, c, fds)
if err != nil {
return runner.Result{
Status: runner.StatusRunnerError,
Error: err.Error(),
}
}
// starts waiter to periodically check cpu usage
go func() {
defer cancel()
c.Waiter(ctx, process)
}()
// ensure waiter exit
<-ctx.Done()
return process.Result()
}
func runSingleExecve(ctx context.Context, m Environment, c *Cmd, fds []*os.File) (Process, error) {
defer closeFiles(fds...)
extraMemoryLimit := c.ExtraMemoryLimit
if extraMemoryLimit == 0 {
extraMemoryLimit = defaultExtraMemoryLimit
@ -51,64 +115,5 @@ func runSingle(pc context.Context, m Environment, c *Cmd, fds []*os.File, ptc []
StrictMemory: c.StrictMemoryLimit,
},
}
// start the cmd (they will be canceled in other goroutines)
ctx, cancel := context.WithCancel(pc)
waiterCtx, waiterCancel := context.WithCancel(ctx)
process, err := m.Execve(ctx, execParam)
// close files
closeFiles(fds)
fdToClose = nil
// starts waiter to periodically check cpu usage
go func() {
c.Waiter(waiterCtx, process)
cancel()
}()
var rt runner.Result
if err == nil {
<-process.Done()
rt = process.Result()
} else {
rt = runner.Result{
Status: runner.StatusRunnerError,
Error: err.Error(),
}
}
waiterCancel()
// collect result
files, err := copyOutAndCollect(m, c, ptc)
result = Result{
Status: convertStatus(rt.Status),
ExitStatus: rt.ExitStatus,
Error: rt.Error,
Time: rt.Time,
RunTime: rt.RunningTime,
Memory: rt.Memory,
Files: files,
}
// collect error (only if the process exits normally)
if rt.Status == runner.StatusNormal && err != nil && result.Error == "" {
switch err := err.(type) {
case runner.Status:
result.Status = convertStatus(err)
default:
result.Status = StatusFileError
}
result.Error = err.Error()
}
if result.Time > c.TimeLimit {
result.Status = StatusTimeLimitExceeded
}
if result.Memory > c.MemoryLimit {
result.Status = StatusMemoryLimitExceeded
}
// make sure waiter exit
<-ctx.Done()
return result, nil
return m.Execve(ctx, execParam)
}

View File

@ -2,15 +2,11 @@ package envexec
import (
"context"
"fmt"
)
// Single defines the running instruction to run single
// exec in restricted within cgroup
type Single struct {
// EnvironmentPool defines pool used for runner environment
EnvironmentPool EnvironmentPool
// Cmd defines Cmd running in parallel in multiple environments
Cmd *Cmd
}
@ -18,22 +14,12 @@ type Single struct {
// Run starts the cmd and returns exec results
func (s *Single) Run(ctx context.Context) (result Result, err error) {
// prepare files
fd, fileToClose, pipeToCollect, err := prepareCmdFd(s.Cmd, len(s.Cmd.Files))
defer func() { closeFiles(fileToClose) }()
fd, pipeToCollect, err := prepareCmdFd(s.Cmd, len(s.Cmd.Files))
if err != nil {
return result, err
}
// prepare environment
m, err := s.EnvironmentPool.Get()
if err != nil {
return result, fmt.Errorf("failed to get environment %v", err)
}
defer s.EnvironmentPool.Put(m)
result, err = runSingle(ctx, m, s.Cmd, fd, pipeToCollect)
fileToClose = nil // already closed by runOne
result, err = runSingle(ctx, s.Cmd, fd, pipeToCollect)
if err != nil {
result.Status = StatusInternalError
result.Error = err.Error()

View File

@ -37,8 +37,11 @@ func getFdArray(fd []*os.File) []uintptr {
return r
}
func closeFiles(files []*os.File) {
func closeFiles(files ...*os.File) {
for _, f := range files {
if f == nil {
continue
}
f.Close()
}
}

View File

@ -1,26 +0,0 @@
package file
import (
"io"
"os"
)
// Opener opens the file in readonly mode
// caller should close afterwards
type Opener interface {
Open() (*os.File, error)
}
// ReaderOpener creates readCloser for caller
type ReaderOpener interface {
Reader() (io.ReadCloser, error)
}
// File defines file name with its content
// file could on file system or memory
type File interface {
Opener
ReaderOpener
Content() ([]byte, error)
Name() string
}

View File

@ -1,42 +0,0 @@
package file
import (
"fmt"
"io"
"os"
)
var _ File = &localFile{}
// localFile stores a path to represent a local file
type localFile struct {
name, path string
}
// NewLocalFile creates a wrapper to file system by path
func NewLocalFile(name, path string) File {
return &localFile{
name: name,
path: path,
}
}
func (f *localFile) Name() string {
return f.name
}
func (f *localFile) Content() ([]byte, error) {
return os.ReadFile(f.path)
}
func (f *localFile) Open() (*os.File, error) {
return os.Open(f.path)
}
func (f *localFile) String() string {
return fmt.Sprintf("[localfile:%v(%v)]", f.path, f.name)
}
func (f *localFile) Reader() (io.ReadCloser, error) {
return os.Open(f.path)
}

View File

@ -1,39 +0,0 @@
package file
import (
"bytes"
"fmt"
"io"
)
var _ File = &memFile{}
// memFile represent a file like byte array
type memFile struct {
name string
content []byte
}
// NewMemFile create a file interface from content and content should not be modified afterwards
func NewMemFile(name string, content []byte) File {
return &memFile{
name: name,
content: content,
}
}
func (m *memFile) Name() string {
return m.name
}
func (m *memFile) Content() ([]byte, error) {
return m.content, nil
}
func (m *memFile) String() string {
return fmt.Sprintf("[memfile:%v,%d]", m.name, len(m.content))
}
func (m *memFile) Reader() (io.ReadCloser, error) {
return io.NopCloser(bytes.NewReader(m.content)), nil
}

View File

@ -1,29 +0,0 @@
package file
import (
"bytes"
"os"
"github.com/criyle/go-sandbox/pkg/memfd"
)
var enableMemFd = true
func (m *memFile) Open() (*os.File, error) {
if enableMemFd {
f, err := memfd.DupToMemfd(m.name, bytes.NewReader(m.content))
if err == nil {
return f, err
}
enableMemFd = false
}
r, w, err := os.Pipe()
if err != nil {
return nil, err
}
go func() {
defer w.Close()
w.Write(m.content)
}()
return r, nil
}

View File

@ -1,19 +0,0 @@
// +build !linux
package file
import (
"os"
)
func (m *memFile) Open() (*os.File, error) {
r, w, err := os.Pipe()
if err != nil {
return nil, err
}
go func() {
defer w.Close()
w.Write(m.content)
}()
return r, nil
}

View File

@ -1,14 +0,0 @@
package file
// SourceCode defines source code with its language
type SourceCode struct {
Language string
Code File
ExtraFiles []File
}
// CompiledExec defines compiled executable
type CompiledExec struct {
Language string
Exec []File
}

View File

@ -5,7 +5,7 @@ import (
"path"
"sync"
"github.com/criyle/go-judge/file"
"github.com/criyle/go-judge/envexec"
)
type fileLocalStore struct {
@ -51,19 +51,19 @@ func (s *fileLocalStore) Add(name string, content []byte) (string, error) {
return id, err
}
func (s *fileLocalStore) Get(id string) file.File {
func (s *fileLocalStore) Get(id string) (string, envexec.File) {
s.mu.RLock()
defer s.mu.RUnlock()
p := path.Join(s.dir, id)
if _, err := os.Stat(p); os.IsNotExist(err) {
return nil
return "", nil
}
name, ok := s.name[id]
if !ok {
name = id
}
return file.NewLocalFile(name, p)
return name, envexec.NewFileInput(p)
}
func (s *fileLocalStore) Remove(id string) bool {

View File

@ -1,24 +1,30 @@
package filestore
import (
"bytes"
"sync"
"github.com/criyle/go-judge/file"
"github.com/criyle/go-judge/envexec"
)
type fileMemoryStore struct {
store map[string]file.File
store map[string]fileMemory
mu sync.RWMutex
}
type fileMemory struct {
name string
content []byte
}
// NewFileMemoryStore create new memory file store
func NewFileMemoryStore() FileStore {
return &fileMemoryStore{
store: make(map[string]file.File),
store: make(map[string]fileMemory),
}
}
func (s *fileMemoryStore) Add(fileName string, content []byte) (string, error) {
func (s *fileMemoryStore) Add(name string, content []byte) (string, error) {
s.mu.Lock()
defer s.mu.Unlock()
@ -40,7 +46,7 @@ func (s *fileMemoryStore) Add(fileName string, content []byte) (string, error) {
return "", err
}
s.store[id] = file.NewMemFile(fileName, content)
s.store[id] = fileMemory{name: name, content: content}
return id, err
}
@ -53,19 +59,22 @@ func (s *fileMemoryStore) Remove(fileID string) bool {
return ok
}
func (s *fileMemoryStore) Get(fileID string) file.File {
func (s *fileMemoryStore) Get(fileID string) (string, envexec.File) {
s.mu.RLock()
defer s.mu.RUnlock()
f := s.store[fileID]
return f
f, ok := s.store[fileID]
if !ok {
return "", nil
}
return f.name, envexec.NewFileReader(bytes.NewReader(f.content), false)
}
func (s *fileMemoryStore) List() []string {
s.mu.RLock()
defer s.mu.RUnlock()
var b []string
b := make([]string, 0, len(s.store))
for n := range s.store {
b = append(b, n)
}

View File

@ -5,17 +5,17 @@ import (
"crypto/rand"
"encoding/base32"
"github.com/criyle/go-judge/file"
"github.com/criyle/go-judge/envexec"
)
const randIDLength = 12
// FileStore defines interface to store file
type FileStore interface {
Add(string, []byte) (string, error) // Add creates a file with name & content to the storage, returns id
Remove(string) bool // Remove deletes a file by id
Get(string) file.File // Get file by id, nil if not exists
List() []string // List return all file ids
Add(name string, content []byte) (string, error) // Add creates a file with name & content to the storage, returns id
Remove(string) bool // Remove deletes a file by id
Get(string) (string, envexec.File) // Get file by id, nil if not exists
List() []string // List return all file ids
}
func generateID() (string, error) {

21
go.mod
View File

@ -3,7 +3,7 @@ module github.com/criyle/go-judge
go 1.16
require (
cloud.google.com/go v0.77.0 // indirect
cloud.google.com/go v0.79.0 // indirect
github.com/creack/pty v1.1.11
github.com/criyle/go-sandbox v0.6.7
github.com/elastic/go-seccomp-bpf v1.1.0
@ -22,26 +22,25 @@ require (
github.com/koding/multiconfig v0.0.0-20171124222453-69c27309b2d7
github.com/kr/text v0.2.0 // indirect
github.com/leodido/go-urn v1.2.1 // indirect
github.com/magefile/mage v1.11.0 // indirect
github.com/prometheus/client_golang v1.9.0
github.com/prometheus/common v0.17.0 // indirect
github.com/prometheus/common v0.19.0 // indirect
github.com/prometheus/procfs v0.6.0 // indirect
github.com/sirupsen/logrus v1.8.0 // indirect
github.com/sirupsen/logrus v1.8.1 // indirect
github.com/ugorji/go v1.2.4 // indirect
github.com/zsais/go-gin-prometheus v0.1.0
go.uber.org/multierr v1.6.0 // indirect
go.uber.org/zap v1.16.0
golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83
golang.org/x/net v0.0.0-20210222171744-9060382bd457
golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93 // indirect
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110
golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84 // indirect
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
golang.org/x/sys v0.0.0-20210225014209-683adc9d29d7
golang.org/x/sys v0.0.0-20210313202042-bd2e13477e9c
golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d // indirect
google.golang.org/genproto v0.0.0-20210223151946-22b48be4551b // indirect
google.golang.org/grpc v1.35.0
google.golang.org/grpc/examples v0.0.0-20210218181225-26c143bd5f59 // indirect
google.golang.org/genproto v0.0.0-20210312152112-fc591d9ea70f // indirect
google.golang.org/grpc v1.36.0
google.golang.org/grpc/examples v0.0.0-20210312231957-21976fa3e38a // indirect
google.golang.org/protobuf v1.25.0
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
gopkg.in/yaml.v2 v2.4.0
honnef.co/go/tools v0.1.2 // indirect
honnef.co/go/tools v0.1.3 // indirect
)

58
go.sum
View File

@ -15,8 +15,9 @@ cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOY
cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY=
cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI=
cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk=
cloud.google.com/go v0.77.0 h1:qA5V5+uQf6Mgr+tmFI8UT3D/ELyhIYkPwNGao/3Y+sQ=
cloud.google.com/go v0.77.0/go.mod h1:R8fYSLIilC247Iu8WS2OGHw1E/Ufn7Pd7HiDjTqiURs=
cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg=
cloud.google.com/go v0.79.0 h1:oqqswrt4x6b9OGBnNqdssxBl1xf0rSUNjU2BR4BZar0=
cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8=
cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE=
cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc=
@ -167,6 +168,7 @@ github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt
github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4=
github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8=
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
@ -193,8 +195,10 @@ github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.4 h1:L8R9j+yAqZuZjsqh/z+F1NCffTKKLShY6zXTItVIZ8M=
github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
@ -209,6 +213,7 @@ github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hf
github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
@ -289,9 +294,6 @@ github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ic
github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM=
github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4=
github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ=
github.com/magefile/mage v1.10.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A=
github.com/magefile/mage v1.11.0 h1:C/55Ywp9BpgVVclD3lRnSYCwXTYxmSppIgLeDYlNuls=
github.com/magefile/mage v1.11.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A=
github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
@ -372,8 +374,8 @@ github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y8
github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA=
github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo=
github.com/prometheus/common v0.15.0/go.mod h1:U+gB1OBLb1lF3O42bTCL+FK18tX9Oar16Clt/msog/s=
github.com/prometheus/common v0.17.0 h1:kDIZLI74SS+3tedSvEkykgBkD7txMxaJAPj8DtJUKYA=
github.com/prometheus/common v0.17.0/go.mod h1:U+gB1OBLb1lF3O42bTCL+FK18tX9Oar16Clt/msog/s=
github.com/prometheus/common v0.19.0 h1:Itb4+NjG9wRdkAWgVucbM/adyIXxEhbw0866e0uZE6A=
github.com/prometheus/common v0.19.0/go.mod h1:U+gB1OBLb1lF3O42bTCL+FK18tX9Oar16Clt/msog/s=
github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
@ -393,8 +395,8 @@ github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeV
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
github.com/sirupsen/logrus v1.8.0 h1:nfhvjKcUMhBMVqbKHJlk5RPrrfYr/NMo3692g0dwfWU=
github.com/sirupsen/logrus v1.8.0/go.mod h1:4GuYW9TZmE769R5STWrRakJc4UqQ3+QQ95fyz7ENv1A=
github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE=
github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
@ -439,6 +441,7 @@ go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk=
go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E=
go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
@ -538,10 +541,12 @@ golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81R
golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20210222171744-9060382bd457 h1:hMm9lBjyNLe/c9C6bElQxp4wsrleaJn1vXMZIQkNN44=
golang.org/x/net v0.0.0-20210222171744-9060382bd457/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110 h1:qWPm9rbaAMKs8Bq/9LRpbMqxWRVUAQwMI9fVrssnTfw=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@ -550,9 +555,10 @@ golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4Iltr
golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
golang.org/x/oauth2 v0.0.0-20210113205817-d3ed898aa8a3/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93 h1:alLDrZkL34Y2bnGHfvC1CYBRBXCXgx8AC2vY4MRtYX4=
golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84 h1:duBc5zuJsmJXYOVVE/6PxejI+N3AaCqKjtsoLn1Je5Q=
golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@ -616,9 +622,11 @@ golang.org/x/sys v0.0.0-20201214210602-f9fddec55a1e/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210223212115-eede4237b368/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210225014209-683adc9d29d7 h1:pk3Y+QnSKjMLfO/HIqzn/Zvv3/IHjRPhwblrmUuodzw=
golang.org/x/sys v0.0.0-20210225014209-683adc9d29d7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210313202042-bd2e13477e9c h1:coiPEfMv+ThsjULRDygLrJVlNE1gDdL2g65s0LhV2os=
golang.org/x/sys v0.0.0-20210313202042-bd2e13477e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d h1:SZxvLBoTP5yHO3Frd4z4vrF+DBX9vMVanchswa69toE=
@ -715,6 +723,7 @@ google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz513
google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg=
google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE=
google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8=
google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU=
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
@ -761,9 +770,11 @@ google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6D
google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
google.golang.org/genproto v0.0.0-20210212180131-e7f2df4ecc2d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
google.golang.org/genproto v0.0.0-20210223151946-22b48be4551b h1:GXCSqFSSKq+L4Pi31A2Ba7j8BZCwHN8oJkREab1VokI=
google.golang.org/genproto v0.0.0-20210223151946-22b48be4551b/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
google.golang.org/genproto v0.0.0-20210312152112-fc591d9ea70f h1:YRBxgxUW6GFi+AKsn8WGA9k1SZohK+gGuEqdeT5aoNQ=
google.golang.org/genproto v0.0.0-20210312152112-fc591d9ea70f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs=
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
google.golang.org/grpc v1.20.0/go.mod h1:chYK+tFQF0nDUGJgXMSgLCQk3phJEuONr2DCgLDdAQM=
@ -784,10 +795,11 @@ google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM
google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc=
google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8=
google.golang.org/grpc v1.35.0 h1:TwIQcH3es+MojMVojxxfQ3l3OF2KzlRxML2xZq0kRo8=
google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
google.golang.org/grpc/examples v0.0.0-20210218181225-26c143bd5f59 h1:a1Ho6wK91TIxb1HOnT0ZEh0N6ibCVKMnn//ck3YyQrk=
google.golang.org/grpc/examples v0.0.0-20210218181225-26c143bd5f59/go.mod h1:Ly7ZA/ARzg8fnPU9TyZIxoz33sEUuWX7txiqs8lPTgE=
google.golang.org/grpc v1.36.0 h1:o1bcQ6imQMIOpdrO3SWf2z5RV72WbDwdXuK0MDlc8As=
google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
google.golang.org/grpc/examples v0.0.0-20210312231957-21976fa3e38a h1:vcAu3eKd/uZ5IfeibKEpP5Or6HXorHv5LD0OC6ZFUUo=
google.golang.org/grpc/examples v0.0.0-20210312231957-21976fa3e38a/go.mod h1:Ly7ZA/ARzg8fnPU9TyZIxoz33sEUuWX7txiqs8lPTgE=
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
@ -834,8 +846,8 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh
honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
honnef.co/go/tools v0.1.2 h1:SMdYLJl312RXuxXziCCHhRsp/tvct9cGKey0yv95tZM=
honnef.co/go/tools v0.1.2/go.mod h1:NgwopIslSNH47DimFoV78dnkksY2EFtX0ajyb3K/las=
honnef.co/go/tools v0.1.3 h1:qTakTkI6ni6LFD5sBwwsdSO+AQqbSIxOauHTTQKZ/7o=
honnef.co/go/tools v0.1.3/go.mod h1:NgwopIslSNH47DimFoV78dnkksY2EFtX0ajyb3K/las=
rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=

View File

@ -1,17 +1,17 @@
package worker
import (
"bytes"
"fmt"
"github.com/criyle/go-judge/envexec"
"github.com/criyle/go-judge/file"
"github.com/criyle/go-judge/filestore"
)
// CmdFile defines file used in the cmd
type CmdFile interface {
// EnvFile prepares file for envexec file
EnvFile(fs filestore.FileStore) (interface{}, error)
EnvFile(fs filestore.FileStore) (envexec.File, error)
// Stringer to print debug infomation
String() string
}
@ -29,8 +29,8 @@ type LocalFile struct {
}
// EnvFile prepares file for envexec file
func (f *LocalFile) EnvFile(fs filestore.FileStore) (interface{}, error) {
return file.NewLocalFile(f.Src, f.Src), nil
func (f *LocalFile) EnvFile(fs filestore.FileStore) (envexec.File, error) {
return envexec.NewFileInput(f.Src), nil
}
func (f *LocalFile) String() string {
@ -43,8 +43,8 @@ type MemoryFile struct {
}
// EnvFile prepares file for envexec file
func (f *MemoryFile) EnvFile(fs filestore.FileStore) (interface{}, error) {
return file.NewMemFile("", f.Content), nil
func (f *MemoryFile) EnvFile(fs filestore.FileStore) (envexec.File, error) {
return envexec.NewFileReader(bytes.NewReader(f.Content), false), nil
}
func (f *MemoryFile) String() string {
@ -57,8 +57,8 @@ type CachedFile struct {
}
// EnvFile prepares file for envexec file
func (f *CachedFile) EnvFile(fs filestore.FileStore) (interface{}, error) {
fd := fs.Get(f.FileID)
func (f *CachedFile) EnvFile(fs filestore.FileStore) (envexec.File, error) {
_, fd := fs.Get(f.FileID)
if fd == nil {
return nil, fmt.Errorf("file not exists with id %v", f.FileID)
}
@ -71,13 +71,14 @@ func (f *CachedFile) String() string {
// PipeCollector defines on the output (stdout / stderr) to be collected over pipe
type PipeCollector struct {
Name string // pseudo name generated into copyOut
Max int64 // max size to be collected
Name string // pseudo name generated into copyOut
Max envexec.Size // max size to be collected
}
// EnvFile prepares file for envexec file
func (f *PipeCollector) EnvFile(fs filestore.FileStore) (interface{}, error) {
return envexec.PipeCollector{Name: f.Name, SizeLimit: f.Max}, nil
func (f *PipeCollector) EnvFile(fs filestore.FileStore) (envexec.File, error) {
return envexec.NewFilePipeCollector(f.Name, f.Max), nil
}
func (f *PipeCollector) String() string {

View File

@ -36,6 +36,9 @@ func (w *waiter) Wait(ctx context.Context, u envexec.Process) bool {
case <-ctx.Done():
return false
case <-u.Done():
return false
case <-ticker.C:
if time.Since(start) > w.realTimeLimit {
return true

View File

@ -8,16 +8,20 @@ import (
"time"
"github.com/criyle/go-judge/envexec"
"github.com/criyle/go-judge/file"
"github.com/criyle/go-judge/filestore"
)
const maxWaiting = 512
type EnvironmentPool interface {
Get() (envexec.Environment, error)
Put(envexec.Environment)
}
// Config defines worker configuration
type Config struct {
FileStore filestore.FileStore
EnvironmentPool envexec.EnvironmentPool
EnvironmentPool EnvironmentPool
Parallelism int
WorkDir string
TimeLimitTickInterval time.Duration
@ -38,7 +42,7 @@ type Worker interface {
// worker defines executor worker
type worker struct {
fs filestore.FileStore
envPool envexec.EnvironmentPool
envPool EnvironmentPool
parallelism int
workDir string
@ -159,9 +163,19 @@ func (w *worker) workDoSingle(ctx context.Context, rc Cmd) (rt Response) {
rt.Error = err
return
}
// prepare environment
env, err := w.envPool.Get()
if err != nil {
return Response{Results: []Result{{
Status: envexec.StatusInternalError,
Error: fmt.Sprintf("failed to get environment %v", err),
}}}
}
defer w.envPool.Put(env)
c.Environment = env
s := &envexec.Single{
EnvironmentPool: w.envPool,
Cmd: c,
Cmd: c,
}
result, err := s.Run(ctx)
if err != nil {
@ -187,9 +201,22 @@ func (w *worker) workDoGroup(ctx context.Context, rc []Cmd, pm []PipeMap) (rt Re
cs = append(cs, c)
copyOutSets = append(copyOutSets, os)
}
for i := range cs {
env, err := w.envPool.Get()
if err != nil {
res := make([]Result, 0, len(cs))
for range cs {
res = append(res, Result{
Status: envexec.StatusInternalError,
Error: fmt.Sprintf("failed to get environment %v", err),
})
}
return Response{Results: res}
}
defer w.envPool.Put(env)
cs[i].Environment = env
}
g := envexec.Group{
EnvironmentPool: w.envPool,
Cmd: cs,
Pipes: p,
}
@ -217,13 +244,7 @@ func (w *worker) convertResult(result envexec.Result, copyOutSet map[string]bool
res.Files = make(map[string][]byte)
res.FileIDs = make(map[string]string)
for name, fi := range result.Files {
b, err := fi.Content()
if err != nil {
res.Status = envexec.StatusFileError
res.Error = err.Error()
return
}
for name, b := range result.Files {
if copyOutSet[name] {
res.Files[name] = b
} else {
@ -322,8 +343,8 @@ func preparePipeMapping(pm []PipeMap) []*envexec.Pipe {
return rt
}
func (w *worker) prepareCopyIn(cf map[string]CmdFile) (map[string]file.File, error) {
rt := make(map[string]file.File)
func (w *worker) prepareCopyIn(cf map[string]CmdFile) (map[string]envexec.File, error) {
rt := make(map[string]envexec.File)
for name, f := range cf {
if f == nil {
return nil, fmt.Errorf("nil type cannot be used for copyIn %s", name)
@ -332,17 +353,13 @@ func (w *worker) prepareCopyIn(cf map[string]CmdFile) (map[string]file.File, err
if err != nil {
return nil, err
}
fi, ok := pcf.(file.File)
if !ok {
return nil, fmt.Errorf("pipe type cannot be used for copyIn %s", name)
}
rt[name] = fi
rt[name] = pcf
}
return rt, nil
}
func (w *worker) prepareCmdFiles(files []CmdFile) ([]interface{}, map[string]bool, error) {
rt := make([]interface{}, 0, len(files))
func (w *worker) prepareCmdFiles(files []CmdFile) ([]envexec.File, map[string]bool, error) {
rt := make([]envexec.File, 0, len(files))
pipeFileName := make(map[string]bool)
for _, f := range files {
if f == nil {
@ -354,7 +371,7 @@ func (w *worker) prepareCmdFiles(files []CmdFile) ([]interface{}, map[string]boo
return nil, nil, err
}
rt = append(rt, cf)
if t, ok := cf.(envexec.PipeCollector); ok {
if t, ok := cf.(*envexec.FilePipeCollector); ok {
pipeFileName[t.Name] = true
}
}