add loacl storeage driver for executor server

This commit is contained in:
criyle 2020-03-04 16:41:52 -05:00
parent 7951f37bd4
commit fa072e373f
7 changed files with 214 additions and 84 deletions

View File

@ -13,6 +13,7 @@ The goal to to reimplement [syzoj/judge-v3](https://github.com/syzoj/judge-v3) i
A rest service to run program in restricted environment and it is basically a wrapper for `pkg/envexec` to run single / multiple programs.
- /run POST execute program in the restricted environment
- /file GET list all cached file
- /file POST prepare a file in the executor service (in memory), returns fileId (can be referenced in /run parameter)
- /file/:fileId GET downloads file from executor service (in memory), returns file content
- /file/:fileId DELETE delete file specified by fileId
@ -32,6 +33,8 @@ The default binding address for the executor server is `:5050`. Can be specified
The default concurrency is `4`, Can be specified with `-parallism` flag.
The default file store is in memory, local cache can be specified wieh `-dir` flag.
#### Planed API interface
```typescript

View File

@ -3,24 +3,18 @@ package main
import (
"bytes"
"crypto/rand"
"encoding/base64"
"sync"
"encoding/base32"
"github.com/criyle/go-judge/file"
)
const randIDLength = 12
type fileData struct {
Content []byte
FileName string
}
var (
fileStore map[string]*fileData
fileStoreLock sync.RWMutex
)
func init() {
fileStore = make(map[string]*fileData)
type fileStore interface {
Add(string, []byte) (string, error) // Add creates a file with name & content to the storage, returns id
Remove(string) bool // Remove deletes a file by id
Get(string) file.File // Get file by id, nil if not exists
List() []string // List return all file ids
}
func generateID() (string, error) {
@ -30,65 +24,8 @@ func generateID() (string, error) {
}
var buf bytes.Buffer
if _, err := base64.NewEncoder(base64.StdEncoding, &buf).Write(b); err != nil {
if _, err := base32.NewEncoder(base32.StdEncoding, &buf).Write(b); err != nil {
return "", err
}
return string(buf.Bytes()), nil
}
func addFile(content []byte, fileName string) (string, error) {
fileStoreLock.Lock()
defer fileStoreLock.Unlock()
var (
id string
err error
)
// generate until unique id (try maximun 50 times)
for i := 0; i < 50; i++ {
id, err = generateID()
if err != nil {
return "", err
}
if _, ok := fileStore[id]; !ok {
break
}
}
if err != nil {
return "", err
}
fileStore[id] = &fileData{
Content: content,
FileName: fileName,
}
return id, err
}
func removeFile(fileID string) bool {
fileStoreLock.Lock()
defer fileStoreLock.Unlock()
_, ok := fileStore[fileID]
delete(fileStore, fileID)
return ok
}
func getFile(fileID string) (*fileData, bool) {
fileStoreLock.RLock()
defer fileStoreLock.RUnlock()
f, ok := fileStore[fileID]
return f, ok
}
func getAllFileID() []string {
fileStoreLock.RLock()
defer fileStoreLock.RUnlock()
var b []string
for n := range fileStore {
b = append(b, n)
}
return b
}

View File

@ -1,6 +1,7 @@
package main
import (
"fmt"
"io/ioutil"
"mime"
"net/http"
@ -10,7 +11,7 @@ import (
)
func fileGet(c *gin.Context) {
ids := getAllFileID()
ids := fs.List()
c.JSON(http.StatusOK, ids)
}
@ -32,8 +33,11 @@ func filePost(c *gin.Context) {
return
}
id, err := addFile(b, fh.Filename)
id, err := fs.Add(fh.Filename, b)
if err != nil {
c.AbortWithError(http.StatusInternalServerError, err)
return
}
c.JSON(http.StatusOK, id)
}
@ -47,13 +51,20 @@ func fileIDGet(c *gin.Context) {
return
}
f, ok := getFile(uri.FileID)
if !ok {
f := fs.Get(uri.FileID)
if f == nil {
c.AbortWithStatus(http.StatusNotFound)
return
}
typ := mime.TypeByExtension(path.Ext(f.FileName))
c.Data(http.StatusOK, typ, f.Content)
content, err := f.Content()
if err != nil {
c.AbortWithError(http.StatusInternalServerError, err)
return
}
typ := mime.TypeByExtension(path.Ext(f.Name()))
c.Header("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", f.Name()))
c.Data(http.StatusOK, typ, content)
}
func fileIDDelete(c *gin.Context) {
@ -66,7 +77,7 @@ func fileIDDelete(c *gin.Context) {
return
}
ok := removeFile(uri.FileID)
ok := fs.Remove(uri.FileID)
if !ok {
c.AbortWithStatus(http.StatusNotFound)
return

View File

@ -0,0 +1,94 @@
package main
import (
"io/ioutil"
"os"
"path"
"sync"
"github.com/criyle/go-judge/file"
)
var _ fileStore = &fileLocalStore{}
type fileLocalStore struct {
dir string // directory to store file
name map[string]string // id to name mapping if exists
mu sync.RWMutex
}
func newFileLocalStore(dir string) *fileLocalStore {
return &fileLocalStore{
dir: dir,
name: make(map[string]string),
}
}
func (s *fileLocalStore) Add(name string, content []byte) (string, error) {
s.mu.Lock()
defer s.mu.Unlock()
var (
id string
err error
)
// generate until unique id (try maximun 50 times)
for i := 0; i < 50; i++ {
id, err = generateID()
if err != nil {
return "", err
}
if _, err := os.Stat(path.Join(s.dir, id)); err == nil {
break
}
}
if err != nil {
return "", err
}
err = ioutil.WriteFile(path.Join(s.dir, id), content, 0644)
if err != nil {
return "", err
}
s.name[id] = name
return id, err
}
func (s *fileLocalStore) Get(id string) file.File {
s.mu.RLock()
defer s.mu.RUnlock()
p := path.Join(s.dir, id)
if _, err := os.Stat(p); os.IsNotExist(err) {
return nil
}
name, ok := s.name[id]
if !ok {
name = id
}
return file.NewLocalFile(name, p)
}
func (s *fileLocalStore) Remove(id string) bool {
s.mu.Lock()
defer s.mu.Unlock()
delete(s.name, id)
p := path.Join(s.dir, id)
if _, err := os.Stat(p); os.IsNotExist(err) {
return false
}
os.Remove(p)
return true
}
func (s *fileLocalStore) List() []string {
var names []string
fi, err := ioutil.ReadDir(s.dir)
if err != nil {
return nil
}
for _, f := range fi {
names = append(names, f.Name())
}
return names
}

View File

@ -0,0 +1,74 @@
package main
import (
"sync"
"github.com/criyle/go-judge/file"
)
var _ fileStore = &fileMemoryStore{}
type fileMemoryStore struct {
store map[string]file.File
mu sync.RWMutex
}
func newFileMemoryStore() *fileMemoryStore {
return &fileMemoryStore{
store: make(map[string]file.File),
}
}
func (s *fileMemoryStore) Add(fileName string, content []byte) (string, error) {
s.mu.Lock()
defer s.mu.Unlock()
var (
id string
err error
)
// generate until unique id (try maximun 50 times)
for i := 0; i < 50; i++ {
id, err = generateID()
if err != nil {
return "", err
}
if _, ok := s.store[id]; !ok {
break
}
}
if err != nil {
return "", err
}
s.store[id] = file.NewMemFile(fileName, content)
return id, err
}
func (s *fileMemoryStore) Remove(fileID string) bool {
s.mu.Lock()
defer s.mu.Unlock()
_, ok := s.store[fileID]
delete(s.store, fileID)
return ok
}
func (s *fileMemoryStore) Get(fileID string) file.File {
s.mu.RLock()
defer s.mu.RUnlock()
f := s.store[fileID]
return f
}
func (s *fileMemoryStore) List() []string {
s.mu.RLock()
defer s.mu.RUnlock()
var b []string
for n := range s.store {
b = append(b, n)
}
return b
}

View File

@ -6,6 +6,7 @@ import (
"flag"
"io/ioutil"
"log"
"os"
"sync/atomic"
"syscall"
@ -21,9 +22,12 @@ var (
addr = flag.String("http", ":5050", "specifies the http binding address")
parallism = flag.Int("parallism", 4, "control the # of concurrency execution")
tmpFsParam = flag.String("tmpfs", "size=8m,nr_inodes=4k", "tmpfs mount data")
dir = flag.String("dir", "", "specifies direcotry to store file upload / download (in memory by default)")
envPool envexec.EnvironmentPool
cgroupPool envexec.CgroupPool
fs fileStore
)
func init() {
@ -33,6 +37,13 @@ func init() {
func main() {
flag.Parse()
if *dir == "" {
fs = newFileMemoryStore()
} else {
os.MkdirAll(*dir, 0755)
fs = newFileLocalStore(*dir)
}
root, err := ioutil.TempDir("", "dm")
if err != nil {
panic(err)

View File

@ -129,7 +129,7 @@ func workDoSingle(rc cmd) (res response) {
if copyOutSet[name] {
res.Files[name] = string(b)
} else {
id, err := addFile(b, name)
id, err := fs.Add(name, b)
if err != nil {
res.Status = status(envexec.StatusFileError)
res.Error = err.Error()
@ -182,11 +182,11 @@ func prepareCmdFile(f *cmdFile) (interface{}, error) {
case f.Content != nil:
return file.NewMemFile("file", []byte(*f.Content)), nil
case f.FileID != nil:
fd, ok := getFile(*f.FileID)
if !ok {
fd := fs.Get(*f.FileID)
if fd == nil {
return nil, fmt.Errorf("file not exists for %v", *f.FileID)
}
return file.NewMemFile(fd.FileName, fd.Content), nil
return fd, nil
case f.Max != nil && f.Name != nil:
return envexec.PipeCollector{Name: *f.Name, SizeLimit: *f.Max}, nil
default: