filestore: use math/rand & reduce fileId to 40bit

This commit is contained in:
criyle 2022-01-08 19:25:23 -08:00
parent dd4c6ee994
commit 5fd53e07ca
7 changed files with 106 additions and 84 deletions

View File

@ -51,7 +51,7 @@
- 默认 gRPC 监听地址是 `:5051` ,使用 `-grpc-addr` 指定
- 默认日志等级是 debug ,使用 `-silent` 关闭 或 使用 `-release` 开启 release 级别日志
- 默认没有开启鉴权,使用 `-auth-token` 指定令牌鉴权
- 默认没有开启 go 语音调试接口,使用 `-enable-debug` 开启
- 默认没有开启 go 语音调试接口,使用 `-enable-debug` 开启,同时将日志层级设为 Debug
- 默认没有开启监控接口,使用 `-enable-metrics` 开启
沙箱相关:
@ -219,7 +219,7 @@ interface PreparedFile {
interface Collector {
name: string; // copyOut 文件名
max: number; // 最大大小限制
pipe: boolean; // 通过管道收集默认值为false文件收集
pipe?: boolean; // 通过管道收集默认值为false文件收集
}
interface Cmd {
@ -249,18 +249,18 @@ interface Cmd {
// 和 copyOut 相同,不过文件不返回内容,而是返回一个对应文件 ID ,内容可以通过 /file/:fileId 接口下载
copyOutCached?: string[];
// 指定 copyOut 复制文件大小限制,单位 byte
copyOutMax: number;
copyOutMax?: number;
}
enum Status {
Accepted, // normal
MemoryLimitExceeded, // mle
TimeLimitExceeded, // tle
OutputLimitExceeded, // ole
FileError, // fe
RuntimeError, // re
DangerousSyscall, // dgs
InternalError, // system error
Accepted = 'Accepted', // normal
MemoryLimitExceeded = 'Memory Limit Exceeded', // mle
TimeLimitExceeded = 'Time Limit Exceeded', // tle
OutputLimitExceeded = 'Output Limit Exceeded', // ole
FileError = 'File Error', // fe
NonzeroExitStatus = 'Nonzero Exit Status',
Signalled = 'Signalled',
InternalError = 'Internal Error', // system error
}
interface PipeIndex {
@ -273,22 +273,22 @@ interface PipeMap {
out: PipeIndex; // 管道的输出端
// 开启管道代理,传输内容会从输出端复制到输入端
// 输入端内容在输出端关闭以后会丢弃 (防止 SIGPIPE
proxy: boolean;
name: string; // 如果代理开启,内容会作为 copyOut 放在输入端 (用来 debug
proxy?: boolean;
name?: string; // 如果代理开启,内容会作为 copyOut 放在输入端 (用来 debug
// 限制 copyOut 的最大大小,代理会在超出大小之后正常复制
max: number;
max?: number;
}
enum FileErrorType {
CopyInOpenFile,
CopyInCreateFile,
CopyInCopyContent,
CopyOutOpen,
CopyOutNotRegularFile,
CopyOutSizeExceeded,
CopyOutCreateFile,
CopyOutCopyContent,
CollectSizeExceeded,
CopyInOpenFile = 'CopyInOpenFile',
CopyInCreateFile = 'CopyInCreateFile',
CopyInCopyContent = 'CopyInCopyContent',
CopyOutOpen = 'CopyOutOpen',
CopyOutNotRegularFile = 'CopyOutNotRegularFile',
CopyOutSizeExceeded = 'CopyOutSizeExceeded',
CopyOutCreateFile = 'CopyOutCreateFile',
CopyOutCopyContent = 'CopyOutCopyContent',
CollectSizeExceeded = 'CollectSizeExceeded',
}
interface FileError {
@ -322,13 +322,13 @@ interface Result {
// copyFileCached 指定的文件 id
fileIds?: {[name:string]:string};
// 文件错误详细信息
fileError?: []FileError;
fileError?: FileError[];
}
// WebSocket 结果
interface WSResult {
requestId: string;
results: []Result;
results: Result[];
error?: string;
}
```

View File

@ -51,7 +51,7 @@ Server:
- The default binding address for the gRPC executor server is `:5051`. Can be specified with `-grpc-addr` flag.
- The default log level is debug, use `-silent` to disable logs or use `-release` to enable release logger (auto turn on if in docker).
- `-auth-token` to add token-based authentication to REST / gRPC
- By default, the GO debug endpoints are disabled, to enable, specifies `-enable-debug`
- By default, the GO debug endpoints are disabled, to enable, specifies `-enable-debug`, and it also enables debug log
- By default, the prometheus metrics endpoints are disabled, to enable, specifies `-enable-metrics`
Sandbox:
@ -260,7 +260,7 @@ interface PreparedFile {
interface Collector {
name: string; // file name in copyOut
max: number; // maximum bytes to collect from pipe
pipe: boolean; // collect over pipe or not (default false)
pipe?: boolean; // collect over pipe or not (default false)
}
interface Cmd {
@ -294,18 +294,18 @@ interface Cmd {
// specifies the directory to dump container /w content
copyOutDir: string
// specifies the max file size to copy out
copyOutMax: number; // byte
copyOutMax?: number; // byte
}
enum Status {
Accepted, // normal
MemoryLimitExceeded, // mle
TimeLimitExceeded, // tle
OutputLimitExceeded, // ole
FileError, // fe
RuntimeError, // re
DangerousSyscall, // dgs
InternalError, // system error
Accepted = 'Accepted', // normal
MemoryLimitExceeded = 'Memory Limit Exceeded', // mle
TimeLimitExceeded = 'Time Limit Exceeded', // tle
OutputLimitExceeded = 'Output Limit Exceeded', // ole
FileError = 'File Error', // fe
NonzeroExitStatus = 'Nonzero Exit Status',
Signalled = 'Signalled',
InternalError = 'Internal Error', // system error
}
interface PipeIndex {
@ -318,23 +318,23 @@ interface PipeMap {
out: PipeIndex; // output end of the pipe
// enable pipe proxy from in to out,
// content from in will be discarded if out closes
proxy: boolean;
name: string; // copy out proxy content if proxy enabled
proxy?: boolean;
name?: string; // copy out proxy content if proxy enabled
// limit the copy out content size,
// proxy will still functioning after max
max: number;
max?: number;
}
enum FileErrorType {
CopyInOpenFile,
CopyInCreateFile,
CopyInCopyContent,
CopyOutOpen,
CopyOutNotRegularFile,
CopyOutSizeExceeded,
CopyOutCreateFile,
CopyOutCopyContent,
CollectSizeExceeded,
CopyInOpenFile = 'CopyInOpenFile',
CopyInCreateFile = 'CopyInCreateFile',
CopyInCopyContent = 'CopyInCopyContent',
CopyOutOpen = 'CopyOutOpen',
CopyOutNotRegularFile = 'CopyOutNotRegularFile',
CopyOutSizeExceeded = 'CopyOutSizeExceeded',
CopyOutCreateFile = 'CopyOutCreateFile',
CopyOutCopyContent = 'CopyOutCopyContent',
CollectSizeExceeded = 'CollectSizeExceeded',
}
interface FileError {
@ -346,7 +346,7 @@ interface FileError {
interface Request {
requestId?: string; // for WebSocket requests
cmd: Cmd[];
pipeMapping: PipeMap[];
pipeMapping?: PipeMap[];
}
interface CancelRequest {
@ -368,13 +368,13 @@ interface Result {
// copyFileCached name -> fileId
fileIds?: {[name:string]:string};
// fileError contains detailed file errors
fileError?: []FileError;
fileError?: FileError[];
}
// WebSocket results
interface WSResult {
requestId: string;
results: []Result;
results: Result[];
error?: string;
}
```

View File

@ -4,8 +4,11 @@ package main
import (
"context"
crypto_rand "crypto/rand"
"encoding/binary"
"flag"
"log"
math_rand "math/rand"
"net"
"net/http"
"os"
@ -48,7 +51,9 @@ var logger *zap.Logger
func main() {
conf := loadConf()
initLogger(conf)
defer logger.Sync()
logger.Sugar().Infof("config loaded: %+v", conf)
initRand()
// Init environment pool
fs, fsDir, fsCleanUp, err := newFilsStore(conf.Dir, conf.FileTimeout, conf.EnableMetrics)
@ -157,22 +162,36 @@ func loadConf() *config.Config {
}
func initLogger(conf *config.Config) {
if !conf.Silent {
var err error
if conf.Release {
logger, err = zap.NewProduction()
} else {
config := zap.NewDevelopmentConfig()
config.EncoderConfig.EncodeLevel = zapcore.CapitalColorLevelEncoder
logger, err = config.Build()
}
if err != nil {
log.Fatalln("init logger failed", err)
}
defer logger.Sync()
} else {
if conf.Silent {
logger = zap.NewNop()
return
}
var err error
if conf.Release {
logger, err = zap.NewProduction()
} else {
config := zap.NewDevelopmentConfig()
config.EncoderConfig.EncodeLevel = zapcore.CapitalColorLevelEncoder
if !conf.EnableDebug {
config.Level.SetLevel(zap.InfoLevel)
}
logger, err = config.Build()
}
if err != nil {
log.Fatalln("init logger failed", err)
}
}
func initRand() {
var b [8]byte
_, err := crypto_rand.Read(b[:])
if err != nil {
logger.Fatal("random generator init failed", zap.Error(err))
}
sd := int64(binary.LittleEndian.Uint64(b[:]))
logger.Sugar().Infof("random seed: %d", sd)
math_rand.Seed(sd)
}
func prefork(envPool worker.EnvironmentPool, prefork int) {
@ -202,7 +221,7 @@ func initHTTPMux(conf *config.Config, work worker.Worker, fs filestore.FileStore
if conf.Silent {
r.Use(gin.Recovery())
} else {
r.Use(ginzap.Ginzap(logger, time.RFC3339, true))
r.Use(ginzap.Ginzap(logger, "", false))
r.Use(ginzap.RecoveryWithZap(logger, true))
}

View File

@ -2,15 +2,15 @@ package filestore
import (
"bytes"
"crypto/rand"
"encoding/base32"
"errors"
"math/rand"
"os"
"github.com/criyle/go-judge/envexec"
)
const randIDLength = 12
const randIDLength = 5
var errUniqueIDNotGenerated = errors.New("unique id does not exists after tried 50 times")

10
go.mod
View File

@ -4,7 +4,7 @@ go 1.17
require (
github.com/creack/pty v1.1.17
github.com/criyle/go-sandbox v0.9.1
github.com/criyle/go-sandbox v0.9.2
github.com/elastic/go-seccomp-bpf v1.2.0
github.com/elastic/go-ucfg v0.8.4
github.com/gin-contrib/pprof v1.3.0
@ -17,9 +17,9 @@ require (
github.com/koding/multiconfig v0.0.0-20171124222453-69c27309b2d7
github.com/prometheus/client_golang v1.11.0
github.com/zsais/go-gin-prometheus v0.1.0
go.uber.org/zap v1.19.1
go.uber.org/zap v1.20.0
golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3
golang.org/x/net v0.0.0-20211216030914-fe4d6282115f
golang.org/x/net v0.0.0-20220107192237-5cfca573fb4d
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e
google.golang.org/grpc v1.43.0
@ -36,7 +36,7 @@ require (
github.com/gin-contrib/sse v0.1.0 // indirect
github.com/go-playground/locales v0.14.0 // indirect
github.com/go-playground/universal-translator v0.18.0 // indirect
github.com/go-playground/validator/v10 v10.9.0 // indirect
github.com/go-playground/validator/v10 v10.10.0 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/leodido/go-urn v1.2.1 // indirect
github.com/mattn/go-isatty v0.0.14 // indirect
@ -53,7 +53,7 @@ require (
go.uber.org/multierr v1.7.0 // indirect
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect
golang.org/x/text v0.3.7 // indirect
google.golang.org/genproto v0.0.0-20211223182754-3ac035c7e7cb // indirect
google.golang.org/genproto v0.0.0-20220107163113-42d7afdf6368 // indirect
)
retract (

22
go.sum
View File

@ -69,8 +69,8 @@ github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWH
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
github.com/creack/pty v1.1.17 h1:QeVUsEDNrLBW4tMgZHvxy18sKtr6VI492kBhUfhDJNI=
github.com/creack/pty v1.1.17/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4=
github.com/criyle/go-sandbox v0.9.1 h1:T+82p3cROHC4kU6kOMmOZvf/871Eky7chHb41DeUk1w=
github.com/criyle/go-sandbox v0.9.1/go.mod h1:+w66tiv66TUTslkseyfUHUdYZaawGbC6tSeRSGKR9uI=
github.com/criyle/go-sandbox v0.9.2 h1:9kS4g5rLm+/1ddTc13S0knH4efBtiwqJHFGbRqq8AVk=
github.com/criyle/go-sandbox v0.9.2/go.mod h1:kRjZfsqwuFap7vl7DI3WogothEg2xSsmpGejdOglkBA=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@ -119,8 +119,8 @@ github.com/go-playground/universal-translator v0.18.0 h1:82dyy6p4OuJq4/CByFNOn/j
github.com/go-playground/universal-translator v0.18.0/go.mod h1:UvRDBj+xPUEGrFYl+lu/H90nyDXpg0fqeB/AQUGNTVA=
github.com/go-playground/validator/v10 v10.2.0/go.mod h1:uOYAAleCW8F/7oMFd6aG0GOhaH6EGOAJShg8Id5JGkI=
github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4=
github.com/go-playground/validator/v10 v10.9.0 h1:NgTtmN58D0m8+UuxtYmGztBJB7VnPgjj221I1QHci2A=
github.com/go-playground/validator/v10 v10.9.0/go.mod h1:74x4gJWsvQexRdW8Pn3dXSGrTK4nAUsbPlLADvpJkos=
github.com/go-playground/validator/v10 v10.10.0 h1:I7mrTYv78z8k8VXa/qJlOlEXn/nBh+BF8dHX5nt/dr0=
github.com/go-playground/validator/v10 v10.10.0/go.mod h1:74x4gJWsvQexRdW8Pn3dXSGrTK4nAUsbPlLADvpJkos=
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
@ -305,15 +305,17 @@ go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE=
go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
go.uber.org/goleak v1.1.11-0.20210813005559-691160354723 h1:sHOAIxRGBp443oHZIPB+HsUGaksVCXVQENPxwTfQdH4=
go.uber.org/goleak v1.1.11-0.20210813005559-691160354723/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ=
go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI=
go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ=
go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
go.uber.org/multierr v1.7.0 h1:zaiO/rmgFjbmCXdSYJWQcdvOCsthmdaHfr3Gm2Kx4Ec=
go.uber.org/multierr v1.7.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak=
go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
go.uber.org/zap v1.19.1 h1:ue41HOKd1vGURxrmeKIgELGb3jPW9DMUDGtsinblHwI=
go.uber.org/zap v1.19.1/go.mod h1:j3DNczoxDZroyBnOT1L/Q79cfUMGZxlv/9dzN7SM1rI=
go.uber.org/zap v1.20.0 h1:N4oPlghZwYG55MlU6LXk/Zp00FVNE9X9wrYO8CEs4lc=
go.uber.org/zap v1.20.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw=
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
@ -387,8 +389,8 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v
golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20211216030914-fe4d6282115f h1:hEYJvxw1lSnWIl8X9ofsYMklzaDs90JI2az5YMd4fPM=
golang.org/x/net v0.0.0-20211216030914-fe4d6282115f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20220107192237-5cfca573fb4d h1:62NvYBuaanGXR2ZOfwDFkhhl6X1DUgf8qg3GuQvxZsE=
golang.org/x/net v0.0.0-20220107192237-5cfca573fb4d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@ -571,8 +573,8 @@ google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7Fc
google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
google.golang.org/genproto v0.0.0-20211223182754-3ac035c7e7cb h1:ZrsicilzPCS/Xr8qtBZZLpy4P9TYXAfl49ctG1/5tgw=
google.golang.org/genproto v0.0.0-20211223182754-3ac035c7e7cb/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
google.golang.org/genproto v0.0.0-20220107163113-42d7afdf6368 h1:Et6SkiuvnBn+SgrSYXs/BrUpGB4mbdwt4R3vaPIlicA=
google.golang.org/genproto v0.0.0-20220107163113-42d7afdf6368/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=

View File

@ -3,6 +3,7 @@ package worker
import (
"fmt"
"os"
"path/filepath"
"time"
"github.com/criyle/go-judge/envexec"
@ -90,7 +91,7 @@ func (r Result) String() string {
FileError: r.FileError,
}
for k, v := range r.Files {
d.Files[k] = fmt.Sprintf("(path:%s)", v.Name())
d.Files[k] = filepath.Base(v.Name())
}
return fmt.Sprintf("%+v", d)
}