From 8dd368a65576affce0c65b58177bd68f81eeee93 Mon Sep 17 00:00:00 2001 From: criyle Date: Mon, 5 Feb 2024 08:57:12 +0000 Subject: [PATCH] stream: rename types to avoid stutters --- cmd/go-judge-grpc-proxy/proxy.go | 12 +++++++++--- cmd/go-judge-shell/shell.go | 3 ++- cmd/go-judge/grpc_executor/grpc_stream.go | 12 ++++++------ cmd/go-judge/listener.go | 4 ++-- cmd/go-judge/model/model.go | 3 +++ cmd/go-judge/model/util.go | 8 ++++---- cmd/go-judge/model/util_linux.go | 7 +++---- cmd/go-judge/stream/stream.go | 20 +++++++++++++------ cmd/go-judge/version/version.go | 1 + cmd/go-judge/ws_executor/websocket.go | 24 +++++++++++------------ env/env_linux.go | 2 +- envexec/cmd.go | 11 ++++++++--- envexec/doc.go | 8 ++++---- envexec/file_util_linux.go | 4 +++- go.mod | 6 +++--- go.sum | 8 ++++---- 16 files changed, 79 insertions(+), 54 deletions(-) diff --git a/cmd/go-judge-grpc-proxy/proxy.go b/cmd/go-judge-grpc-proxy/proxy.go index 942ded9..f196e25 100644 --- a/cmd/go-judge-grpc-proxy/proxy.go +++ b/cmd/go-judge-grpc-proxy/proxy.go @@ -11,9 +11,10 @@ import ( "github.com/criyle/go-judge/pb" "github.com/gin-gonic/gin" - "github.com/golang/protobuf/jsonpb" "google.golang.org/grpc" "google.golang.org/grpc/credentials" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/protobuf/encoding/protojson" "google.golang.org/protobuf/types/known/emptypb" ) @@ -28,7 +29,12 @@ type execProxy struct { func (p *execProxy) Exec(c *gin.Context) { req := new(pb.Request) - if err := jsonpb.Unmarshal(c.Request.Body, req); err != nil { + b, err := io.ReadAll(c.Request.Body) + if err != nil { + c.AbortWithError(http.StatusBadRequest, err) + return + } + if err := protojson.Unmarshal(b, req); err != nil { c.AbortWithError(http.StatusBadRequest, err) return } @@ -125,7 +131,7 @@ func (p *execProxy) FileDelete(c *gin.Context) { func main() { flag.Parse() token := os.Getenv("TOKEN") - opts := []grpc.DialOption{grpc.WithInsecure()} + opts := []grpc.DialOption{grpc.WithTransportCredentials(insecure.NewCredentials())} if token != "" { opts = append(opts, grpc.WithPerRPCCredentials(newTokenAuth(token))) } diff --git a/cmd/go-judge-shell/shell.go b/cmd/go-judge-shell/shell.go index d599d6f..568e566 100644 --- a/cmd/go-judge-shell/shell.go +++ b/cmd/go-judge-shell/shell.go @@ -15,6 +15,7 @@ import ( "golang.org/x/term" "google.golang.org/grpc" "google.golang.org/grpc/credentials" + "google.golang.org/grpc/credentials/insecure" ) var ( @@ -43,7 +44,7 @@ func main() { } token := os.Getenv("TOKEN") - opts := []grpc.DialOption{grpc.WithInsecure()} + opts := []grpc.DialOption{grpc.WithTransportCredentials(insecure.NewCredentials())} if token != "" { opts = append(opts, grpc.WithPerRPCCredentials(newTokenAuth(token))) } diff --git a/cmd/go-judge/grpc_executor/grpc_stream.go b/cmd/go-judge/grpc_executor/grpc_stream.go index aa87784..0034db3 100644 --- a/cmd/go-judge/grpc_executor/grpc_stream.go +++ b/cmd/go-judge/grpc_executor/grpc_stream.go @@ -16,7 +16,7 @@ type streamWrapper struct { es pb.Executor_ExecStreamServer } -func (sw *streamWrapper) Send(r stream.StreamResponse) error { +func (sw *streamWrapper) Send(r stream.Response) error { res := &pb.StreamResponse{} switch { case r.Response != nil: @@ -34,21 +34,21 @@ func (sw *streamWrapper) Send(r stream.StreamResponse) error { return sw.es.Send(res) } -func (sw *streamWrapper) Recv() (*stream.StreamRequest, error) { +func (sw *streamWrapper) Recv() (*stream.Request, error) { req, err := sw.es.Recv() if err != nil { return nil, err } switch i := req.Request.(type) { case *pb.StreamRequest_ExecRequest: - return &stream.StreamRequest{Request: convertPBStreamRequest(i.ExecRequest)}, nil + return &stream.Request{Request: convertPBStreamRequest(i.ExecRequest)}, nil case *pb.StreamRequest_ExecInput: - return &stream.StreamRequest{Input: &stream.InputRequest{ + return &stream.Request{Input: &stream.InputRequest{ Name: i.ExecInput.Name, Content: i.ExecInput.Content, }}, nil case *pb.StreamRequest_ExecResize: - return &stream.StreamRequest{Resize: &stream.ResizeRequest{ + return &stream.Request{Resize: &stream.ResizeRequest{ Name: i.ExecResize.Name, Rows: int(i.ExecResize.Rows), Cols: int(i.ExecResize.Cols), @@ -56,7 +56,7 @@ func (sw *streamWrapper) Recv() (*stream.StreamRequest, error) { Y: int(i.ExecResize.Y), }}, nil case *pb.StreamRequest_ExecCancel: - return &stream.StreamRequest{Cancel: &struct{}{}}, nil + return &stream.Request{Cancel: &struct{}{}}, nil } return nil, errors.ErrUnsupported } diff --git a/cmd/go-judge/listener.go b/cmd/go-judge/listener.go index bb5ef52..fdcac1f 100644 --- a/cmd/go-judge/listener.go +++ b/cmd/go-judge/listener.go @@ -33,7 +33,7 @@ func newListener(addr string) (net.Listener, error) { if host == "" { return net.Listen("tcp", addr) } else if host == "localhost" { - ips, err = getLocalhostIp() + ips, err = getLocalhostIP() if err != nil { return nil, err } @@ -51,7 +51,7 @@ func newListener(addr string) (net.Listener, error) { return newMultiListener(ips, iPort) } -func getLocalhostIp() ([]net.IP, error) { +func getLocalhostIP() ([]net.IP, error) { addrs, err := net.InterfaceAddrs() if err != nil { return nil, err diff --git a/cmd/go-judge/model/model.go b/cmd/go-judge/model/model.go index 1fef4d8..06beb30 100644 --- a/cmd/go-judge/model/model.go +++ b/cmd/go-judge/model/model.go @@ -118,6 +118,7 @@ type Response struct { mmap bool } +// Close need to be called when mmap specified to be true func (r *Response) Close() { if !r.mmap { return @@ -127,6 +128,7 @@ func (r *Response) Close() { } } +// Close need to be called when mmap specified to be true func (r *Result) Close() { // remove temporary files for _, f := range r.files { @@ -318,6 +320,7 @@ func convertCmdFile(f *CmdFile, srcPrefix []string) (worker.CmdFile, error) { } } +// CheckPathPrefixes ensure path is allowed by prefixes func CheckPathPrefixes(path string, prefixes []string) (bool, error) { for _, p := range prefixes { ok, err := checkPathPrefix(path, p) diff --git a/cmd/go-judge/model/util.go b/cmd/go-judge/model/util.go index 2cd0899..3d308b5 100644 --- a/cmd/go-judge/model/util.go +++ b/cmd/go-judge/model/util.go @@ -12,12 +12,12 @@ func fileToByteGeneric(f *os.File) ([]byte, error) { if _, err := f.Seek(0, 0); err != nil { return nil, err } - var s int64 - if fi, err := f.Stat(); err != nil { + fi, err := f.Stat() + if err != nil { return nil, err - } else { - s = fi.Size() } + s := fi.Size() + c := make([]byte, s) if _, err := io.ReadFull(f, c); err != nil { return nil, err diff --git a/cmd/go-judge/model/util_linux.go b/cmd/go-judge/model/util_linux.go index c9bbb23..6b3c272 100644 --- a/cmd/go-judge/model/util_linux.go +++ b/cmd/go-judge/model/util_linux.go @@ -15,12 +15,11 @@ func fileToByte(f *os.File, mmap bool) ([]byte, error) { func fileToByteMmap(f *os.File) ([]byte, error) { defer f.Close() - var s int64 - if fi, err := f.Stat(); err != nil { + fi, err := f.Stat() + if err != nil { return nil, err - } else { - s = fi.Size() } + s := fi.Size() if s == 0 { return []byte{}, nil } diff --git a/cmd/go-judge/stream/stream.go b/cmd/go-judge/stream/stream.go index ca65202..bc6a061 100644 --- a/cmd/go-judge/stream/stream.go +++ b/cmd/go-judge/stream/stream.go @@ -17,23 +17,28 @@ const ( minBuffLen = 4 << 10 ) +// Stream defines the transport layer for the stream execution that +// stream input and output interactively type Stream interface { - Send(StreamResponse) error - Recv() (*StreamRequest, error) + Send(Response) error + Recv() (*Request, error) } -type StreamRequest struct { +// Request defines operations receive from the remote +type Request struct { Request *model.Request Resize *ResizeRequest Input *InputRequest Cancel *struct{} } -type StreamResponse struct { +// Response defines response to the remote +type Response struct { Response *model.Response Output *OutputResponse } +// ResizeRequest defines resize operation to the virtual terminal type ResizeRequest struct { Name string Rows int @@ -42,11 +47,13 @@ type ResizeRequest struct { Y int } +// InputRequest defines input operation from the remote type InputRequest struct { Name string Content []byte } +// OutputResponse defines output result to the remote type OutputResponse struct { Name string Content []byte @@ -56,6 +63,7 @@ var ( errFirstMustBeExec = errors.New("the first stream request must be exec request") ) +// Start initiate a interactive execution on the worker and transmit the request and response over Stream transport layer func Start(baseCtx context.Context, s Stream, w worker.Worker, srcPrefix []string, logger *zap.Logger) error { req, err := s.Recv() if err != nil { @@ -122,7 +130,7 @@ func sendLoop(ctx context.Context, s Stream, outCh chan *OutputResponse, rtCh <- return ctx.Err() case o := <-outCh: - err := s.Send(StreamResponse{Output: o}) + err := s.Send(Response{Output: o}) if err != nil { return fmt.Errorf("send output: %w", err) } @@ -133,7 +141,7 @@ func sendLoop(ctx context.Context, s Stream, outCh chan *OutputResponse, rtCh <- if err != nil { return fmt.Errorf("convert response: %w", err) } - return s.Send(StreamResponse{Response: &model.Response{Results: ret.Results}}) + return s.Send(Response{Response: &model.Response{Results: ret.Results}}) } } } diff --git a/cmd/go-judge/version/version.go b/cmd/go-judge/version/version.go index a831878..1309a41 100644 --- a/cmd/go-judge/version/version.go +++ b/cmd/go-judge/version/version.go @@ -10,6 +10,7 @@ import ( //go:embed version.* var versions embed.FS +// Version defines the version of go-judge var Version string = "unable to get version" func init() { diff --git a/cmd/go-judge/ws_executor/websocket.go b/cmd/go-judge/ws_executor/websocket.go index 5e33b84..3f5d538 100644 --- a/cmd/go-judge/ws_executor/websocket.go +++ b/cmd/go-judge/ws_executor/websocket.go @@ -48,7 +48,7 @@ type wsHandle struct { type wsRequest struct { model.Request - CancelRequestId string `json:"cancelRequestId"` + CancelRequestID string `json:"cancelRequestId"` } func (h *wsHandle) Register(r *gin.Engine) { @@ -66,9 +66,9 @@ func (h *wsHandle) handleWS(c *gin.Context) { cm := newContextMap() handleRequest := func(baseCtx context.Context, req *wsRequest) error { - if req.CancelRequestId != "" { - h.logger.Sugar().Debugf("ws cancel: %s", req.CancelRequestId) - cm.Remove(req.CancelRequestId) + if req.CancelRequestID != "" { + h.logger.Sugar().Debugf("ws cancel: %s", req.CancelRequestID) + cm.Remove(req.CancelRequestID) return nil } r, err := model.ConvertRequest(&req.Request, h.srcPrefix) @@ -185,26 +185,26 @@ func newContextMap() *contextMap { return &contextMap{m: make(map[string]context.CancelFunc)} } -func (c *contextMap) Add(reqId string, cancel context.CancelFunc) error { - if reqId == "" { +func (c *contextMap) Add(reqID string, cancel context.CancelFunc) error { + if reqID == "" { return fmt.Errorf("empty request id") } c.mu.Lock() defer c.mu.Unlock() - if _, exist := c.m[reqId]; exist { - return fmt.Errorf("duplicated request id: %v", reqId) + if _, exist := c.m[reqID]; exist { + return fmt.Errorf("duplicated request id: %v", reqID) } - c.m[reqId] = cancel + c.m[reqID] = cancel return nil } -func (c *contextMap) Remove(reqId string) { +func (c *contextMap) Remove(reqID string) { c.mu.Lock() defer c.mu.Unlock() - if cancel, exist := c.m[reqId]; exist { - delete(c.m, reqId) + if cancel, exist := c.m[reqID]; exist { + delete(c.m, reqID) cancel() } } diff --git a/env/env_linux.go b/env/env_linux.go index 6ad1cf2..7d9545e 100644 --- a/env/env_linux.go +++ b/env/env_linux.go @@ -155,7 +155,7 @@ func newCgroup(c Config) (cgroup.Cgroup, error) { c.Error("Failed to get available controllers", err) return nil, err } - if t == cgroup.CgroupTypeV2 { + if t == cgroup.TypeV2 { // Check if running on a systemd enabled system c.Info("Running with cgroup v2, connecting systemd dbus to create cgroup") var conn *dbus.Conn diff --git a/envexec/cmd.go b/envexec/cmd.go index 35a0f28..1d0e164 100644 --- a/envexec/cmd.go +++ b/envexec/cmd.go @@ -86,8 +86,10 @@ type Result struct { FileError []FileError } +// FileErrorType defines the location that file operation fails type FileErrorType int +// FileError enums const ( ErrCopyInOpenFile FileErrorType = iota ErrCopyInCreateDir @@ -102,6 +104,7 @@ const ( ErrSymlink ) +// FileError defines the location, file name and the detailed message for a failed file operation type FileError struct { Name string `json:"name"` Type FileErrorType `json:"type"` @@ -131,17 +134,19 @@ func (t FileErrorType) String() string { return "" } +// MarshalJSON encodes file error into json string func (t FileErrorType) MarshalJSON() ([]byte, error) { return []byte(`"` + t.String() + `"`), nil } +// UnmarshalJSON decodes file error from json string func (t *FileErrorType) UnmarshalJSON(b []byte) error { str := string(b) - if v, ok := fileErrorStringReverse[str]; ok { + v, ok := fileErrorStringReverse[str] + if ok { return fmt.Errorf("%s is not file error type", str) - } else { - *t = v } + *t = v return nil } diff --git a/envexec/doc.go b/envexec/doc.go index 5cf492d..7e896d3 100644 --- a/envexec/doc.go +++ b/envexec/doc.go @@ -1,16 +1,16 @@ // Package envexec provides utility function to run program in restricted environments // through container and cgroup. // -// Cmd +// # Cmd // // Cmd defines single program to run, including copyin files before exec, run the program and copy // out files after exec // -// Single +// ## Single // -// Single defines single Cmd with Environment and Cgroup Pool +// Single defines single Cmd with Environment and Cgroup Pool // -// Group +// ## Group // // Group defines multiple Cmd with Environment and Cgroup Pool, together with Pipe mapping between // different Cmd diff --git a/envexec/file_util_linux.go b/envexec/file_util_linux.go index 46060ca..8264575 100644 --- a/envexec/file_util_linux.go +++ b/envexec/file_util_linux.go @@ -35,7 +35,9 @@ func readerToFile(reader io.Reader) (*os.File, error) { func copyDir(src *os.File, dst string) error { // make sure dir exists - os.MkdirAll(dst, 0777) + if err := os.MkdirAll(dst, 0777); err != nil { + return err + } newDir, err := os.Open(dst) if err != nil { return err diff --git a/go.mod b/go.mod index 1c8cf4f..965fb5a 100644 --- a/go.mod +++ b/go.mod @@ -5,13 +5,12 @@ go 1.21 require ( github.com/coreos/go-systemd/v22 v22.5.0 github.com/creack/pty v1.1.21 - github.com/criyle/go-sandbox v0.10.0 + github.com/criyle/go-sandbox v0.10.1 github.com/elastic/go-seccomp-bpf v1.4.0 github.com/elastic/go-ucfg v0.8.6 github.com/gin-contrib/zap v0.2.0 github.com/gin-gonic/gin v1.9.1 github.com/godbus/dbus/v5 v5.1.0 - github.com/golang/protobuf v1.5.3 github.com/gorilla/websocket v1.5.1 github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 @@ -44,9 +43,10 @@ require ( github.com/go-playground/universal-translator v0.18.1 // indirect github.com/go-playground/validator/v10 v10.17.0 // indirect github.com/goccy/go-json v0.10.2 // indirect + github.com/golang/protobuf v1.5.3 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/cpuid/v2 v2.2.6 // indirect - github.com/leodido/go-urn v1.3.0 // indirect + github.com/leodido/go-urn v1.4.0 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect diff --git a/go.sum b/go.sum index 0c29734..c15a7f1 100644 --- a/go.sum +++ b/go.sum @@ -30,8 +30,8 @@ github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8 github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/creack/pty v1.1.21 h1:1/QdRyBaHHJP61QkWMXlOIBfsgdDeeKfK8SYVUWJKf0= github.com/creack/pty v1.1.21/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= -github.com/criyle/go-sandbox v0.10.0 h1:sOuOOw75GzBVPWj40J1719KMpbZluQOVoHfCuIB130o= -github.com/criyle/go-sandbox v0.10.0/go.mod h1:UYdir/vjn/Q1WqUsdBFz8jzVqWQ4qcngBy+pveq+XDc= +github.com/criyle/go-sandbox v0.10.1 h1:z9Il/UXQwKEvIwdr1wVheWWWAqGWtdTItBmEsWqFqT4= +github.com/criyle/go-sandbox v0.10.1/go.mod h1:ivPw/HEh5unxVRlXJxCgkgTCuy+cxTkQDX7D2XQf/kg= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -110,8 +110,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/leodido/go-urn v1.3.0 h1:jX8FDLfW4ThVXctBNZ+3cIWnCSnrACDV73r76dy0aQQ= -github.com/leodido/go-urn v1.3.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= +github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= +github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=