stream: rename types to avoid stutters

This commit is contained in:
criyle 2024-02-05 08:57:12 +00:00
parent c29d0adce2
commit 8dd368a655
16 changed files with 79 additions and 54 deletions

View File

@ -11,9 +11,10 @@ import (
"github.com/criyle/go-judge/pb"
"github.com/gin-gonic/gin"
"github.com/golang/protobuf/jsonpb"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials"
"google.golang.org/grpc/credentials/insecure"
"google.golang.org/protobuf/encoding/protojson"
"google.golang.org/protobuf/types/known/emptypb"
)
@ -28,7 +29,12 @@ type execProxy struct {
func (p *execProxy) Exec(c *gin.Context) {
req := new(pb.Request)
if err := jsonpb.Unmarshal(c.Request.Body, req); err != nil {
b, err := io.ReadAll(c.Request.Body)
if err != nil {
c.AbortWithError(http.StatusBadRequest, err)
return
}
if err := protojson.Unmarshal(b, req); err != nil {
c.AbortWithError(http.StatusBadRequest, err)
return
}
@ -125,7 +131,7 @@ func (p *execProxy) FileDelete(c *gin.Context) {
func main() {
flag.Parse()
token := os.Getenv("TOKEN")
opts := []grpc.DialOption{grpc.WithInsecure()}
opts := []grpc.DialOption{grpc.WithTransportCredentials(insecure.NewCredentials())}
if token != "" {
opts = append(opts, grpc.WithPerRPCCredentials(newTokenAuth(token)))
}

View File

@ -15,6 +15,7 @@ import (
"golang.org/x/term"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials"
"google.golang.org/grpc/credentials/insecure"
)
var (
@ -43,7 +44,7 @@ func main() {
}
token := os.Getenv("TOKEN")
opts := []grpc.DialOption{grpc.WithInsecure()}
opts := []grpc.DialOption{grpc.WithTransportCredentials(insecure.NewCredentials())}
if token != "" {
opts = append(opts, grpc.WithPerRPCCredentials(newTokenAuth(token)))
}

View File

@ -16,7 +16,7 @@ type streamWrapper struct {
es pb.Executor_ExecStreamServer
}
func (sw *streamWrapper) Send(r stream.StreamResponse) error {
func (sw *streamWrapper) Send(r stream.Response) error {
res := &pb.StreamResponse{}
switch {
case r.Response != nil:
@ -34,21 +34,21 @@ func (sw *streamWrapper) Send(r stream.StreamResponse) error {
return sw.es.Send(res)
}
func (sw *streamWrapper) Recv() (*stream.StreamRequest, error) {
func (sw *streamWrapper) Recv() (*stream.Request, error) {
req, err := sw.es.Recv()
if err != nil {
return nil, err
}
switch i := req.Request.(type) {
case *pb.StreamRequest_ExecRequest:
return &stream.StreamRequest{Request: convertPBStreamRequest(i.ExecRequest)}, nil
return &stream.Request{Request: convertPBStreamRequest(i.ExecRequest)}, nil
case *pb.StreamRequest_ExecInput:
return &stream.StreamRequest{Input: &stream.InputRequest{
return &stream.Request{Input: &stream.InputRequest{
Name: i.ExecInput.Name,
Content: i.ExecInput.Content,
}}, nil
case *pb.StreamRequest_ExecResize:
return &stream.StreamRequest{Resize: &stream.ResizeRequest{
return &stream.Request{Resize: &stream.ResizeRequest{
Name: i.ExecResize.Name,
Rows: int(i.ExecResize.Rows),
Cols: int(i.ExecResize.Cols),
@ -56,7 +56,7 @@ func (sw *streamWrapper) Recv() (*stream.StreamRequest, error) {
Y: int(i.ExecResize.Y),
}}, nil
case *pb.StreamRequest_ExecCancel:
return &stream.StreamRequest{Cancel: &struct{}{}}, nil
return &stream.Request{Cancel: &struct{}{}}, nil
}
return nil, errors.ErrUnsupported
}

View File

@ -33,7 +33,7 @@ func newListener(addr string) (net.Listener, error) {
if host == "" {
return net.Listen("tcp", addr)
} else if host == "localhost" {
ips, err = getLocalhostIp()
ips, err = getLocalhostIP()
if err != nil {
return nil, err
}
@ -51,7 +51,7 @@ func newListener(addr string) (net.Listener, error) {
return newMultiListener(ips, iPort)
}
func getLocalhostIp() ([]net.IP, error) {
func getLocalhostIP() ([]net.IP, error) {
addrs, err := net.InterfaceAddrs()
if err != nil {
return nil, err

View File

@ -118,6 +118,7 @@ type Response struct {
mmap bool
}
// Close need to be called when mmap specified to be true
func (r *Response) Close() {
if !r.mmap {
return
@ -127,6 +128,7 @@ func (r *Response) Close() {
}
}
// Close need to be called when mmap specified to be true
func (r *Result) Close() {
// remove temporary files
for _, f := range r.files {
@ -318,6 +320,7 @@ func convertCmdFile(f *CmdFile, srcPrefix []string) (worker.CmdFile, error) {
}
}
// CheckPathPrefixes ensure path is allowed by prefixes
func CheckPathPrefixes(path string, prefixes []string) (bool, error) {
for _, p := range prefixes {
ok, err := checkPathPrefix(path, p)

View File

@ -12,12 +12,12 @@ func fileToByteGeneric(f *os.File) ([]byte, error) {
if _, err := f.Seek(0, 0); err != nil {
return nil, err
}
var s int64
if fi, err := f.Stat(); err != nil {
fi, err := f.Stat()
if err != nil {
return nil, err
} else {
s = fi.Size()
}
s := fi.Size()
c := make([]byte, s)
if _, err := io.ReadFull(f, c); err != nil {
return nil, err

View File

@ -15,12 +15,11 @@ func fileToByte(f *os.File, mmap bool) ([]byte, error) {
func fileToByteMmap(f *os.File) ([]byte, error) {
defer f.Close()
var s int64
if fi, err := f.Stat(); err != nil {
fi, err := f.Stat()
if err != nil {
return nil, err
} else {
s = fi.Size()
}
s := fi.Size()
if s == 0 {
return []byte{}, nil
}

View File

@ -17,23 +17,28 @@ const (
minBuffLen = 4 << 10
)
// Stream defines the transport layer for the stream execution that
// stream input and output interactively
type Stream interface {
Send(StreamResponse) error
Recv() (*StreamRequest, error)
Send(Response) error
Recv() (*Request, error)
}
type StreamRequest struct {
// Request defines operations receive from the remote
type Request struct {
Request *model.Request
Resize *ResizeRequest
Input *InputRequest
Cancel *struct{}
}
type StreamResponse struct {
// Response defines response to the remote
type Response struct {
Response *model.Response
Output *OutputResponse
}
// ResizeRequest defines resize operation to the virtual terminal
type ResizeRequest struct {
Name string
Rows int
@ -42,11 +47,13 @@ type ResizeRequest struct {
Y int
}
// InputRequest defines input operation from the remote
type InputRequest struct {
Name string
Content []byte
}
// OutputResponse defines output result to the remote
type OutputResponse struct {
Name string
Content []byte
@ -56,6 +63,7 @@ var (
errFirstMustBeExec = errors.New("the first stream request must be exec request")
)
// Start initiate a interactive execution on the worker and transmit the request and response over Stream transport layer
func Start(baseCtx context.Context, s Stream, w worker.Worker, srcPrefix []string, logger *zap.Logger) error {
req, err := s.Recv()
if err != nil {
@ -122,7 +130,7 @@ func sendLoop(ctx context.Context, s Stream, outCh chan *OutputResponse, rtCh <-
return ctx.Err()
case o := <-outCh:
err := s.Send(StreamResponse{Output: o})
err := s.Send(Response{Output: o})
if err != nil {
return fmt.Errorf("send output: %w", err)
}
@ -133,7 +141,7 @@ func sendLoop(ctx context.Context, s Stream, outCh chan *OutputResponse, rtCh <-
if err != nil {
return fmt.Errorf("convert response: %w", err)
}
return s.Send(StreamResponse{Response: &model.Response{Results: ret.Results}})
return s.Send(Response{Response: &model.Response{Results: ret.Results}})
}
}
}

View File

@ -10,6 +10,7 @@ import (
//go:embed version.*
var versions embed.FS
// Version defines the version of go-judge
var Version string = "unable to get version"
func init() {

View File

@ -48,7 +48,7 @@ type wsHandle struct {
type wsRequest struct {
model.Request
CancelRequestId string `json:"cancelRequestId"`
CancelRequestID string `json:"cancelRequestId"`
}
func (h *wsHandle) Register(r *gin.Engine) {
@ -66,9 +66,9 @@ func (h *wsHandle) handleWS(c *gin.Context) {
cm := newContextMap()
handleRequest := func(baseCtx context.Context, req *wsRequest) error {
if req.CancelRequestId != "" {
h.logger.Sugar().Debugf("ws cancel: %s", req.CancelRequestId)
cm.Remove(req.CancelRequestId)
if req.CancelRequestID != "" {
h.logger.Sugar().Debugf("ws cancel: %s", req.CancelRequestID)
cm.Remove(req.CancelRequestID)
return nil
}
r, err := model.ConvertRequest(&req.Request, h.srcPrefix)
@ -185,26 +185,26 @@ func newContextMap() *contextMap {
return &contextMap{m: make(map[string]context.CancelFunc)}
}
func (c *contextMap) Add(reqId string, cancel context.CancelFunc) error {
if reqId == "" {
func (c *contextMap) Add(reqID string, cancel context.CancelFunc) error {
if reqID == "" {
return fmt.Errorf("empty request id")
}
c.mu.Lock()
defer c.mu.Unlock()
if _, exist := c.m[reqId]; exist {
return fmt.Errorf("duplicated request id: %v", reqId)
if _, exist := c.m[reqID]; exist {
return fmt.Errorf("duplicated request id: %v", reqID)
}
c.m[reqId] = cancel
c.m[reqID] = cancel
return nil
}
func (c *contextMap) Remove(reqId string) {
func (c *contextMap) Remove(reqID string) {
c.mu.Lock()
defer c.mu.Unlock()
if cancel, exist := c.m[reqId]; exist {
delete(c.m, reqId)
if cancel, exist := c.m[reqID]; exist {
delete(c.m, reqID)
cancel()
}
}

2
env/env_linux.go vendored
View File

@ -155,7 +155,7 @@ func newCgroup(c Config) (cgroup.Cgroup, error) {
c.Error("Failed to get available controllers", err)
return nil, err
}
if t == cgroup.CgroupTypeV2 {
if t == cgroup.TypeV2 {
// Check if running on a systemd enabled system
c.Info("Running with cgroup v2, connecting systemd dbus to create cgroup")
var conn *dbus.Conn

View File

@ -86,8 +86,10 @@ type Result struct {
FileError []FileError
}
// FileErrorType defines the location that file operation fails
type FileErrorType int
// FileError enums
const (
ErrCopyInOpenFile FileErrorType = iota
ErrCopyInCreateDir
@ -102,6 +104,7 @@ const (
ErrSymlink
)
// FileError defines the location, file name and the detailed message for a failed file operation
type FileError struct {
Name string `json:"name"`
Type FileErrorType `json:"type"`
@ -131,17 +134,19 @@ func (t FileErrorType) String() string {
return ""
}
// MarshalJSON encodes file error into json string
func (t FileErrorType) MarshalJSON() ([]byte, error) {
return []byte(`"` + t.String() + `"`), nil
}
// UnmarshalJSON decodes file error from json string
func (t *FileErrorType) UnmarshalJSON(b []byte) error {
str := string(b)
if v, ok := fileErrorStringReverse[str]; ok {
v, ok := fileErrorStringReverse[str]
if ok {
return fmt.Errorf("%s is not file error type", str)
} else {
*t = v
}
*t = v
return nil
}

View File

@ -1,16 +1,16 @@
// Package envexec provides utility function to run program in restricted environments
// through container and cgroup.
//
// Cmd
// # Cmd
//
// Cmd defines single program to run, including copyin files before exec, run the program and copy
// out files after exec
//
// Single
// ## Single
//
// Single defines single Cmd with Environment and Cgroup Pool
// Single defines single Cmd with Environment and Cgroup Pool
//
// Group
// ## Group
//
// Group defines multiple Cmd with Environment and Cgroup Pool, together with Pipe mapping between
// different Cmd

View File

@ -35,7 +35,9 @@ func readerToFile(reader io.Reader) (*os.File, error) {
func copyDir(src *os.File, dst string) error {
// make sure dir exists
os.MkdirAll(dst, 0777)
if err := os.MkdirAll(dst, 0777); err != nil {
return err
}
newDir, err := os.Open(dst)
if err != nil {
return err

6
go.mod
View File

@ -5,13 +5,12 @@ go 1.21
require (
github.com/coreos/go-systemd/v22 v22.5.0
github.com/creack/pty v1.1.21
github.com/criyle/go-sandbox v0.10.0
github.com/criyle/go-sandbox v0.10.1
github.com/elastic/go-seccomp-bpf v1.4.0
github.com/elastic/go-ucfg v0.8.6
github.com/gin-contrib/zap v0.2.0
github.com/gin-gonic/gin v1.9.1
github.com/godbus/dbus/v5 v5.1.0
github.com/golang/protobuf v1.5.3
github.com/gorilla/websocket v1.5.1
github.com/grpc-ecosystem/go-grpc-middleware v1.4.0
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0
@ -44,9 +43,10 @@ require (
github.com/go-playground/universal-translator v0.18.1 // indirect
github.com/go-playground/validator/v10 v10.17.0 // indirect
github.com/goccy/go-json v0.10.2 // indirect
github.com/golang/protobuf v1.5.3 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/klauspost/cpuid/v2 v2.2.6 // indirect
github.com/leodido/go-urn v1.3.0 // indirect
github.com/leodido/go-urn v1.4.0 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect

8
go.sum
View File

@ -30,8 +30,8 @@ github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8
github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
github.com/creack/pty v1.1.21 h1:1/QdRyBaHHJP61QkWMXlOIBfsgdDeeKfK8SYVUWJKf0=
github.com/creack/pty v1.1.21/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4=
github.com/criyle/go-sandbox v0.10.0 h1:sOuOOw75GzBVPWj40J1719KMpbZluQOVoHfCuIB130o=
github.com/criyle/go-sandbox v0.10.0/go.mod h1:UYdir/vjn/Q1WqUsdBFz8jzVqWQ4qcngBy+pveq+XDc=
github.com/criyle/go-sandbox v0.10.1 h1:z9Il/UXQwKEvIwdr1wVheWWWAqGWtdTItBmEsWqFqT4=
github.com/criyle/go-sandbox v0.10.1/go.mod h1:ivPw/HEh5unxVRlXJxCgkgTCuy+cxTkQDX7D2XQf/kg=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@ -110,8 +110,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/leodido/go-urn v1.3.0 h1:jX8FDLfW4ThVXctBNZ+3cIWnCSnrACDV73r76dy0aQQ=
github.com/leodido/go-urn v1.3.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI=
github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ=
github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI=
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=