From 4646ce7fdc9c2b436ae9d1a3e409c385bda582f3 Mon Sep 17 00:00:00 2001 From: criyle <6821729+criyle@users.noreply.github.com> Date: Sun, 26 Apr 2020 03:31:54 -0400 Subject: [PATCH] Add windows support --- README.md | 67 +++++ cmd/executorserver/main_linux.go | 3 +- cmd/executorserver/main_windows.go | 13 + go.mod | 2 +- go.sum | 4 +- pkg/envexec/file_util_others.go | 15 +- pkg/pool/envbuilder_linux.go | 42 +++ pkg/pool/environment_linux.go | 18 +- pkg/pool/envpool_linux.go | 93 ------- pkg/pool/interface.go | 55 ++++ pkg/winc/builder_windows.go | 74 ++++++ pkg/winc/environment_windows.go | 409 +++++++++++++++++++++++++++++ pkg/winc/job_object_windows.go | 64 +++++ pkg/winc/mksyscall.go | 3 + pkg/winc/process_windows.go | 53 ++++ pkg/winc/syscall_windows.go | 121 +++++++++ pkg/winc/token_windows.go | 43 +++ pkg/winc/zsyscall_windows.go | 112 ++++++++ 18 files changed, 1084 insertions(+), 107 deletions(-) create mode 100644 pkg/pool/envbuilder_linux.go delete mode 100644 pkg/pool/envpool_linux.go create mode 100644 pkg/pool/interface.go create mode 100644 pkg/winc/builder_windows.go create mode 100644 pkg/winc/environment_windows.go create mode 100644 pkg/winc/job_object_windows.go create mode 100644 pkg/winc/mksyscall.go create mode 100644 pkg/winc/process_windows.go create mode 100644 pkg/winc/syscall_windows.go create mode 100644 pkg/winc/token_windows.go create mode 100644 pkg/winc/zsyscall_windows.go diff --git a/README.md b/README.md index 2709b0b..8ca1a34 100644 --- a/README.md +++ b/README.md @@ -125,6 +125,24 @@ The following mounts point are examples that can be configured through config fi - pkg/envexec: run single / group of programs in parallel within restricted environment and resource constraints - pkg/pool: reference implementation for Cgroup & Environment Pool +### Windows Support + +Build `executorserver` by: + +`go build ./cmd/executorserver/` + +Build `executor_server.dll`: (need to install `gcc` as well) + +`go build -buildmode=c-shared -o executor_server.so ./cmd/executorserver/` + +Run: `./executorserver` + +#### Security + +- Resources are limited by [JobObject](https://docs.microsoft.com/en-us/windows/win32/procthread/job-objects) +- Privillege are limited by [Restricted Low Mandatory Level Token](https://docs.microsoft.com/en-us/windows/win32/secauthz/access-tokens) +- Low Mandatory Level directory is created for read / write + ### API interface ```typescript @@ -336,6 +354,53 @@ Multiple (interaction problem): ] ``` +Compile On Windows (cygwin): + +```json +{ + "cmd": [{ + "args": ["C:\\Cygwin\\bin\\g++", "a.cc", "-o", "a"], + "env": ["PATH=C:\\Cygwin\\bin;"], + "files": [{ + "content": "" + }, { + "name": "stdout", + "max": 10240 + }, { + "name": "stderr", + "max": 10240 + }], + "cpuLimit": 10000000000, + "memoryLimit": 104857600, + "procLimit": 50, + "copyIn": { + "a.cc": { + "content": "#include \n#include \n#include \nusing namespace std;\nint main() {\nint a, b;\ncin >> a >> b;\ncout << a + b << endl;\n}" + } + }, + "copyOutCached": ["a.exe"] + }] +} +``` + +```json +[ + { + "status": "Accepted", + "exitStatus": 0, + "time": 140625000, + "memory": 36286464, + "files": { + "stderr": "", + "stdout": "" + }, + "fileIds": { + "a.exe": "HLQH2OF4MXUUJBCB" + } + } +] +``` + ## TODO - [x] Github actions to auto build @@ -343,4 +408,6 @@ Multiple (interaction problem): - [ ] Investigate root-free running mechanism (no cgroup && not set uid / gid) - [ ] Investigate RLimit settings (cpu, data, fsize, stack, noFile) - [x] Add WebSocket for job submission +- [x] Windows support +- [ ] MacOS support \ No newline at end of file diff --git a/cmd/executorserver/main_linux.go b/cmd/executorserver/main_linux.go index 29f0010..bd4b4be 100644 --- a/cmd/executorserver/main_linux.go +++ b/cmd/executorserver/main_linux.go @@ -57,7 +57,8 @@ func initEnvPool() { printLog("Created cgroup builder with:", cgb) cgroupPool := pool.NewFakeCgroupPool(cgb) - envPool = pool.NewEnvPool(b, cgroupPool) + builder := pool.NewEnvBuilder(b, cgroupPool) + envPool = pool.NewPool(builder) } type credGen struct { diff --git a/cmd/executorserver/main_windows.go b/cmd/executorserver/main_windows.go index d39a228..886d558 100644 --- a/cmd/executorserver/main_windows.go +++ b/cmd/executorserver/main_windows.go @@ -1,4 +1,17 @@ package main +import ( + "log" + + "github.com/criyle/go-judge/pkg/pool" + "github.com/criyle/go-judge/pkg/winc" +) + func initEnvPool() { + b, err := winc.NewBuilder("") + if err != nil { + log.Fatalln("init container", err) + } + printLog("created winc builder") + envPool = pool.NewPool(b) } diff --git a/go.mod b/go.mod index 766cdbd..eeab5c6 100644 --- a/go.mod +++ b/go.mod @@ -10,6 +10,6 @@ require ( github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.1 // indirect golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a - golang.org/x/sys v0.0.0-20200327173247-9dae0f8f5775 // indirect + golang.org/x/sys v0.0.0-20200420163511-1957bb5e6d1f gopkg.in/yaml.v2 v2.2.8 ) diff --git a/go.sum b/go.sum index 39bc06a..ef42050 100644 --- a/go.sum +++ b/go.sum @@ -53,8 +53,8 @@ golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200316214734-08c61614bede h1:+jwX0O5IlL7Cj6iam25CdV2pKzCV737/4ewDRGFvwFQ= golang.org/x/sys v0.0.0-20200316214734-08c61614bede/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200327173247-9dae0f8f5775 h1:TC0v2RSO1u2kn1ZugjrFXkRZAEaqMN/RW+OTZkBzmLE= -golang.org/x/sys v0.0.0-20200327173247-9dae0f8f5775/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200420163511-1957bb5e6d1f h1:gWF768j/LaZugp8dyS4UwsslYCYz9XgFxvlgsn0n9H8= +golang.org/x/sys v0.0.0-20200420163511-1957bb5e6d1f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= diff --git a/pkg/envexec/file_util_others.go b/pkg/envexec/file_util_others.go index fd283a2..950eb70 100644 --- a/pkg/envexec/file_util_others.go +++ b/pkg/envexec/file_util_others.go @@ -17,7 +17,12 @@ func copyDir(src *os.File, dst string) error { } defer newDir.Close() - dir := src + dir, err := os.Open(src.Name()) + if err != nil { + return err + } + defer dir.Close() + names, err := dir.Readdirnames(-1) if err != nil { return err @@ -35,13 +40,17 @@ func copyDirFile(src, dst, name string) error { if err != nil { return err } - t, err := os.OpenFile(path.Join(dst, name), os.O_WRONLY, 0777) + defer s.Close() + + t, err := os.OpenFile(path.Join(dst, name), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0777) if err != nil { return err } + defer t.Close() + _, err = io.Copy(t, s) if err != nil { return err } return nil -} \ No newline at end of file +} diff --git a/pkg/pool/envbuilder_linux.go b/pkg/pool/envbuilder_linux.go new file mode 100644 index 0000000..da97e11 --- /dev/null +++ b/pkg/pool/envbuilder_linux.go @@ -0,0 +1,42 @@ +package pool + +import ( + "fmt" + "syscall" + + "github.com/criyle/go-sandbox/container" +) + +type environmentBuilder struct { + builder EnvironmentBuilder + cgPool CgroupPool +} + +// NewEnvBuilder creates builder for linux container pools +func NewEnvBuilder(builder EnvironmentBuilder, cgPool CgroupPool) EnvBuilder { + return &environmentBuilder{ + builder: builder, + cgPool: cgPool, + } +} + +// Build creates linux container +func (b *environmentBuilder) Build() (Environment, error) { + m, err := b.builder.Build() + if err != nil { + return nil, err + } + wd, err := m.Open([]container.OpenCmd{{ + Path: "/w", + Flag: syscall.O_CLOEXEC | syscall.O_DIRECTORY, + Perm: 0777, + }}) + if err != nil { + return nil, fmt.Errorf("container: failed to prepare work directory") + } + return &environ{ + Environment: m, + cgPool: b.cgPool, + wd: wd[0], + }, nil +} diff --git a/pkg/pool/environment_linux.go b/pkg/pool/environment_linux.go index a767714..edc43e0 100644 --- a/pkg/pool/environment_linux.go +++ b/pkg/pool/environment_linux.go @@ -10,22 +10,26 @@ import ( "github.com/criyle/go-sandbox/container" ) -var _ envexec.Environment = &Environment{} +var _ envexec.Environment = &environ{} -// Environment defines interface to access container resources -type Environment struct { +// environ defines interface to access container resources +type environ struct { container.Environment cgPool CgroupPool wd *os.File // container work dir } // Destory destories the environment -func (c *Environment) Destory() error { +func (c *environ) Destory() error { return c.Environment.Destroy() } +func (c *environ) Reset() error { + return c.Environment.Reset() +} + // Execve execute process inside the environment -func (c *Environment) Execve(ctx context.Context, param envexec.ExecveParam) (envexec.Process, error) { +func (c *environ) Execve(ctx context.Context, param envexec.ExecveParam) (envexec.Process, error) { cg, err := c.cgPool.Get() if err != nil { return nil, fmt.Errorf("execve: failed to get cgroup %v", err) @@ -45,13 +49,13 @@ func (c *Environment) Execve(ctx context.Context, param envexec.ExecveParam) (en } // WorkDir returns opened work directory, should not close after -func (c *Environment) WorkDir() *os.File { +func (c *environ) WorkDir() *os.File { c.wd.Seek(0, 0) return c.wd } // Open opens file relative to work directory -func (c *Environment) Open(path string, flags int, perm os.FileMode) (*os.File, error) { +func (c *environ) Open(path string, flags int, perm os.FileMode) (*os.File, error) { fd, err := syscall.Openat(int(c.wd.Fd()), path, flags|syscall.O_CLOEXEC, uint32(perm)) if err != nil { return nil, fmt.Errorf("openAtWorkDir: %v", err) diff --git a/pkg/pool/envpool_linux.go b/pkg/pool/envpool_linux.go deleted file mode 100644 index b984104..0000000 --- a/pkg/pool/envpool_linux.go +++ /dev/null @@ -1,93 +0,0 @@ -package pool - -import ( - "fmt" - "sync" - "syscall" - - "github.com/criyle/go-judge/pkg/envexec" - "github.com/criyle/go-sandbox/container" -) - -// EnvPool implements container environment pool -type EnvPool struct { - builder EnvironmentBuilder - cgPool CgroupPool - - env []*Environment - mu sync.Mutex -} - -// NewEnvPool creats new EnvPool with a builder -func NewEnvPool(builder EnvironmentBuilder, cgPool CgroupPool) *EnvPool { - return &EnvPool{ - builder: builder, - cgPool: cgPool, - } -} - -// Get gets environment from pool, if pool is empty, creates new environment -func (p *EnvPool) Get() (envexec.Environment, error) { - p.mu.Lock() - defer p.mu.Unlock() - - if len(p.env) > 0 { - rt := p.env[len(p.env)-1] - p.env = p.env[:len(p.env)-1] - return rt, nil - } - m, err := p.builder.Build() - if err != nil { - return nil, err - } - wd, err := m.Open([]container.OpenCmd{{ - Path: "/w", - Flag: syscall.O_CLOEXEC | syscall.O_DIRECTORY, - Perm: 0777, - }}) - if err != nil { - return nil, fmt.Errorf("container: failed to prepare work directory") - } - return &Environment{ - Environment: m, - cgPool: p.cgPool, - wd: wd[0], - }, nil -} - -// Put puts environment to the pool with reset the environment -func (p *EnvPool) Put(e envexec.Environment) { - env, _ := e.(*Environment) - env.Reset() - - p.mu.Lock() - defer p.mu.Unlock() - - p.env = append(p.env, env) -} - -// Destroy destory an environment -func (p *EnvPool) Destroy(e envexec.Environment) { - env, _ := e.(*Environment) - env.Destroy() -} - -// Release clears the pool -func (p *EnvPool) Release() { - p.mu.Lock() - defer p.mu.Unlock() - - for _, e := range p.env { - p.Destroy(e) - } -} - -// Shutdown indicates a shutdown event, not implemented yes -func (p *EnvPool) Shutdown() { - p.mu.Lock() - defer p.mu.Unlock() - - for _, e := range p.env { - p.Destroy(e) - } -} diff --git a/pkg/pool/interface.go b/pkg/pool/interface.go new file mode 100644 index 0000000..d05f314 --- /dev/null +++ b/pkg/pool/interface.go @@ -0,0 +1,55 @@ +package pool + +import ( + "sync" + + "github.com/criyle/go-judge/pkg/envexec" +) + +// Environment defines envexec.Environment with destroy +type Environment interface { + envexec.Environment + Reset() error + Destroy() error +} + +// EnvBuilder defines the abstract builder for container environment +type EnvBuilder interface { + Build() (Environment, error) +} + +type pool struct { + builder EnvBuilder + + env []Environment + mu sync.Mutex +} + +// NewPool returns a pool for EnvBuilder +func NewPool(builder EnvBuilder) envexec.EnvironmentPool { + return &pool{ + builder: builder, + } +} + +func (p *pool) Get() (envexec.Environment, error) { + p.mu.Lock() + defer p.mu.Unlock() + + if len(p.env) > 0 { + rt := p.env[len(p.env)-1] + p.env = p.env[:len(p.env)-1] + return rt, nil + } + return p.builder.Build() +} + +func (p *pool) Put(env envexec.Environment) { + e, _ := env.(Environment) + e.Reset() + + p.mu.Lock() + defer p.mu.Unlock() + + p.env = append(p.env, e) +} diff --git a/pkg/winc/builder_windows.go b/pkg/winc/builder_windows.go new file mode 100644 index 0000000..60e5535 --- /dev/null +++ b/pkg/winc/builder_windows.go @@ -0,0 +1,74 @@ +package winc + +import ( + "io/ioutil" + "os" + + "github.com/criyle/go-judge/pkg/pool" + "golang.org/x/sys/windows" +) + +const sdString = "S:(ML;;NW;;;LW)D:(A;;0x12019f;;;WD)" +const wdPrefix = "es" +const tmpPrefix = "es_tmp" + +var _ pool.EnvBuilder = &builder{} + +type builder struct { + root string + token windows.Token + sd *windows.SECURITY_DESCRIPTOR +} + +// NewBuilder returns a builder for windows container environment +func NewBuilder(root string) (pool.EnvBuilder, error) { + sd, err := windows.SecurityDescriptorFromString(sdString) + if err != nil { + return nil, err + } + + token, err := createLowMandatoryLevelToken() + if err != nil { + return nil, err + } + return &builder{ + root: root, + token: token, + sd: sd, + }, nil +} + +func (b *builder) Build() (pool.Environment, error) { + sacl, _, err := b.sd.SACL() + if err != nil { + return nil, err + } + workDir, err := ioutil.TempDir(b.root, wdPrefix) + if err != nil { + return nil, err + } + if err := windows.SetNamedSecurityInfo(workDir, windows.SE_FILE_OBJECT, + windows.LABEL_SECURITY_INFORMATION, nil, nil, nil, sacl); err != nil { + return nil, err + } + tmpDir, err := ioutil.TempDir(b.root, tmpPrefix) + if err != nil { + return nil, err + } + if err := windows.SetNamedSecurityInfo(tmpDir, windows.SE_FILE_OBJECT, + windows.LABEL_SECURITY_INFORMATION, nil, nil, nil, sacl); err != nil { + return nil, err + } + + wd, err := os.Open(workDir) + if err != nil { + return nil, err + } + return &Environment{ + root: workDir, + tmp: tmpDir, + wd: wd, + token: b.token, + sd: b.sd, + }, nil +} diff --git a/pkg/winc/environment_windows.go b/pkg/winc/environment_windows.go new file mode 100644 index 0000000..18f9f13 --- /dev/null +++ b/pkg/winc/environment_windows.go @@ -0,0 +1,409 @@ +package winc + +import ( + "context" + "crypto/rand" + "encoding/hex" + "errors" + "fmt" + "os" + "path" + "syscall" + "unicode/utf16" + "unsafe" + + "github.com/criyle/go-judge/pkg/envexec" + "github.com/criyle/go-judge/pkg/pool" + "github.com/criyle/go-sandbox/runner" + "golang.org/x/sys/windows" +) + +var _ pool.Environment = &Environment{} + +var ( + errFileCount = errors.New("windows requires std handle to be 3") + errArgs = errors.New("executable name is required") +) + +// Environment implements envexec.Environment interface +type Environment struct { + root string + tmp string + wd *os.File + token windows.Token + sd *windows.SECURITY_DESCRIPTOR +} + +// Execve implements windows sandbox .. +func (e *Environment) Execve(ctx context.Context, param envexec.ExecveParam) (proc envexec.Process, err error) { + if len(param.Files) != 3 { + return nil, errFileCount + } + if len(param.Args) == 0 { + return nil, errArgs + } + + argv0, err := joinExeDirAndFName(e.root, param.Args[0]) + if err != nil { + return nil, err + } + param.Args[0] = argv0 + + // cleanUp defines clean up functions to call when error happens + var cleanUp []func() + handle := func(f func()) { + cleanUp = append(cleanUp, f) + } + defer func() { + if err != nil { + for i := len(cleanUp) - 1; i >= 0; i-- { + cleanUp[i]() + } + } + }() + + // create desktop + random := make([]byte, 8) + if _, err := rand.Read(random); err != nil { + return nil, err + } + + deskName := fmt.Sprintf("winc_%08x_%s", windows.GetCurrentProcessId(), hex.EncodeToString(random)) + deskNameW := syscall.StringToUTF16Ptr(deskName) + + sa := windows.SecurityAttributes{ + Length: uint32(unsafe.Sizeof(windows.SecurityAttributes{})), + SecurityDescriptor: e.sd, + } + + newDesk, err := CreateDesktop(deskNameW, nil, 0, 0, GENERIC_ALL, &sa) + if err != nil { + return nil, err + } + handle(func() { CloseDesktop(newDesk) }) + + // create job object + hJob, err := createJobObject(param.Limit) + if err != nil { + return nil, err + } + handle(func() { windows.CloseHandle(hJob) }) + + // create IOCP + ioPort, err := windows.CreateIoCompletionPort(windows.InvalidHandle, 0, 0, 1) + if err != nil { + return nil, err + } + handle(func() { windows.CloseHandle(ioPort) }) + + completionPort := &JOBOBJECT_ASSOCIATE_COMPLETION_PORT{ + CompletionKey: hJob, + CompletionPort: ioPort, + } + if _, err := windows.SetInformationJobObject(hJob, windows.JobObjectAssociateCompletionPortInformation, + uintptr(unsafe.Pointer(completionPort)), uint32(unsafe.Sizeof(*completionPort))); err != nil { + return nil, err + } + + cmdLine := makeCmdLine(param.Args) + cmdLineW := syscall.StringToUTF16Ptr(cmdLine) + dirW := syscall.StringToUTF16Ptr(e.root) + + var startupInfo syscall.StartupInfo + startupInfo.Cb = uint32(unsafe.Sizeof(startupInfo)) + startupInfo.Flags |= windows.STARTF_USESTDHANDLES // STARTF_FORCEOFFFEEDBACK + startupInfo.Desktop = deskNameW + + var env []string + env = append(env, param.Env...) + env = append(env, "TMP="+e.tmp) + env = append(env, "TEMP="+e.tmp) + env = append(env, "LocalAppData="+e.tmp) + eb := createEnvBlock(env) + + // do create process + var processInfo syscall.ProcessInformation + err = func() error { + syscall.ForkLock.Lock() + defer syscall.ForkLock.Unlock() + + curProc := windows.CurrentProcess() + fd := make([]windows.Handle, len(param.Files)) + for i := range param.Files { + if param.Files[i] <= 0 { + continue + } + if err := windows.DuplicateHandle(curProc, windows.Handle(param.Files[i]), + curProc, &fd[i], 0, true, syscall.DUPLICATE_SAME_ACCESS); err != nil { + return err + } + defer windows.CloseHandle(fd[i]) + } + startupInfo.StdInput = syscall.Handle(fd[0]) + startupInfo.StdOutput = syscall.Handle(fd[1]) + startupInfo.StdErr = syscall.Handle(fd[2]) + + return syscall.CreateProcessAsUser(syscall.Token(e.token), nil, cmdLineW, nil, nil, true, + windows.CREATE_NEW_PROCESS_GROUP|windows.CREATE_NEW_CONSOLE| + windows.CREATE_SUSPENDED|windows.CREATE_UNICODE_ENVIRONMENT, + eb, dirW, &startupInfo, &processInfo) + }() + + if processInfo.Process > 0 { + handle(func() { syscall.CloseHandle(processInfo.Process) }) + } + if processInfo.Thread > 0 { + handle(func() { syscall.CloseHandle(processInfo.Thread) }) + } + + if err != nil { + return nil, err + } + + // assign process to job object + if err := windows.AssignProcessToJobObject(hJob, windows.Handle(processInfo.Process)); err != nil { + return nil, err + } + + // resume thread + if _, err := windows.ResumeThread(windows.Handle(processInfo.Thread)); err != nil { + return nil, err + } + syscall.CloseHandle(processInfo.Thread) + + done := make(chan struct{}) + + procSet := &process{ + done: done, + hProcess: windows.Handle(processInfo.Process), + hJob: hJob, + } + + // wait for ctx to terminate + go func() { + select { + case <-ctx.Done(): + windows.TerminateJobObject(hJob, 1) + case <-done: + } + }() + + // wait for job object to finish + go func() { + defer CloseDesktop(newDesk) + defer syscall.CloseHandle(processInfo.Process) + defer windows.CloseHandle(hJob) + defer windows.CloseHandle(ioPort) + defer close(done) + + var ( + qty, key uint32 + overlapped *windows.Overlapped + ) + result := runner.Result{ + Status: runner.StatusNormal, + } + + loop: + for { + err = windows.GetQueuedCompletionStatus(ioPort, &qty, &key, &overlapped, windows.INFINITE) + if err != nil { + procSet.result = runner.Result{Status: runner.StatusRunnerError, Error: err.Error()} + return + } + switch qty { + case JOB_OBJECT_MSG_ACTIVE_PROCESS_ZERO: + break loop + + case JOB_OBJECT_MSG_END_OF_JOB_TIME, JOB_OBJECT_MSG_END_OF_PROCESS_TIME: + result.Status = runner.StatusTimeLimitExceeded + windows.TerminateJobObject(hJob, 0) + + case JOB_OBJECT_MSG_ACTIVE_PROCESS_LIMIT: + result.Status = runner.StatusMemoryLimitExceeded + windows.TerminateJobObject(hJob, 0) + + // case JOB_OBJECT_MSG_ABNORMAL_EXIT_PROCESS: + // result.Status = runner.StatusNonzeroExitStatus + // windows.TerminateJobObject(hJob, 0) + + case JOB_OBJECT_MSG_PROCESS_MEMORY_LIMIT, JOB_OBJECT_MSG_JOB_MEMORY_LIMIT: + result.Status = runner.StatusMemoryLimitExceeded + windows.TerminateJobObject(hJob, 0) + // JOB_OBJECT_MSG_NEW_PROCESS, JOB_OBJECT_MSG_EXIT_PROCESS + } + } + // collect exit status + if _, err := windows.WaitForSingleObject(windows.Handle(processInfo.Process), windows.INFINITE); err != nil { + procSet.result = runner.Result{Status: runner.StatusRunnerError, Error: err.Error()} + return + } + + var exitCode uint32 + if err := windows.GetExitCodeProcess(windows.Handle(processInfo.Process), &exitCode); err != nil { + procSet.result = runner.Result{Status: runner.StatusRunnerError, Error: err.Error()} + return + } + if exitCode != 0 { + result.Status = runner.StatusNonzeroExitStatus + } + result.ExitStatus = int(exitCode) + + // collect usage + t, m, err := getJobOjbectUsage(hJob) + if err != nil { + procSet.result = runner.Result{Status: runner.StatusRunnerError, Error: err.Error()} + return + } + result.Time = t + result.Memory = m + procSet.result = result + }() + + return procSet, nil +} + +// WorkDir returns the work directory +func (e *Environment) WorkDir() *os.File { + e.wd.Seek(0, 0) + return e.wd +} + +// Open opens file related to root +func (e *Environment) Open(p string, flags int, perm os.FileMode) (*os.File, error) { + return os.OpenFile(path.Join(e.root, p), flags, perm) +} + +// Destroy destorys the environment +func (e *Environment) Destroy() error { + return e.wd.Close() +} + +// Reset remove all files in root directory +func (e *Environment) Reset() error { + return removeContents(e.root) +} + +// removeContents delete content of a directory +func removeContents(dir string) error { + d, err := os.Open(dir) + if err != nil { + return err + } + defer d.Close() + + names, err := d.Readdirnames(-1) + if err != nil { + return err + } + + for _, name := range names { + err = os.RemoveAll(path.Join(dir, name)) + if err != nil { + return err + } + } + return nil +} + +// From syscall/exec_windows.go + +// makeCmdLine builds a command line out of args by escaping "special" +// characters and joining the arguments with spaces. +func makeCmdLine(args []string) string { + var s string + for _, v := range args { + if s != "" { + s += " " + } + s += syscall.EscapeArg(v) + } + return s +} + +// createEnvBlock converts an array of environment strings into +// the representation required by CreateProcess: a sequence of NUL +// terminated strings followed by a nil. +// Last bytes are two UCS-2 NULs, or four NUL bytes. +func createEnvBlock(envv []string) *uint16 { + if len(envv) == 0 { + return &utf16.Encode([]rune("\x00\x00"))[0] + } + length := 0 + for _, s := range envv { + length += len(s) + 1 + } + length++ + + b := make([]byte, length) + i := 0 + for _, s := range envv { + l := len(s) + copy(b[i:i+l], []byte(s)) + copy(b[i+l:i+l+1], []byte{0}) + i = i + l + 1 + } + copy(b[i:i+1], []byte{0}) + + return &utf16.Encode([]rune(string(b)))[0] +} + +func isSlash(c uint8) bool { + return c == '\\' || c == '/' +} + +func normalizeDir(dir string) (name string, err error) { + ndir, err := syscall.FullPath(dir) + if err != nil { + return "", err + } + if len(ndir) > 2 && isSlash(ndir[0]) && isSlash(ndir[1]) { + // dir cannot have \\server\share\path form + return "", syscall.EINVAL + } + return ndir, nil +} + +func volToUpper(ch int) int { + if 'a' <= ch && ch <= 'z' { + ch += 'A' - 'a' + } + return ch +} + +func joinExeDirAndFName(dir, p string) (name string, err error) { + if len(p) == 0 { + return "", syscall.EINVAL + } + if len(p) > 2 && isSlash(p[0]) && isSlash(p[1]) { + // \\server\share\path form + return p, nil + } + if len(p) > 1 && p[1] == ':' { + // has drive letter + if len(p) == 2 { + return "", syscall.EINVAL + } + if isSlash(p[2]) { + return p, nil + } + d, err := normalizeDir(dir) + if err != nil { + return "", err + } + if volToUpper(int(p[0])) == volToUpper(int(d[0])) { + return syscall.FullPath(d + "\\" + p[2:]) + } + return syscall.FullPath(p) + + } + // no drive letter + d, err := normalizeDir(dir) + if err != nil { + return "", err + } + if isSlash(p[0]) { + return syscall.FullPath(d[:2] + p) + } + return syscall.FullPath(d + "\\" + p) +} diff --git a/pkg/winc/job_object_windows.go b/pkg/winc/job_object_windows.go new file mode 100644 index 0000000..c4ce451 --- /dev/null +++ b/pkg/winc/job_object_windows.go @@ -0,0 +1,64 @@ +package winc + +import ( + "time" + "unsafe" + + "github.com/criyle/go-judge/pkg/envexec" + "golang.org/x/sys/windows" +) + +func createJobObject(envLimit envexec.Limit) (windows.Handle, error) { + // create job object + hJob, err := windows.CreateJobObject(nil, nil) + if err != nil { + return 0, err + } + + // define job object limits + var limit windows.JOBOBJECT_EXTENDED_LIMIT_INFORMATION + + // time limits: 100 nanoseconds / unit + if envLimit.Time > 0 { + limit.BasicLimitInformation.LimitFlags |= windows.JOB_OBJECT_LIMIT_JOB_TIME + limit.BasicLimitInformation.PerJobUserTimeLimit = int64(envLimit.Time.Round(100*time.Nanosecond).Nanoseconds() / 100) + } + + // memory limits: byte + if envLimit.Memory > 0 { + limit.BasicLimitInformation.LimitFlags |= windows.JOB_OBJECT_LIMIT_JOB_MEMORY + limit.JobMemoryLimit = uintptr(envLimit.Memory) + } + + // process limit + if envLimit.Proc > 0 { + limit.BasicLimitInformation.LimitFlags |= windows.JOB_OBJECT_LIMIT_ACTIVE_PROCESS + limit.BasicLimitInformation.ActiveProcessLimit = uint32(envLimit.Proc) + } + + // additional limitations + limit.BasicLimitInformation.LimitFlags |= windows.JOB_OBJECT_LIMIT_DIE_ON_UNHANDLED_EXCEPTION | windows.JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE + _, err = windows.SetInformationJobObject(hJob, windows.JobObjectExtendedLimitInformation, uintptr(unsafe.Pointer(&limit)), uint32(unsafe.Sizeof(limit))) + if err != nil { + windows.CloseHandle(hJob) + return 0, err + } + + // ui restrictions + var uiRestriction windows.JOBOBJECT_BASIC_UI_RESTRICTIONS + uiRestriction.UIRestrictionsClass = windows.JOB_OBJECT_UILIMIT_EXITWINDOWS | + windows.JOB_OBJECT_UILIMIT_DESKTOP | + windows.JOB_OBJECT_UILIMIT_DISPLAYSETTINGS | + windows.JOB_OBJECT_UILIMIT_GLOBALATOMS | + windows.JOB_OBJECT_UILIMIT_HANDLES | + windows.JOB_OBJECT_UILIMIT_READCLIPBOARD | + windows.JOB_OBJECT_UILIMIT_SYSTEMPARAMETERS | + windows.JOB_OBJECT_UILIMIT_WRITECLIPBOARD + + _, err = windows.SetInformationJobObject(hJob, windows.JobObjectBasicUIRestrictions, uintptr(unsafe.Pointer(&uiRestriction)), uint32(unsafe.Sizeof(uiRestriction))) + if err != nil { + windows.CloseHandle(hJob) + return 0, err + } + return hJob, nil +} diff --git a/pkg/winc/mksyscall.go b/pkg/winc/mksyscall.go new file mode 100644 index 0000000..7996c3f --- /dev/null +++ b/pkg/winc/mksyscall.go @@ -0,0 +1,3 @@ +package winc + +//go:generate go run golang.org/x/sys/windows/mkwinsyscall -output zsyscall_windows.go syscall_windows.go diff --git a/pkg/winc/process_windows.go b/pkg/winc/process_windows.go new file mode 100644 index 0000000..d4ee69c --- /dev/null +++ b/pkg/winc/process_windows.go @@ -0,0 +1,53 @@ +package winc + +import ( + "time" + "unsafe" + + "github.com/criyle/go-judge/pkg/envexec" + "github.com/criyle/go-sandbox/runner" + "golang.org/x/sys/windows" +) + +var _ envexec.Process = &process{} + +type process struct { + done chan struct{} + result runner.Result + hJob windows.Handle + hProcess windows.Handle +} + +func (p *process) Done() <-chan struct{} { + return p.done +} + +func (p *process) Result() runner.Result { + <-p.done + return p.result +} + +func (p *process) Usage() envexec.Usage { + t, m, _ := getJobOjbectUsage(p.hJob) + return envexec.Usage{ + Time: t, + Memory: m, + } +} + +func getJobOjbectUsage(hJob windows.Handle) (time.Duration, runner.Size, error) { + basicInfo := new(JOBOBJECT_BASIC_ACCOUNTING_INFORMATION) + if _, err := QueryInformationJobObject(hJob, JobObjectBasicAccountingInformation, + uintptr(unsafe.Pointer(basicInfo)), uint32(unsafe.Sizeof(*basicInfo)), nil); err != nil { + return 0, 0, err + } + t := time.Duration(basicInfo.TotalUserTime) * 100 // 100 nanosecond tick + + extendedLimitInfo := new(windows.JOBOBJECT_EXTENDED_LIMIT_INFORMATION) + if _, err := QueryInformationJobObject(hJob, windows.JobObjectExtendedLimitInformation, + uintptr(unsafe.Pointer(extendedLimitInfo)), uint32(unsafe.Sizeof(*extendedLimitInfo)), nil); err != nil { + return 0, 0, err + } + m := runner.Size(extendedLimitInfo.PeakJobMemoryUsed) + return t, m, nil +} diff --git a/pkg/winc/syscall_windows.go b/pkg/winc/syscall_windows.go new file mode 100644 index 0000000..8bdd3e2 --- /dev/null +++ b/pkg/winc/syscall_windows.go @@ -0,0 +1,121 @@ +package winc + +import "golang.org/x/sys/windows" + +// JOBOBJECT_ASSOCIATE_COMPLETION_PORT defines IOCP for job object +type JOBOBJECT_ASSOCIATE_COMPLETION_PORT struct { + CompletionKey windows.Handle + CompletionPort windows.Handle +} + +// JOBOBJECT_BASIC_ACCOUNTING_INFORMATION defines basic accounting information for a job object +type JOBOBJECT_BASIC_ACCOUNTING_INFORMATION struct { + TotalUserTime int64 + TotalKernelTime int64 + ThisPeriodTotalUserTime int64 + ThisPeriodTotalKernelTime int64 + TotalPageFaultCount uint32 + TotalProcesses uint32 + ActiveProcesses uint32 + TotalTerminatedProcess uint32 +} + +// JobObject Information class +const ( + JobObjectBasicAccountingInformation = iota + 1 + JobObjectBasicLimitInformation + JobObjectBasicProcessIDList + JobObjectBasicUIRestrictions + JobObjectSecurityLimitInformation + JobObjectEndOfJobTimeInformation + JobObjectAssociateCompletionPortInformation + JobObjectBasicAndIoAccountingInformation + JobObjectExtendedLimitInformation + _ + JobObjectGroupInformation + JobObjectNotificationLimitInformation + JobObjectLimitViolationInformation + JobObjectGroupInformationEx + JobObjectCPURateControlInformation +) + +// JobObject Messages https://chromium.googlesource.com/chromium/deps/mozprocess/+/11d11bebc8517dcedec71f377cbec07fb91a3b1f/mozprocess/winprocess.py +// https://docs.microsoft.com/en-ca/windows/win32/api/winnt/ns-winnt-jobobject_associate_completion_port?redirectedfrom=MSDN +const ( + JOB_OBJECT_MSG_END_OF_JOB_TIME = iota + 1 + JOB_OBJECT_MSG_END_OF_PROCESS_TIME + JOB_OBJECT_MSG_ACTIVE_PROCESS_LIMIT + JOB_OBJECT_MSG_ACTIVE_PROCESS_ZERO + JOB_OBJECT_MSG_NEW_PROCESS + JOB_OBJECT_MSG_EXIT_PROCESS + JOB_OBJECT_MSG_ABNORMAL_EXIT_PROCESS + JOB_OBJECT_MSG_PROCESS_MEMORY_LIMIT + JOB_OBJECT_MSG_JOB_MEMORY_LIMIT +) + +// https://docs.microsoft.com/en-us/windows/win32/api/securitybaseapi/nf-securitybaseapi-createrestrictedtoken +// CreateRestrictedToken +// CreateRestrictedToken Flags +const ( + DISABLE_MAX_PRIVILEGE = 1 << iota + SANDBOX_INERT + LUA_TOKEN + WRITE_RESTRICTED +) + +// https://docs.microsoft.com/en-ca/windows/win32/winstation/desktop-security-and-access-rights +const ( + DESKTOP_READOBJECTS windows.ACCESS_MASK = 1 << iota + DESKTOP_CREATEWINDOW + DESKTOP_CREATEMENU + DESKTOP_HOOKCONTROL + DESKTOP_JOURNALRECORD + DESKTOP_JOURNALPLAYBACK + DESKTOP_ENUMERATE + DESKTOP_WRITEOBJECTS + DESKTOP_SWITCHDESKTOP // 0x0100L +) + +// permissions +const ( + DELETE windows.ACCESS_MASK = 1 << (iota + 16) + READ_CONTROL + WRITE_DAC + WRITE_OWNER + SYNCHRONIZE +) + +// https://docs.microsoft.com/en-us/windows/win32/secauthz/standard-access-rights +const ( + GENERIC_READ = DESKTOP_ENUMERATE | DESKTOP_READOBJECTS | READ_CONTROL + GENERIC_WRITE = DESKTOP_CREATEMENU | DESKTOP_CREATEWINDOW | + DESKTOP_HOOKCONTROL | DESKTOP_JOURNALPLAYBACK | DESKTOP_JOURNALRECORD | + DESKTOP_WRITEOBJECTS | READ_CONTROL + GENERIC_EXECUTE = DESKTOP_SWITCHDESKTOP | READ_CONTROL + GENERIC_ALL = DESKTOP_CREATEMENU | DESKTOP_CREATEWINDOW | + DESKTOP_ENUMERATE | DESKTOP_HOOKCONTROL | DESKTOP_JOURNALPLAYBACK | + DESKTOP_JOURNALRECORD | DESKTOP_READOBJECTS | DESKTOP_SWITCHDESKTOP | + DESKTOP_WRITEOBJECTS | READ_CONTROL | WRITE_DAC | WRITE_OWNER +) + +// HDESK is handle for desktop +type HDESK windows.Handle + +// HWINSTA is handle for windows station +type HWINSTA windows.Handle + +// Mandatory Level Sids +const ( + SID_SYSTEM_MANDATORY_LEVEL = "S-1-16-16384" + SID_HIGH_MANDATORY_LEVEL = "S-1-16-12288" + SID_MEDIUM_MANDATORY_LEVEL = "S-1-16-8192" + SID_LOW_MANDATORY_LEVEL = "S-1-16-4096" + SID_UNTRUSTED_MANDATORY_LEVEL = "S-1-16-0" +) + +//sys CreateRestrictedToken(existingToken windows.Token, flags uint32, disableSidCount uint32, sidsToDisable *windows.SIDAndAttributes, deletePrivilegeCount uint32, privilegesToDelete *windows.SIDAndAttributes, restrictedSidCount uint32, sidToRestrict *windows.SIDAndAttributes, newTokenHandle *windows.Token) (err error) = advapi32.CreateRestrictedToken +//sys GetThreadDesktop(threadID uint32) (h HDESK) = user32.GetThreadDesktop +//sys GetProcessWindowStation() (h HWINSTA) = user32.GetProcessWindowStation +//sys CreateDesktop(lpszDesktop *uint16, lpszDevice *uint16, pDevmode uintptr, dwFlags uint32, dwDesiredAccess windows.ACCESS_MASK, lpsa *windows.SecurityAttributes) (h HDESK, err error) = user32.CreateDesktopW +//sys CloseDesktop(hDesktop HDESK) (err error) = user32.CloseDesktop +//sys QueryInformationJobObject(job windows.Handle, JobObjectInformationClass uint32, JobObjectInformation uintptr, JobObjectInformationLength uint32, lpReturnLength *uint32) (ret int, err error) = kernel32.QueryInformationJobObject diff --git a/pkg/winc/token_windows.go b/pkg/winc/token_windows.go new file mode 100644 index 0000000..0f3305c --- /dev/null +++ b/pkg/winc/token_windows.go @@ -0,0 +1,43 @@ +package winc + +import ( + "syscall" + "unsafe" + + "golang.org/x/sys/windows" +) + +func createLowMandatoryLevelToken() (token windows.Token, err error) { + // Get current process token + var procToken windows.Token + if err := windows.OpenProcessToken(windows.CurrentProcess(), + windows.TOKEN_QUERY|windows.TOKEN_DUPLICATE|windows.TOKEN_ADJUST_DEFAULT|windows.TOKEN_ASSIGN_PRIMARY, + &procToken); err != nil { + return token, err + } + + // create restricted token + if err := CreateRestrictedToken(procToken, DISABLE_MAX_PRIVILEGE, 0, nil, 0, nil, 0, nil, &token); err != nil { + return token, err + } + defer func() { + if err != nil { + token.Close() + } + }() + + lowSid, err := windows.StringToSid(SID_LOW_MANDATORY_LEVEL) + if err != nil { + return token, err + } + tml := windows.Tokenmandatorylabel{ + Label: windows.SIDAndAttributes{ + Sid: lowSid, + Attributes: windows.SE_GROUP_INTEGRITY, + }, + } + if err = windows.SetTokenInformation(token, syscall.TokenIntegrityLevel, (*byte)(unsafe.Pointer(&tml)), uint32(unsafe.Sizeof(tml))); err != nil { + return token, err + } + return token, nil +} diff --git a/pkg/winc/zsyscall_windows.go b/pkg/winc/zsyscall_windows.go new file mode 100644 index 0000000..82c90c8 --- /dev/null +++ b/pkg/winc/zsyscall_windows.go @@ -0,0 +1,112 @@ +// Code generated by 'go generate'; DO NOT EDIT. + +package winc + +import ( + "syscall" + "unsafe" + + "golang.org/x/sys/windows" +) + +var _ unsafe.Pointer + +// Do the interface allocations only once for common +// Errno values. +const ( + errnoERROR_IO_PENDING = 997 +) + +var ( + errERROR_IO_PENDING error = syscall.Errno(errnoERROR_IO_PENDING) +) + +// errnoErr returns common boxed Errno values, to prevent +// allocations at runtime. +func errnoErr(e syscall.Errno) error { + switch e { + case 0: + return nil + case errnoERROR_IO_PENDING: + return errERROR_IO_PENDING + } + // TODO: add more here, after collecting data on the common + // error values see on Windows. (perhaps when running + // all.bat?) + return e +} + +var ( + modadvapi32 = windows.NewLazySystemDLL("advapi32.dll") + moduser32 = windows.NewLazySystemDLL("user32.dll") + modkernel32 = windows.NewLazySystemDLL("kernel32.dll") + + procCreateRestrictedToken = modadvapi32.NewProc("CreateRestrictedToken") + procGetThreadDesktop = moduser32.NewProc("GetThreadDesktop") + procGetProcessWindowStation = moduser32.NewProc("GetProcessWindowStation") + procCreateDesktopW = moduser32.NewProc("CreateDesktopW") + procCloseDesktop = moduser32.NewProc("CloseDesktop") + procQueryInformationJobObject = modkernel32.NewProc("QueryInformationJobObject") +) + +func CreateRestrictedToken(existingToken windows.Token, flags uint32, disableSidCount uint32, sidsToDisable *windows.SIDAndAttributes, deletePrivilegeCount uint32, privilegesToDelete *windows.SIDAndAttributes, restrictedSidCount uint32, sidToRestrict *windows.SIDAndAttributes, newTokenHandle *windows.Token) (err error) { + r1, _, e1 := syscall.Syscall9(procCreateRestrictedToken.Addr(), 9, uintptr(existingToken), uintptr(flags), uintptr(disableSidCount), uintptr(unsafe.Pointer(sidsToDisable)), uintptr(deletePrivilegeCount), uintptr(unsafe.Pointer(privilegesToDelete)), uintptr(restrictedSidCount), uintptr(unsafe.Pointer(sidToRestrict)), uintptr(unsafe.Pointer(newTokenHandle))) + if r1 == 0 { + if e1 != 0 { + err = errnoErr(e1) + } else { + err = syscall.EINVAL + } + } + return +} + +func GetThreadDesktop(threadID uint32) (h HDESK) { + r0, _, _ := syscall.Syscall(procGetThreadDesktop.Addr(), 1, uintptr(threadID), 0, 0) + h = HDESK(r0) + return +} + +func GetProcessWindowStation() (h HWINSTA) { + r0, _, _ := syscall.Syscall(procGetProcessWindowStation.Addr(), 0, 0, 0, 0) + h = HWINSTA(r0) + return +} + +func CreateDesktop(lpszDesktop *uint16, lpszDevice *uint16, pDevmode uintptr, dwFlags uint32, dwDesiredAccess windows.ACCESS_MASK, lpsa *windows.SecurityAttributes) (h HDESK, err error) { + r0, _, e1 := syscall.Syscall6(procCreateDesktopW.Addr(), 6, uintptr(unsafe.Pointer(lpszDesktop)), uintptr(unsafe.Pointer(lpszDevice)), uintptr(pDevmode), uintptr(dwFlags), uintptr(dwDesiredAccess), uintptr(unsafe.Pointer(lpsa))) + h = HDESK(r0) + if h == 0 { + if e1 != 0 { + err = errnoErr(e1) + } else { + err = syscall.EINVAL + } + } + return +} + +func CloseDesktop(hDesktop HDESK) (err error) { + r1, _, e1 := syscall.Syscall(procCloseDesktop.Addr(), 1, uintptr(hDesktop), 0, 0) + if r1 == 0 { + if e1 != 0 { + err = errnoErr(e1) + } else { + err = syscall.EINVAL + } + } + return +} + +func QueryInformationJobObject(job windows.Handle, JobObjectInformationClass uint32, JobObjectInformation uintptr, JobObjectInformationLength uint32, lpReturnLength *uint32) (ret int, err error) { + r0, _, e1 := syscall.Syscall6(procQueryInformationJobObject.Addr(), 5, uintptr(job), uintptr(JobObjectInformationClass), uintptr(JobObjectInformation), uintptr(JobObjectInformationLength), uintptr(unsafe.Pointer(lpReturnLength)), 0) + ret = int(r0) + if ret == 0 { + if e1 != 0 { + err = errnoErr(e1) + } else { + err = syscall.EINVAL + } + } + return +}