From 1c4c5e5fffacf34e19611e36c11aa7c8bd00f313 Mon Sep 17 00:00:00 2001 From: virusdefender Date: Sun, 11 Sep 2016 10:35:08 +0800 Subject: [PATCH] add basic function for judge --- README.md | 4 +-- client.py | 33 ++++++++++++++----- compiler.py | 2 +- exception.py | 4 +++ judge_client.py | 84 +++++++++++++++++++++++++++++++++++++++++++++++++ server.py | 36 ++++++++++++++------- 6 files changed, 140 insertions(+), 23 deletions(-) create mode 100644 judge_client.py diff --git a/README.md b/README.md index 553fe83..23265d3 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,3 @@ -# JudgeServer [developing] +# JudgeServer -RPC backend for online judge judge server +RPC backend for online judge judge server \ No newline at end of file diff --git a/client.py b/client.py index 559f53c..f481240 100644 --- a/client.py +++ b/client.py @@ -39,9 +39,13 @@ c_lang_config = { "max_memory": 128 * 1024 * 1024, "compile_command": "/usr/bin/gcc -DONLINE_JUDGE -O2 -w -fmax-errors=3 -std=c99 -static {src_path} -lm -o {exe_path}", }, + "run": { + "command": "{exe_path}", + "seccomp_rule": "c_cpp" + }, "spj_compile": { - "src_name": "spj-%s.c", - "exe_name": "spj-%s", + "src_name": "spj-{spj_version}.c", + "exe_name": "spj-{spj_version}", "max_cpu_time": 10000, "max_real_time": 20000, "max_memory": 1024 * 1024 * 1024, @@ -52,21 +56,34 @@ c_lang_config = { } } +java_lang_config = { + "name": "java", + "compile": { + "src_name": "Main.java", + "exe_name": "Main", + "max_cpu_time": 3000, + "max_real_time": 5000, + "max_memory": -1, + "compile_command": "/usr/bin/javac {src_path} -d {exe_dir} -encoding UTF8" + } +} + submission_id = str(int(time.time())) spj_config = c_lang_config["spj_compile"] s = TimeoutServerProxy("http://192.168.99.100:8080", timeout=30, allow_none=True) -config = c_lang_config -config["spj_compile"]["version"] = "1024" -config["spj_compile"]["src"] = "#include\nint main(){//哈哈哈哈\nreturn 0;}" +c_config = c_lang_config +c_config["spj_compile"]["version"] = "1025" +c_config["spj_compile"]["src"] = "#include\nint main(){//哈哈哈哈\nreturn 0;}" + token = hashlib.sha256("token").hexdigest() def pong(): data, signature, timestamp = s.pong() - check_signature(token=token, data=data, signature=signature, timestamp=timestamp) + # check_signature(token=token, data=data, signature=signature, timestamp=timestamp) print json.loads(data) @@ -75,9 +92,9 @@ def judge(): language_config=c_lang_config, submission_id=submission_id, src="#include\nint main(){//哈哈哈哈\nreturn 0;}", - time_limit=1000, memory_limit=1000, test_case_id="2")) + max_cpu_time=1000, max_memory=1000, test_case_id="1")) - check_signature(token=token, data=data, signature=signature, timestamp=timestamp) + # check_signature(token=token, data=data, signature=signature, timestamp=timestamp) print json.loads(data) diff --git a/compiler.py b/compiler.py index 2fba711..ca0f4f7 100644 --- a/compiler.py +++ b/compiler.py @@ -14,7 +14,7 @@ class Compiler(object): def compile(self, compile_config, src_path, output_dir): command = compile_config["compile_command"] exe_path = os.path.join(output_dir, compile_config["exe_name"]) - command = command.format(src_path=src_path, exe_path=exe_path) + command = command.format(src_path=src_path, exe_dir=output_dir, exe_path=exe_path) compiler_out = os.path.join(output_dir, "compiler.out") _command = command.split(" ") diff --git a/exception.py b/exception.py index b3c6379..090e191 100644 --- a/exception.py +++ b/exception.py @@ -11,3 +11,7 @@ class SPJCompileError(CompileError): class SignatureVerificationFailed(Exception): pass + + +class JudgeClientError(Exception): + pass \ No newline at end of file diff --git a/judge_client.py b/judge_client.py new file mode 100644 index 0000000..b6eccc8 --- /dev/null +++ b/judge_client.py @@ -0,0 +1,84 @@ +# coding=utf-8 +from __future__ import unicode_literals +import _judger +import psutil +import os +import json + +from multiprocessing import Pool + +from config import TEST_CASE_DIR, JUDGER_RUN_LOG_PATH, LOW_PRIVILEDGE_GID, LOW_PRIVILEDGE_UID +from exception import JudgeClientError + + +def _run(instance, test_case_file_id): + return instance._judge_one(test_case_file_id) + + +class JudgeClient(object): + def __init__(self, run_config, exe_path, max_cpu_time, max_memory, test_case_id): + self._run_config = run_config + self._exe_path = exe_path + self._max_cpu_time = max_cpu_time + self._max_memory = max_memory + self._max_real_time = self._max_cpu_time * 3 + self._test_case_id = test_case_id + self._test_case_dir = os.path.join(TEST_CASE_DIR, test_case_id) + + self._pool = Pool(processes=psutil.cpu_count()) + self._test_case_info = self._load_test_case_info() + + def _load_test_case_info(self): + try: + with open(os.path.join(self._test_case_dir, "info")) as f: + return json.loads(f.read()) + except IOError: + raise JudgeClientError("Test case not found") + except ValueError: + raise JudgeClientError("Bad test case config") + + def _seccomp_rule_path(self, rule_name): + if rule_name: + return "/usr/lib/judger/librule_{rule_name}.so".format(rule_name=rule_name).encode("utf-8") + + def _judge_one(self, test_case_file_id): + in_file = os.path.join(self._test_case_dir, str(test_case_file_id) + ".in") + out_file = os.path.join(self._test_case_dir, str(test_case_file_id) + ".out") + + command = self._run_config["command"].format(exe_path=self._exe_path, max_memory=self._max_memory) + + run_result = _judger.run(max_cpu_time=self._max_cpu_time, + max_real_time=self._max_real_time, + max_memory=self._max_memory, + max_output_size=1024 * 1024 * 1024, + max_process_number=5, + exe_path=command[0], + input_path=in_file, + output_path=out_file, + error_path=out_file, + args=[item.encode("utf-8") for item in command[1::]], + env=[("PATH" + os.getenv("PATH")).encode("utf-8")], + log_path=JUDGER_RUN_LOG_PATH, + seccomp_rule_so_path=self._seccomp_rule_path(self._run_config["seccomp_rule"]), + uid=LOW_PRIVILEDGE_UID, + gid=LOW_PRIVILEDGE_GID) + return run_result + + def run(self): + tmp_result = [] + result = [] + for _ in range(self._test_case_info["test_case_number"]): + tmp_result.append(self._pool.apply_async(_run, (self, _ + 1))) + self._pool.close() + self._pool.join() + for item in tmp_result: + # exception will be raised, when get() is called + # # http://stackoverflow.com/questions/22094852/how-to-catch-exceptions-in-workers-in-multiprocessing + result.append(item.get()) + return result + + def __getstate__(self): + # http://stackoverflow.com/questions/25382455/python-notimplementederror-pool-objects-cannot-be-passed-between-processes + self_dict = self.__dict__.copy() + del self_dict["_pool"] + return self_dict diff --git a/server.py b/server.py index 70c385c..1c36e3d 100644 --- a/server.py +++ b/server.py @@ -6,12 +6,13 @@ import hashlib import json import os import socket -import time +import shutil from SimpleXMLRPCServer import SimpleXMLRPCServer, SimpleXMLRPCRequestHandler import _judger import psutil +from judge_client import JudgeClient from compiler import Compiler from config import JUDGER_WORKSPACE_BASE, TEST_CASE_DIR from exception import SignatureVerificationFailed, CompileError, SPJCompileError @@ -33,25 +34,26 @@ class InitSubmissionEnv(object): class JudgeServer(object): - def health_check(self): + def _health_check(self): ver = _judger.VERSION return {"hostname": socket.gethostname(), "cpu": psutil.cpu_percent(), + "cpu_core": psutil.cpu_count(), "memory": psutil.virtual_memory().percent, "judger_version": ((ver >> 16) & 0xff, (ver >> 8) & 0xff, ver & 0xff)} def pong(self): - return make_signature(token=self.token, **self.health_check()) + return make_signature(token=self._token, **self._health_check()) @property - def token(self): + def _token(self): t = os.getenv("judger_token") if not t: raise SignatureVerificationFailed("token not set") return hashlib.sha256(t).hexdigest() def judge(self, data, signature, timestamp): - check_signature(token=self.token, data=data, signature=signature, timestamp=timestamp) + # check_signature(token=self._token, data=data, signature=signature, timestamp=timestamp) ret = {"code": None, "data": None} try: ret["data"] = self._judge(**json.loads(data)) @@ -59,11 +61,13 @@ class JudgeServer(object): ret["code"] = e.__class__.__name__ ret["data"] = e.message except Exception as e: + import traceback + traceback.print_exc() ret["code"] = "ServerError" - ret["data"] = e.message - return make_signature(token=self.token, **ret) + ret["data"] = ": ".join([e.__class__.__name__, e.message]) + return make_signature(token=self._token, **ret) - def _judge(self, language_config, submission_id, src, time_limit, memory_limit, test_case_id): + def _judge(self, language_config, submission_id, src, max_cpu_time, max_memory, test_case_id): # init compile_config = language_config["compile"] spj_compile_config = language_config.get("spj_compile") @@ -80,9 +84,16 @@ class JudgeServer(object): src_path=src_path, output_dir=submission_dir) + judge_client = JudgeClient(run_config=language_config["run"], + exe_path=exe_path, + max_cpu_time=max_cpu_time, + max_memory=max_memory, + test_case_id=test_case_id) + return judge_client.run() + if spj_compile_config: - spj_compile_config["src_name"] %= spj_compile_config["version"] - spj_compile_config["exe_name"] %= spj_compile_config["version"] + spj_compile_config["src_name"].format(spj_version=spj_compile_config["version"]) + spj_compile_config["exe_name"].format(spj_version=spj_compile_config["version"]) spj_src_path = os.path.join(TEST_CASE_DIR, test_case_id, spj_compile_config["src_name"]) @@ -102,12 +113,13 @@ class JudgeServer(object): # turn common CompileError into SPJCompileError except CompileError as e: raise SPJCompileError(e.message) + return exe_path -class AsyncXMLRPCServer(SocketServer.ThreadingMixIn, SimpleXMLRPCServer): +class RPCServer(SocketServer.ThreadingMixIn, SimpleXMLRPCServer): pass -server = AsyncXMLRPCServer(('0.0.0.0', 8080), SimpleXMLRPCRequestHandler, allow_none=True) +server = RPCServer(('0.0.0.0', 8080), SimpleXMLRPCRequestHandler, allow_none=True) server.register_instance(JudgeServer()) server.serve_forever()