diff --git a/build/Dockerfile b/build/Dockerfile index 91cbe45..b573593 100644 --- a/build/Dockerfile +++ b/build/Dockerfile @@ -1,16 +1,15 @@ FROM ubuntu:16.04 ENV DEBIAN_FRONTEND noninteractive -RUN apt-get update -RUN apt-get -y install software-properties-common python-software-properties python python-dev gcc g++ git libtool python-pip libseccomp-dev cmake +RUN apt-get update && apt-get -y install software-properties-common python-software-properties python python3 python-dev gcc g++ git libtool python3-pip libseccomp-dev cmake RUN add-apt-repository ppa:openjdk-r/ppa RUN apt-get update RUN apt-get install -y openjdk-7-jdk -RUN cd /tmp && git clone https://github.com/QingdaoU/Judger && cd Judger && git checkout newnew && mkdir build && cd build && cmake .. && make && make install && cd ../bindings/Python && python setup.py install +RUN cd /tmp && git clone https://github.com/QingdaoU/Judger && cd Judger && git checkout newnew && mkdir build && cd build && cmake .. && make && make install && cd ../bindings/Python && python3 setup.py install RUN mkdir -p /judger_run /test_case /log /code COPY java_policy /etc -RUN pip install futures psutil gunicorn web.py requests +RUN pip3 install futures psutil gunicorn web.py==0.40.dev0 requests RUN useradd -r compiler -HEALTHCHECK --interval=5s --retries=3 CMD python /code/service.py +HEALTHCHECK --interval=5s --retries=3 CMD python3 /code/service.py WORKDIR /code EXPOSE 8080 CMD /bin/bash /code/run.sh \ No newline at end of file diff --git a/server/compiler.py b/server/compiler.py index 5f6616f..42cdfbd 100644 --- a/server/compiler.py +++ b/server/compiler.py @@ -1,6 +1,3 @@ -# coding=utf-8 -from __future__ import unicode_literals - import json import os @@ -29,8 +26,8 @@ class Compiler(object): input_path=src_path, output_path=compiler_out, error_path=compiler_out, - args=[item.encode("utf-8") for item in _command[1::]], - env=[("PATH=" + os.getenv("PATH")).encode("utf-8")], + args=_command[1::], + env=["PATH=" + os.getenv("PATH")], log_path=COMPILER_LOG_PATH, seccomp_rule_name=None, uid=COMPILER_USER_UID, @@ -38,12 +35,12 @@ class Compiler(object): if result["result"] != _judger.RESULT_SUCCESS: if os.path.exists(compiler_out): - with open(compiler_out) as f: + with open(compiler_out, encoding="utf-8") as f: error = f.read().strip() os.remove(compiler_out) if error: raise CompileError(error) - raise CompileError("Compiler runtime error, info: %s" % json.dumps(result).decode("utf-8")) + raise CompileError("Compiler runtime error, info: %s" % json.dumps(result)) else: os.remove(compiler_out) return exe_path diff --git a/server/config.py b/server/config.py index 828bd99..59739b0 100644 --- a/server/config.py +++ b/server/config.py @@ -1,6 +1,3 @@ -# coding=utf-8 -from __future__ import unicode_literals - import grp import os import pwd @@ -8,8 +5,8 @@ import pwd JUDGER_WORKSPACE_BASE = "/judger_run" LOG_BASE = "/log" -COMPILER_LOG_PATH = os.path.join(LOG_BASE, "compile.log").encode("utf-8") -JUDGER_RUN_LOG_PATH = os.path.join(LOG_BASE, "judger.log").encode("utf-8") +COMPILER_LOG_PATH = os.path.join(LOG_BASE, "compile.log") +JUDGER_RUN_LOG_PATH = os.path.join(LOG_BASE, "judger.log") RUN_USER_UID = pwd.getpwnam("nobody").pw_uid RUN_GROUP_GID = grp.getgrnam("nogroup").gr_gid diff --git a/server/exception.py b/server/exception.py index 854f912..f8d9d75 100644 --- a/server/exception.py +++ b/server/exception.py @@ -1,6 +1,3 @@ -from __future__ import unicode_literals - - class CompileError(Exception): pass diff --git a/server/judge_client.py b/server/judge_client.py index 2e05aa2..189ef45 100644 --- a/server/judge_client.py +++ b/server/judge_client.py @@ -1,5 +1,3 @@ -# coding=utf-8 -from __future__ import unicode_literals import _judger import psutil import os @@ -46,7 +44,7 @@ class JudgeClient(object): def _load_test_case_info(self): try: - with open(os.path.join(self._test_case_dir, "info")) as f: + with open(os.path.join(self._test_case_dir, "info"), encoding="utf-8") as f: return json.loads(f.read()) except IOError: raise JudgeClientError("Test case not found") @@ -58,9 +56,9 @@ class JudgeClient(object): def _compare_output(self, test_case_file_id): user_output_file = os.path.join(self._submission_dir, str(test_case_file_id) + ".out") - with open(user_output_file, "r") as f: + with open(user_output_file, encoding="utf-8") as f: content = f.read() - output_md5 = hashlib.md5(content.rstrip()).hexdigest() + output_md5 = hashlib.md5(content.rstrip().encode("utf-8")).hexdigest() result = output_md5 == self._get_test_case_file_info(test_case_file_id)["striped_output_md5"] return output_md5, result @@ -68,19 +66,19 @@ class JudgeClient(object): command = self._spj_config["command"].format(exe_path=self._spj_exe, in_file_path=in_file_path, user_out_file_path=user_out_file_path).split(" ") - seccomp_rule_name = self._spj_config["seccomp_rule"].encode("utf-8") if self._spj_config["seccomp_rule"] else None + seccomp_rule_name = self._spj_config["seccomp_rule"] if self._spj_config["seccomp_rule"] else None result = _judger.run(max_cpu_time=self._max_cpu_time * 3, max_real_time=self._max_cpu_time * 9, max_memory=self._max_memory * 3, max_stack=128 * 1024 * 1024, max_output_size=1024 * 1024 * 1024, max_process_number=_judger.UNLIMITED, - exe_path=command[0].encode("utf-8"), - input_path=in_file_path.encode("utf-8"), - output_path="/tmp/spj.out".encode("utf-8"), - error_path="/tmp/spj.out".encode("utf-8"), - args=[item.encode("utf-8") for item in command[1::]], - env=[("PATH=" + os.environ.get("PATH", "")).encode("utf-8")], + exe_path=command[0], + input_path=in_file_path, + output_path="/tmp/spj.out", + error_path="/tmp/spj.out", + args=command[1::], + env=["PATH=" + os.environ.get("PATH", "")], log_path=JUDGER_RUN_LOG_PATH, seccomp_rule_name=seccomp_rule_name, uid=RUN_USER_UID, @@ -94,13 +92,13 @@ class JudgeClient(object): return SPJ_ERROR def _judge_one(self, test_case_file_id): - in_file = os.path.join(self._test_case_dir, self._get_test_case_file_info(test_case_file_id)["input_name"]).encode("utf-8") - user_output_file = os.path.join(self._submission_dir, test_case_file_id + ".out").encode("utf-8") + in_file = os.path.join(self._test_case_dir, self._get_test_case_file_info(test_case_file_id)["input_name"]) + user_output_file = os.path.join(self._submission_dir, test_case_file_id + ".out") command = self._run_config["command"].format(exe_path=self._exe_path, exe_dir=os.path.dirname(self._exe_path), - max_memory=self._max_memory / 1024).split(" ") - seccomp_rule_name = self._run_config["seccomp_rule"].encode("utf-8") if self._run_config["seccomp_rule"] else None - env = [item.encode("utf-8") for item in ["PATH=" + os.environ.get("PATH", "")] + self._run_config.get("env", [])] + max_memory=int(self._max_memory / 1024)).split(" ") + seccomp_rule_name = self._run_config["seccomp_rule"] if self._run_config["seccomp_rule"] else None + env = ["PATH=" + os.environ.get("PATH", "")] + self._run_config.get("env", []) run_result = _judger.run(max_cpu_time=self._max_cpu_time, max_real_time=self._max_real_time, @@ -108,11 +106,11 @@ class JudgeClient(object): max_stack=128 * 1024 * 1024, max_output_size=1024 * 1024 * 1024, max_process_number=_judger.UNLIMITED, - exe_path=command[0].encode("utf-8"), + exe_path=command[0], input_path=in_file, output_path=user_output_file, error_path=user_output_file, - args=[item.encode("utf-8") for item in command[1::]], + args=command[1::], env=env, log_path=JUDGER_RUN_LOG_PATH, seccomp_rule_name=seccomp_rule_name, @@ -143,8 +141,8 @@ class JudgeClient(object): if self._output: try: - with open(user_output_file, "r") as f: - run_result["output"] = f.read().decode("utf-8") + with open(user_output_file, encoding="utf-8") as f: + run_result["output"] = f.read() except Exception: pass @@ -153,7 +151,7 @@ class JudgeClient(object): def run(self): tmp_result = [] result = [] - for test_case_file_id, _ in self._test_case_info["test_cases"].iteritems(): + for test_case_file_id, _ in self._test_case_info["test_cases"].items(): tmp_result.append(self._pool.apply_async(_run, (self, test_case_file_id))) self._pool.close() self._pool.join() diff --git a/server/run.sh b/server/run.sh index 021a6f5..2bad495 100644 --- a/server/run.sh +++ b/server/run.sh @@ -1,8 +1,6 @@ #!/usr/bin/env bash chown compiler:compiler /spj -echo 0 > /tmp/counter core=$(grep --count ^processor /proc/cpuinfo) n=$(($core*4)) -chmod 400 /token.txt /tmp/counter chmod -R 400 /test_case gunicorn --workers $n --threads $n --error-logfile /log/gunicorn.log --time 600 --bind 0.0.0.0:8080 server:wsgiapp \ No newline at end of file diff --git a/server/server.py b/server/server.py index 5cb6299..d424409 100644 --- a/server/server.py +++ b/server/server.py @@ -1,6 +1,3 @@ -# coding=utf-8 -from __future__ import unicode_literals - import json import os import shutil @@ -25,7 +22,7 @@ class InitSubmissionEnv(object): def __enter__(self): try: os.mkdir(self.path) - os.chmod(self.path, 0777) + os.chmod(self.path, 0o777) except Exception as e: logger.exception(e) raise JudgeClientError("failed to create runtime dir") @@ -58,13 +55,13 @@ class JudgeServer(object): spj_compile_config=spj_compile_config, test_case_id=test_case_id) - with InitSubmissionEnv(JUDGER_WORKSPACE_BASE, submission_id=str(submission_id)) as submission_dir: + with InitSubmissionEnv(JUDGER_WORKSPACE_BASE, submission_id=submission_id) as submission_dir: if compile_config: src_path = os.path.join(submission_dir, compile_config["src_name"]) # write source code into file - with open(src_path, "w") as f: - f.write(src.encode("utf-8")) + with open(src_path, "w", encoding="utf-8") as f: + f.write(src) # compile source code, return exe file path exe_path = Compiler().compile(compile_config=compile_config, @@ -72,8 +69,8 @@ class JudgeServer(object): output_dir=submission_dir) else: exe_path = os.path.join(submission_dir, run_config["exe_name"]) - with open(exe_path, "w") as f: - f.write(src.encode("utf-8")) + with open(exe_path, "w", encoding="utf-8") as f: + f.write(src) judge_client = JudgeClient(run_config=language_config["run"], exe_path=exe_path, @@ -96,8 +93,8 @@ class JudgeServer(object): # if spj source code not found, then write it into file if not os.path.exists(spj_src_path): - with open(spj_src_path, "w") as f: - f.write(src.encode("utf-8")) + with open(spj_src_path, "w", encoding="utf-8") as f: + f.write(src) try: Compiler().compile(compile_config=spj_compile_config, src_path=spj_src_path, @@ -114,10 +111,10 @@ class JudgeServer(object): raise TokenVerificationFailed("invalid token") if web.data(): try: - data = json.loads(web.data()) + data = json.loads(web.data().decode("utf-8")) except Exception as e: - logger.info(web.data()) - return {"ret": "ServerError", "data": "invalid json"} + logger.error(web.data()) + return json.dumps({"ret": "ServerError", "data": "invalid json"}) else: data = {} @@ -140,7 +137,7 @@ class JudgeServer(object): logger.exception(e) ret = dict() ret["err"] = "JudgeClientError" - ret["data"] =e.__class__.__name__ + ":" + e.message + ret["data"] =e.__class__.__name__ + ":" + str(e) return json.dumps(ret) diff --git a/server/service.py b/server/service.py index 7bb90d2..76ddd2d 100644 --- a/server/service.py +++ b/server/service.py @@ -1,9 +1,6 @@ -# coding=utf-8 -from __future__ import unicode_literals import os import json import requests -import hashlib from exception import JudgeServiceError from utils import server_info, logger, token diff --git a/server/utils.py b/server/utils.py index bb89f2b..ced98b8 100644 --- a/server/utils.py +++ b/server/utils.py @@ -1,5 +1,3 @@ -# coding=utf-8 -from __future__ import unicode_literals import _judger import psutil import socket @@ -35,5 +33,5 @@ def get_token(): raise JudgeClientError("ENV token not found") -token = hashlib.sha256(get_token()).hexdigest() +token = hashlib.sha256(get_token().encode("utf-8")).hexdigest()