From d8966ed48af556a4234f54f4c6e637c84ed2830f Mon Sep 17 00:00:00 2001 From: virusdefender <1670873886@qq.com> Date: Sat, 28 Nov 2015 20:32:49 +0800 Subject: [PATCH] dev test --- judge/judger/__init__.py => db1.sqlite3 | 0 judge/__init__.py | 1 - judge/{judger => }/client.py | 0 judge/{judger => }/compiler.py | 0 judge/{judger => }/judge_exceptions.py | 0 judge/judger/run.py | 94 ------------------- judge/judger_controller/README.md | 1 - judge/judger_controller/celery.py | 9 -- judge/judger_controller/settings.py | 39 -------- judge/judger_controller/tasks.py | 45 --------- judge/{judger => }/language.py | 0 judge/{judger => }/logger.py | 0 judge/{judger => }/result.py | 0 judge/run.py | 52 ++++++++++ .../__init__.py => server.py} | 0 judge/{judger => }/settings.py | 0 judge/{judger => }/utils.py | 0 judge_dispatcher/__init__.py | 1 + judge_dispatcher/judge.py | 16 ++++ judge_dispatcher/models.py | 15 +++ judge_dispatcher/rpc_client.py | 24 +++++ judge_dispatcher/settings.py | 9 ++ 22 files changed, 117 insertions(+), 189 deletions(-) rename judge/judger/__init__.py => db1.sqlite3 (100%) rename judge/{judger => }/client.py (100%) rename judge/{judger => }/compiler.py (100%) rename judge/{judger => }/judge_exceptions.py (100%) delete mode 100644 judge/judger/run.py delete mode 100644 judge/judger_controller/README.md delete mode 100644 judge/judger_controller/celery.py delete mode 100644 judge/judger_controller/settings.py delete mode 100644 judge/judger_controller/tasks.py rename judge/{judger => }/language.py (100%) rename judge/{judger => }/logger.py (100%) rename judge/{judger => }/result.py (100%) create mode 100644 judge/run.py rename judge/{judger_controller/__init__.py => server.py} (100%) rename judge/{judger => }/settings.py (100%) rename judge/{judger => }/utils.py (100%) create mode 100644 judge_dispatcher/__init__.py create mode 100644 judge_dispatcher/judge.py create mode 100644 judge_dispatcher/models.py create mode 100644 judge_dispatcher/rpc_client.py create mode 100644 judge_dispatcher/settings.py diff --git a/judge/judger/__init__.py b/db1.sqlite3 similarity index 100% rename from judge/judger/__init__.py rename to db1.sqlite3 diff --git a/judge/__init__.py b/judge/__init__.py index 9bad5790..e69de29b 100644 --- a/judge/__init__.py +++ b/judge/__init__.py @@ -1 +0,0 @@ -# coding=utf-8 diff --git a/judge/judger/client.py b/judge/client.py similarity index 100% rename from judge/judger/client.py rename to judge/client.py diff --git a/judge/judger/compiler.py b/judge/compiler.py similarity index 100% rename from judge/judger/compiler.py rename to judge/compiler.py diff --git a/judge/judger/judge_exceptions.py b/judge/judge_exceptions.py similarity index 100% rename from judge/judger/judge_exceptions.py rename to judge/judge_exceptions.py diff --git a/judge/judger/run.py b/judge/judger/run.py deleted file mode 100644 index 8aeb5476..00000000 --- a/judge/judger/run.py +++ /dev/null @@ -1,94 +0,0 @@ -# coding=utf-8 -import sys -import json -import MySQLdb - -from client import JudgeClient -from language import languages -from compiler import compile_ -from result import result -from settings import judger_workspace, submission_db -from logger import logger - - -# 简单的解析命令行参数 -# 参数有 -solution_id -time_limit -memory_limit -test_case_id -# 获取到的值是['xxx.py', '-solution_id', '1111', '-time_limit', '1000', '-memory_limit', '100', '-test_case_id', 'aaaa'] -args = sys.argv -submission_id = args[2] -time_limit = args[4] -memory_limit = args[6] -test_case_id = args[8] - - -def db_conn(): - return MySQLdb.connect(db=submission_db["db"], - user=submission_db["user"], - passwd=submission_db["password"], - host=submission_db["host"], - port=submission_db["port"], charset="utf8") - - -conn = db_conn() -cur = conn.cursor() -cur.execute("select language, code from submission where id = %s", (submission_id,)) -data = cur.fetchall() -if not data: - exit() -language_code = data[0][0] -code = data[0][1] - -conn.close() - -# 将代码写入文件 -language = languages[language_code] -src_path = judger_workspace + "run/" + language["src_name"] -f = open(src_path, "w") -f.write(code.encode("utf8")) -f.close() - -# 编译 -try: - exe_path = compile_(language, src_path, judger_workspace + "run/") -except Exception as e: - print e - conn = db_conn() - cur = conn.cursor() - cur.execute("update submission set result=%s, info=%s where id=%s", - (result["compile_error"], str(e), submission_id)) - conn.commit() - exit() - -# 运行 -try: - client = JudgeClient(language_code=language_code, - exe_path=exe_path, - max_cpu_time=int(time_limit), - max_real_time=int(time_limit) * 2, - max_memory=int(memory_limit), - test_case_dir=judger_workspace + "test_case/" + test_case_id + "/") - judge_result = {"result": result["accepted"], "info": client.run(), "accepted_answer_time": None} - - for item in judge_result["info"]: - if item["result"]: - judge_result["result"] = item["result"] - break - else: - l = sorted(judge_result["info"], key=lambda k: k["cpu_time"]) - judge_result["accepted_answer_time"] = l[-1]["cpu_time"] - -except Exception as e: - logger.error(e) - conn = db_conn() - cur = conn.cursor() - cur.execute("update submission set result=%s, info=%s where id=%s", (result["system_error"], str(e), submission_id)) - conn.commit() - exit() - -conn = db_conn() -cur = conn.cursor() -cur.execute("update submission set result=%s, info=%s, accepted_answer_time=%s where id=%s", - (judge_result["result"], json.dumps(judge_result["info"]), judge_result["accepted_answer_time"], - submission_id)) -conn.commit() -conn.close() diff --git a/judge/judger_controller/README.md b/judge/judger_controller/README.md deleted file mode 100644 index 9dd56032..00000000 --- a/judge/judger_controller/README.md +++ /dev/null @@ -1 +0,0 @@ -celery -A judge.controller worker -l DEBUG \ No newline at end of file diff --git a/judge/judger_controller/celery.py b/judge/judger_controller/celery.py deleted file mode 100644 index 4c64ab04..00000000 --- a/judge/judger_controller/celery.py +++ /dev/null @@ -1,9 +0,0 @@ -# coding=utf-8 -from __future__ import absolute_import -from celery import Celery, platforms -from .settings import redis_config - -app = Celery("judge", broker='redis://%s:%s/%s' % (redis_config["host"], redis_config["port"], redis_config["db"]), - include=["judge.judger_controller.tasks"]) - -platforms.C_FORCE_ROOT =True diff --git a/judge/judger_controller/settings.py b/judge/judger_controller/settings.py deleted file mode 100644 index 4d48340e..00000000 --- a/judge/judger_controller/settings.py +++ /dev/null @@ -1,39 +0,0 @@ -# coding=utf-8 -""" -注意: -此文件包含 celery 的部分配置,但是 celery 并不是运行在docker 中的,所以本配置文件中的 redis和 MySQL 的地址就应该是 -运行 redis 和 MySQL 的 docker 容器的地址了。怎么获取这个地址见帮助文档。测试用例的路径和源代码路径同理。 -""" -import os -# 这个redis 是 celery 使用的,包括存储队列信息还有部分统计信息 -redis_config = { - "host": os.environ.get("REDIS_PORT_6379_TCP_ADDR"), - "port": 6379, - "db": 0 -} - - -# 判题的 docker 容器的配置参数 -docker_config = { - "image_name": "judger", - "docker_path": "docker", - "shell": True -} - - -# 测试用例的路径,是主机上的实际路径 -test_case_dir = "/root/test_case/" -# 源代码路径,也就是 manage.py 所在的实际路径 -source_code_dir = "/root/qduoj/" -# 日志文件夹路径 -log_dir = "/root/log/" - - -# 存储提交信息的数据库,是 celery 使用的,与 oj.settings/local_settings 等区分,那是 web 服务器访问的地址 -submission_db = { - "host": os.environ.get("submission_db_host"), - "port": 3306, - "db": "oj_submission", - "user": "root", - "password": "root" -} diff --git a/judge/judger_controller/tasks.py b/judge/judger_controller/tasks.py deleted file mode 100644 index 2219d96e..00000000 --- a/judge/judger_controller/tasks.py +++ /dev/null @@ -1,45 +0,0 @@ -# coding=utf-8 -import json -import redis -import MySQLdb -import subprocess -from ..judger.result import result -from ..judger_controller.celery import app -from settings import docker_config, source_code_dir, test_case_dir, log_dir, submission_db, redis_config - - -@app.task -def judge(submission_id, time_limit, memory_limit, test_case_id): - try: - command = "%s run --privileged --rm " \ - "--link mysql " \ - "-v %s:/var/judger/test_case/:ro " \ - "-v %s:/var/judger/code/:ro " \ - "-v %s:/var/judger/code/log/ " \ - "--device /dev/null:/dev/null " \ - "%s " \ - "python judge/judger/run.py " \ - "--solution_id %s --time_limit %s --memory_limit %s --test_case_id %s" % \ - (docker_config["docker_path"], - test_case_dir, - source_code_dir, - log_dir, - docker_config["image_name"], - submission_id, str(time_limit), str(memory_limit), test_case_id) - subprocess.call(command, shell=docker_config["shell"]) - except Exception as e: - conn = MySQLdb.connect(db=submission_db["db"], - user=submission_db["user"], - passwd=submission_db["password"], - host=submission_db["host"], - port=submission_db["port"], - charset="utf8") - - cur = conn.cursor() - cur.execute("update submission set result=%s, info=%s where id=%s", - (result["system_error"], str(e), submission_id)) - conn.commit() - conn.close() - r = redis.Redis(host=redis_config["host"], port=redis_config["port"], db=redis_config["db"]) - r.decr("judge_queue_length") - r.lpush("queue", submission_id) diff --git a/judge/judger/language.py b/judge/language.py similarity index 100% rename from judge/judger/language.py rename to judge/language.py diff --git a/judge/judger/logger.py b/judge/logger.py similarity index 100% rename from judge/judger/logger.py rename to judge/logger.py diff --git a/judge/judger/result.py b/judge/result.py similarity index 100% rename from judge/judger/result.py rename to judge/result.py diff --git a/judge/run.py b/judge/run.py new file mode 100644 index 00000000..2118521a --- /dev/null +++ b/judge/run.py @@ -0,0 +1,52 @@ +# coding=utf-8 +import sys +import json +import MySQLdb + +from client import JudgeClient +from language import languages +from compiler import compile_ +from result import result +from settings import judger_workspace, submission_db +from logger import logger + + +class JudgeInstanceRunner(object): + def __init__(self): + pass + + def run(self, submission_id, language_code, code, time_limit, memory_limit, test_case_id): + language = languages[language_code] + # 将代码写入文件 + src_path = judger_workspace + "run/" + submission_id + "/" + language["src_name"] + f = open(src_path, "w") + f.write(code.encode("utf8")) + f.close() + + # 编译 + try: + exe_path = compile_(language, src_path, judger_workspace + "run/" + submission_id + "/") + except Exception as e: + return {"code": 1, "data": str(e)} + + # 运行 + try: + client = JudgeClient(language_code=language_code, + exe_path=exe_path, + max_cpu_time=int(time_limit), + max_real_time=int(time_limit) * 2, + max_memory=int(memory_limit), + test_case_dir=judger_workspace + "test_case/" + test_case_id + "/") + judge_result = {"result": result["accepted"], "info": client.run(), "accepted_answer_time": None} + + for item in judge_result["info"]: + if item["result"]: + judge_result["result"] = item["result"] + break + else: + l = sorted(judge_result["info"], key=lambda k: k["cpu_time"]) + judge_result["accepted_answer_time"] = l[-1]["cpu_time"] + return {"code": 0, "data": judge_result} + + except Exception as e: + return {"code": 1, "data": str(e)} \ No newline at end of file diff --git a/judge/judger_controller/__init__.py b/judge/server.py similarity index 100% rename from judge/judger_controller/__init__.py rename to judge/server.py diff --git a/judge/judger/settings.py b/judge/settings.py similarity index 100% rename from judge/judger/settings.py rename to judge/settings.py diff --git a/judge/judger/utils.py b/judge/utils.py similarity index 100% rename from judge/judger/utils.py rename to judge/utils.py diff --git a/judge_dispatcher/__init__.py b/judge_dispatcher/__init__.py new file mode 100644 index 00000000..9bad5790 --- /dev/null +++ b/judge_dispatcher/__init__.py @@ -0,0 +1 @@ +# coding=utf-8 diff --git a/judge_dispatcher/judge.py b/judge_dispatcher/judge.py new file mode 100644 index 00000000..8e4e1dd8 --- /dev/null +++ b/judge_dispatcher/judge.py @@ -0,0 +1,16 @@ +# coding=utf-8 +import socket +import redis + +from .rpc_client import TimeoutServerProxy +from .settings import redis_config +from .models import JudgeServer + + +class JudgeDispatcher(object): + def __init__(self): + self.redis = redis.StrictRedis(host=redis_config["host"], port=redis_config["port"], db=redis_config["db"]) + + def judge(self): + pass + diff --git a/judge_dispatcher/models.py b/judge_dispatcher/models.py new file mode 100644 index 00000000..29f52267 --- /dev/null +++ b/judge_dispatcher/models.py @@ -0,0 +1,15 @@ +# coding=utf-8 +from django.db import models + + +class JudgeServer(models.Model): + ip = models.IPAddressField() + port = models.IntegerField() + # 这个服务器最大可能运行的判题实例数量 + max_instance_number = models.IntegerField() + left_instance_number = models.IntegerField() + # status 为 false 的时候代表不使用这个服务器 + status = models.BooleanField(default=True) + + class Meta: + db_table = "judge_server" diff --git a/judge_dispatcher/rpc_client.py b/judge_dispatcher/rpc_client.py new file mode 100644 index 00000000..c095cb5e --- /dev/null +++ b/judge_dispatcher/rpc_client.py @@ -0,0 +1,24 @@ +# coding=utf-8 +import xmlrpclib +import httplib + + +class TimeoutHTTPConnection(httplib.HTTPConnection): + def __init__(self, host, timeout=10): + httplib.HTTPConnection.__init__(self, host, timeout=timeout) + + +class TimeoutTransport(xmlrpclib.Transport): + def __init__(self, timeout=10, *args, **kwargs): + xmlrpclib.Transport.__init__(self, *args, **kwargs) + self.timeout = timeout + + def make_connection(self, host): + conn = TimeoutHTTPConnection(host, self.timeout) + return conn + + +class TimeoutServerProxy(xmlrpclib.ServerProxy): + def __init__(self, uri, timeout=10, *args, **kwargs): + kwargs['transport'] = TimeoutTransport(timeout=timeout, use_datetime=kwargs.get('use_datetime', 0)) + xmlrpclib.ServerProxy.__init__(self, uri, *args, **kwargs) diff --git a/judge_dispatcher/settings.py b/judge_dispatcher/settings.py new file mode 100644 index 00000000..332cefe8 --- /dev/null +++ b/judge_dispatcher/settings.py @@ -0,0 +1,9 @@ +# coding=utf-8 +import os + + +redis_config = { + "host": os.environ.get("REDIS_PORT_6379_TCP_ADDR"), + "port": 6379, + "db": 0 +} \ No newline at end of file