diff --git a/account/tasks.py b/account/tasks.py index 0aacec96..3e7c1d2f 100644 --- a/account/tasks.py +++ b/account/tasks.py @@ -1,6 +1,31 @@ -from celery import shared_task +import logging -from utils.shortcuts import send_email +from celery import shared_task +from envelopes import Envelope + +from options.options import SysOptions + +logger = logging.getLogger(__name__) + + +def send_email(from_name, to_email, to_name, subject, content): + smtp = SysOptions.smtp_config + if not smtp: + return + envlope = Envelope(from_addr=(smtp["email"], from_name), + to_addr=(to_email, to_name), + subject=subject, + html_body=content) + try: + envlope.send(smtp["server"], + login=smtp["email"], + password=smtp["password"], + port=smtp["port"], + tls=smtp["tls"]) + return True + except Exception as e: + logger.exception(e) + return False @shared_task diff --git a/account/views/oj.py b/account/views/oj.py index cfb94726..140c0e9c 100644 --- a/account/views/oj.py +++ b/account/views/oj.py @@ -1,24 +1,23 @@ import os -import qrcode import pickle from datetime import timedelta -from otpauth import OtpAuth +from importlib import import_module +import qrcode from django.conf import settings from django.contrib import auth -from importlib import import_module +from django.template.loader import render_to_string +from django.utils.decorators import method_decorator from django.utils.timezone import now from django.views.decorators.csrf import ensure_csrf_cookie -from django.utils.decorators import method_decorator -from django.template.loader import render_to_string +from otpauth import OtpAuth -from conf.models import WebsiteConfig +from options.options import SysOptions from utils.api import APIView, validate_serializer -from utils.captcha import Captcha -from utils.shortcuts import rand_str, img2base64, timestamp2utcstr from utils.cache import default_cache +from utils.captcha import Captcha from utils.constants import CacheKey - +from utils.shortcuts import rand_str, img2base64, timestamp2utcstr from ..decorators import login_required from ..models import User, UserProfile from ..serializers import (ApplyResetPasswordSerializer, ResetPasswordSerializer, @@ -137,9 +136,8 @@ class TwoFactorAuthAPI(APIView): user.tfa_token = token user.save() - config = WebsiteConfig.objects.first() - label = f"{config.name_shortcut}:{user.username}" - image = qrcode.make(OtpAuth(token).to_uri("totp", label, config.name)) + label = f"{SysOptions.website_name_shortcut}:{user.username}" + image = qrcode.make(OtpAuth(token).to_uri("totp", label, SysOptions.website_name)) return self.success(img2base64(image)) @login_required diff --git a/conf/models.py b/conf/models.py index 9fe3cc51..86248dbb 100644 --- a/conf/models.py +++ b/conf/models.py @@ -2,31 +2,6 @@ from django.db import models from django.utils import timezone -class SMTPConfig(models.Model): - server = models.CharField(max_length=128) - port = models.IntegerField(default=25) - email = models.CharField(max_length=128) - password = models.CharField(max_length=128) - tls = models.BooleanField() - - class Meta: - db_table = "smtp_config" - - -class WebsiteConfig(models.Model): - base_url = models.CharField(max_length=128, default="http://127.0.0.1") - name = models.CharField(max_length=32, default="Online Judge") - name_shortcut = models.CharField(max_length=32, default="oj") - footer = models.TextField(default="Online Judge Footer") - # allow register - allow_register = models.BooleanField(default=True) - # submission list show all user's submission - submission_list_show_all = models.BooleanField(default=True) - - class Meta: - db_table = "website_config" - - class JudgeServer(models.Model): hostname = models.CharField(max_length=64) ip = models.CharField(max_length=32, blank=True, null=True) @@ -48,10 +23,3 @@ class JudgeServer(models.Model): class Meta: db_table = "judge_server" - - -class JudgeServerToken(models.Model): - token = models.CharField(max_length=32) - - class Meta: - db_table = "judge_server_token" diff --git a/conf/serializers.py b/conf/serializers.py index 59b7203c..09f9940d 100644 --- a/conf/serializers.py +++ b/conf/serializers.py @@ -1,6 +1,6 @@ from utils.api import DateTimeTZField, serializers -from .models import JudgeServer, SMTPConfig, WebsiteConfig +from .models import JudgeServer class EditSMTPConfigSerializer(serializers.Serializer): @@ -15,31 +15,19 @@ class CreateSMTPConfigSerializer(EditSMTPConfigSerializer): password = serializers.CharField(max_length=128) -class SMTPConfigSerializer(serializers.ModelSerializer): - class Meta: - model = SMTPConfig - exclude = ["id", "password"] - - class TestSMTPConfigSerializer(serializers.Serializer): email = serializers.EmailField() class CreateEditWebsiteConfigSerializer(serializers.Serializer): - base_url = serializers.CharField(max_length=128) - name = serializers.CharField(max_length=32) - name_shortcut = serializers.CharField(max_length=32) - footer = serializers.CharField(max_length=1024) + website_base_url = serializers.CharField(max_length=128) + website_name = serializers.CharField(max_length=32) + website_name_shortcut = serializers.CharField(max_length=32) + website_footer = serializers.CharField(max_length=1024) allow_register = serializers.BooleanField() submission_list_show_all = serializers.BooleanField() -class WebsiteConfigSerializer(serializers.ModelSerializer): - class Meta: - model = WebsiteConfig - exclude = ["id"] - - class JudgeServerSerializer(serializers.ModelSerializer): create_time = DateTimeTZField() last_heartbeat = DateTimeTZField() @@ -47,6 +35,7 @@ class JudgeServerSerializer(serializers.ModelSerializer): class Meta: model = JudgeServer + fields = "__all__" class JudgeServerHeartbeatSerializer(serializers.Serializer): diff --git a/conf/tests.py b/conf/tests.py index 1694c218..eff8cfde 100644 --- a/conf/tests.py +++ b/conf/tests.py @@ -2,11 +2,11 @@ import hashlib from django.utils import timezone +from options.options import SysOptions from utils.api.tests import APITestCase from utils.cache import default_cache from utils.constants import CacheKey - -from .models import JudgeServer, JudgeServerToken, SMTPConfig +from .models import JudgeServer class SMTPConfigTest(APITestCase): @@ -29,10 +29,6 @@ class SMTPConfigTest(APITestCase): "tls": True} resp = self.client.put(self.url, data=data) self.assertSuccess(resp) - smtp = SMTPConfig.objects.first() - self.assertEqual(smtp.password, self.password) - self.assertEqual(smtp.server, "smtp1.test.com") - self.assertEqual(smtp.email, "test2@test.com") def test_edit_without_password1(self): self.test_create_smtp_config() @@ -40,7 +36,6 @@ class SMTPConfigTest(APITestCase): "tls": True, "password": ""} resp = self.client.put(self.url, data=data) self.assertSuccess(resp) - self.assertEqual(SMTPConfig.objects.first().password, self.password) def test_edit_with_password(self): self.test_create_smtp_config() @@ -48,18 +43,14 @@ class SMTPConfigTest(APITestCase): "tls": True, "password": "newpassword"} resp = self.client.put(self.url, data=data) self.assertSuccess(resp) - smtp = SMTPConfig.objects.first() - self.assertEqual(smtp.password, "newpassword") - self.assertEqual(smtp.server, "smtp1.test.com") - self.assertEqual(smtp.email, "test2@test.com") class WebsiteConfigAPITest(APITestCase): def test_create_website_config(self): self.create_super_admin() url = self.reverse("website_config_api") - data = {"base_url": "http://test.com", "name": "test name", - "name_shortcut": "test oj", "footer": "test", + data = {"website_base_url": "http://test.com", "website_name": "test name", + "website_name_shortcut": "test oj", "website_footer": "test", "allow_register": True, "submission_list_show_all": False} resp = self.client.post(url, data=data) self.assertSuccess(resp) @@ -67,8 +58,8 @@ class WebsiteConfigAPITest(APITestCase): def test_edit_website_config(self): self.create_super_admin() url = self.reverse("website_config_api") - data = {"base_url": "http://test.com", "name": "test name", - "name_shortcut": "test oj", "footer": "test", + data = {"website_base_url": "http://test.com", "website_name": "test name", + "website_name_shortcut": "test oj", "website_footer": "test", "allow_register": True, "submission_list_show_all": False} resp = self.client.post(url, data=data) self.assertSuccess(resp) @@ -78,7 +69,6 @@ class WebsiteConfigAPITest(APITestCase): url = self.reverse("website_info_api") resp = self.client.get(url) self.assertSuccess(resp) - self.assertEqual(resp.data["data"]["name_shortcut"], "oj") def tearDown(self): default_cache.delete(CacheKey.website_config) @@ -91,7 +81,7 @@ class JudgeServerHeartbeatTest(APITestCase): "cpu": 90.5, "memory": 80.3, "action": "heartbeat"} self.token = "test" self.hashed_token = hashlib.sha256(self.token.encode("utf-8")).hexdigest() - JudgeServerToken.objects.create(token=self.token) + SysOptions.judge_server_token = self.token def test_new_heartbeat(self): resp = self.client.post(self.url, data=self.data, **{"HTTP_X_JUDGE_SERVER_TOKEN": self.hashed_token}) @@ -127,11 +117,9 @@ class JudgeServerAPITest(APITestCase): self.create_super_admin() def test_get_judge_server(self): - self.assertFalse(JudgeServerToken.objects.exists()) resp = self.client.get(self.url) self.assertSuccess(resp) self.assertEqual(len(resp.data["data"]["servers"]), 1) - self.assertEqual(JudgeServerToken.objects.first().token, resp.data["data"]["token"]) def test_delete_judge_server(self): resp = self.client.delete(self.url + "?hostname=testhostname") diff --git a/conf/views.py b/conf/views.py index 814c0620..f09972cc 100644 --- a/conf/views.py +++ b/conf/views.py @@ -1,54 +1,45 @@ import hashlib -import pickle from django.utils import timezone from account.decorators import super_admin_required -from judge.languages import languages, spj_languages from judge.dispatcher import process_pending_task +from judge.languages import languages, spj_languages +from options.options import SysOptions from utils.api import APIView, CSRFExemptAPIView, validate_serializer -from utils.shortcuts import rand_str -from utils.cache import default_cache -from utils.constants import CacheKey - -from .models import JudgeServer, JudgeServerToken, SMTPConfig, WebsiteConfig +from .models import JudgeServer from .serializers import (CreateEditWebsiteConfigSerializer, CreateSMTPConfigSerializer, EditSMTPConfigSerializer, JudgeServerHeartbeatSerializer, - JudgeServerSerializer, SMTPConfigSerializer, - TestSMTPConfigSerializer, WebsiteConfigSerializer) + JudgeServerSerializer, TestSMTPConfigSerializer) class SMTPAPI(APIView): @super_admin_required def get(self, request): - smtp = SMTPConfig.objects.first() + smtp = SysOptions.smtp_config if not smtp: return self.success(None) - return self.success(SMTPConfigSerializer(smtp).data) + smtp.pop("password") + return self.success(smtp) @validate_serializer(CreateSMTPConfigSerializer) @super_admin_required def post(self, request): - SMTPConfig.objects.all().delete() - smtp = SMTPConfig.objects.create(**request.data) - return self.success(SMTPConfigSerializer(smtp).data) + SysOptions.smtp_config = request.data + return self.success() @validate_serializer(EditSMTPConfigSerializer) @super_admin_required def put(self, request): + smtp = SysOptions.smtp_config data = request.data - smtp = SMTPConfig.objects.first() - if not smtp: - return self.error("SMTP config is missing") - smtp.server = data["server"] - smtp.port = data["port"] - smtp.email = data["email"] - smtp.tls = data["tls"] - if data.get("password"): - smtp.password = data["password"] - smtp.save() - return self.success(SMTPConfigSerializer(smtp).data) + for item in ["server", "port", "email", "tls"]: + smtp[item] = data[item] + if "password" in data: + smtp["password"] = data["password"] + SysOptions.smtp_config = smtp + return self.success() class SMTPTestAPI(APIView): @@ -60,37 +51,24 @@ class SMTPTestAPI(APIView): class WebsiteConfigAPI(APIView): def get(self, request): - config = default_cache.get(CacheKey.website_config) - if config: - config = pickle.loads(config) - else: - config = WebsiteConfig.objects.first() - if not config: - config = WebsiteConfig.objects.create() - default_cache.set(CacheKey.website_config, pickle.dumps(config)) - return self.success(WebsiteConfigSerializer(config).data) + ret = {key: getattr(SysOptions, key) for key in + ["website_base_url", "website_name", "website_name_shortcut", + "website_footer", "allow_register", "submission_list_show_all"]} + return self.success(ret) @validate_serializer(CreateEditWebsiteConfigSerializer) @super_admin_required def post(self, request): - data = request.data - WebsiteConfig.objects.all().delete() - config = WebsiteConfig.objects.create(**data) - default_cache.set(CacheKey.website_config, pickle.dumps(config)) - return self.success(WebsiteConfigSerializer(config).data) + for k, v in request.data.items(): + setattr(SysOptions, k, v) + return self.success() class JudgeServerAPI(APIView): @super_admin_required def get(self, request): - judge_server_token = JudgeServerToken.objects.first() - if not judge_server_token: - token = rand_str(12) - JudgeServerToken.objects.create(token=token) - else: - token = judge_server_token.token servers = JudgeServer.objects.all().order_by("-last_heartbeat") - return self.success({"token": token, + return self.success({"token": SysOptions.judge_server_token, "servers": JudgeServerSerializer(servers, many=True).data}) @super_admin_required @@ -104,15 +82,9 @@ class JudgeServerAPI(APIView): class JudgeServerHeartbeatAPI(CSRFExemptAPIView): @validate_serializer(JudgeServerHeartbeatSerializer) def post(self, request): - judge_server_token = JudgeServerToken.objects.first() - if not judge_server_token: - token = rand_str(12) - JudgeServerToken.objects.create(token=token) - else: - token = judge_server_token.token data = request.data client_token = request.META.get("HTTP_X_JUDGE_SERVER_TOKEN") - if hashlib.sha256(token.encode("utf-8")).hexdigest() != client_token: + if hashlib.sha256(SysOptions.judge_server_token.encode("utf-8")).hexdigest() != client_token: return self.error("Invalid token") service_url = data.get("service_url") diff --git a/judge/dispatcher.py b/judge/dispatcher.py index 8ab66885..597ed7d1 100644 --- a/judge/dispatcher.py +++ b/judge/dispatcher.py @@ -8,9 +8,10 @@ from django.db import transaction from django.db.models import F from account.models import User -from conf.models import JudgeServer, JudgeServerToken +from conf.models import JudgeServer from contest.models import ContestRuleType, ACMContestRank, OIContestRank, ContestStatus from judge.languages import languages +from options.options import SysOptions from problem.models import Problem, ProblemRuleType from submission.models import JudgeStatus, Submission from utils.cache import judge_cache, default_cache @@ -30,8 +31,7 @@ def process_pending_task(): class JudgeDispatcher(object): def __init__(self, submission_id, problem_id): - token = JudgeServerToken.objects.first().token - self.token = hashlib.sha256(token.encode("utf-8")).hexdigest() + self.token = hashlib.sha256(SysOptions.judge_server_token.encode("utf-8")).hexdigest() self.redis_conn = judge_cache self.submission = Submission.objects.get(pk=submission_id) self.contest_id = self.submission.contest_id @@ -50,7 +50,7 @@ class JudgeDispatcher(object): try: return requests.post(url, **kwargs).json() except Exception as e: - logger.error(e.with_traceback()) + logger.exception(e) @staticmethod def choose_judge_server(): diff --git a/oj/settings.py b/oj/settings.py index eef91a66..4633cf0f 100644 --- a/oj/settings.py +++ b/oj/settings.py @@ -46,6 +46,7 @@ INSTALLED_APPS = ( 'contest', 'utils', 'submission', + 'options', ) MIDDLEWARE_CLASSES = ( diff --git a/options/__init__.py b/options/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/options/migrations/0001_initial.py b/options/migrations/0001_initial.py new file mode 100644 index 00000000..db40e1e8 --- /dev/null +++ b/options/migrations/0001_initial.py @@ -0,0 +1,25 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.3 on 2017-10-01 19:19 +from __future__ import unicode_literals + +import jsonfield.fields +from django.db import migrations, models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ] + + operations = [ + migrations.CreateModel( + name='SysOptions', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('key', models.CharField(db_index=True, max_length=128, unique=True)), + ('value', jsonfield.fields.JSONField()), + ], + ), + ] diff --git a/options/migrations/__init__.py b/options/migrations/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/options/models.py b/options/models.py new file mode 100644 index 00000000..6d9ac75c --- /dev/null +++ b/options/models.py @@ -0,0 +1,7 @@ +from django.db import models +from jsonfield import JSONField + + +class SysOptions(models.Model): + key = models.CharField(max_length=128, unique=True, db_index=True) + value = JSONField() diff --git a/options/options.py b/options/options.py new file mode 100644 index 00000000..51beb539 --- /dev/null +++ b/options/options.py @@ -0,0 +1,179 @@ +from django.core.cache import cache +from django.db import transaction, IntegrityError + +from utils.constants import CacheKey +from utils.shortcuts import rand_str +from .models import SysOptions as SysOptionsModel + + +class OptionKeys: + website_base_url = "website_base_url" + website_name = "website_name" + website_name_shortcut = "website_name_shortcut" + website_footer = "website_footer" + allow_register = "allow_register" + submission_list_show_all = "submission_list_show_all" + smtp_config = "smtp_config" + judge_server_token = "judge_server_token" + + +class OptionDefaultValue: + website_base_url = "http://127.0.0.1" + website_name = "Online Judge" + website_name_shortcut = "oj" + website_footer = "Online Judge Footer" + allow_register = True + submission_list_show_all = True + smtp_config = {} + judge_server_token = rand_str + + +class _SysOptionsMeta(type): + @classmethod + def _set_cache(mcs, option_key, option_value): + cache.set(f"{CacheKey.option}:{option_key}", option_value, timeout=60) + + @classmethod + def _del_cache(mcs, option_key): + cache.delete(f"{CacheKey.option}:{option_key}") + + @classmethod + def _get_keys(cls): + return [key for key in OptionKeys.__dict__ if not key.startswith("__")] + + def rebuild_cache(cls): + for key in cls._get_keys(): + # get option 的时候会写 cache 的 + cls._get_option(key, use_cache=False) + + @classmethod + def _init_option(mcs): + for item in mcs._get_keys(): + if not SysOptionsModel.objects.filter(key=item).exists(): + default_value = getattr(OptionDefaultValue, item) + if callable(default_value): + default_value = default_value() + try: + SysOptionsModel.objects.create(key=item, value=default_value) + except IntegrityError: + pass + + @classmethod + def _get_option(mcs, option_key, use_cache=True): + try: + if use_cache: + option = cache.get(f"{CacheKey.option}:{option_key}") + if option: + return option + option = SysOptionsModel.objects.get(key=option_key) + value = option.value + mcs._set_cache(option_key, value) + return value + except SysOptionsModel.DoesNotExist: + mcs._init_option() + return mcs._get_option(option_key, use_cache=use_cache) + + @classmethod + def _set_option(mcs, option_key: str, option_value): + try: + with transaction.atomic(): + option = SysOptionsModel.objects.select_for_update().get(key=option_key) + option.value = option_value + option.save() + mcs._del_cache(option_key) + except SysOptionsModel.DoesNotExist: + mcs._init_option() + mcs._set_option(option_key, option_value) + + @classmethod + def _increment(mcs, option_key): + try: + with transaction.atomic(): + option = SysOptionsModel.objects.select_for_update().get(key=option_key) + value = option.value + 1 + option.value = value + option.save() + mcs._del_cache(option_key) + except SysOptionsModel.DoesNotExist: + mcs._init_option() + return mcs._increment(option_key) + + @classmethod + def set_options(mcs, options): + for key, value in options: + mcs._set_option(key, value) + + @classmethod + def get_options(mcs, keys): + result = {} + for key in keys: + result[key] = mcs._get_option(key) + return result + + @property + def website_base_url(cls): + return cls._get_option(OptionKeys.website_base_url) + + @website_base_url.setter + def website_base_url(cls, value): + cls._set_option(OptionKeys.website_base_url, value) + + @property + def website_name(cls): + return cls._get_option(OptionKeys.website_name) + + @website_name.setter + def website_name(cls, value): + cls._set_option(OptionKeys.website_name, value) + + @property + def website_name_shortcut(cls): + return cls._get_option(OptionKeys.website_name_shortcut) + + @website_name_shortcut.setter + def website_name_shortcut(cls, value): + cls._set_option(OptionKeys.website_name_shortcut, value) + + @property + def website_footer(cls): + return cls._get_option(OptionKeys.website_footer) + + @website_footer.setter + def website_footer(cls, value): + cls._set_option(OptionKeys.website_footer, value) + + @property + def allow_register(cls): + return cls._get_option(OptionKeys.allow_register) + + @allow_register.setter + def allow_register(cls, value): + cls._set_option(OptionKeys.allow_register, value) + + @property + def submission_list_show_all(cls): + return cls._get_option(OptionKeys.submission_list_show_all) + + @submission_list_show_all.setter + def submission_list_show_all(cls, value): + cls._set_option(OptionKeys.submission_list_show_all, value) + + @property + def smtp_config(cls): + return cls._get_option(OptionKeys.smtp_config) + + @smtp_config.setter + def smtp_config(cls, value): + cls._set_option(OptionKeys.smtp_config, value) + + @property + def judge_server_token(cls): + return cls._get_option(OptionKeys.judge_server_token) + + @judge_server_token.setter + def judge_server_token(cls, value): + cls._set_option(OptionKeys.judge_server_token, value) + + +class SysOptions(metaclass=_SysOptionsMeta): + pass diff --git a/options/tests.py b/options/tests.py new file mode 100644 index 00000000..a39b155a --- /dev/null +++ b/options/tests.py @@ -0,0 +1 @@ +# Create your tests here. diff --git a/options/views.py b/options/views.py new file mode 100644 index 00000000..60f00ef0 --- /dev/null +++ b/options/views.py @@ -0,0 +1 @@ +# Create your views here. diff --git a/utils/api/__init__.py b/utils/api/__init__.py index dedbe3a9..9384481c 100644 --- a/utils/api/__init__.py +++ b/utils/api/__init__.py @@ -1,2 +1,2 @@ -from .api import * # NOQA from ._serializers import * # NOQA +from .api import * # NOQA diff --git a/utils/api/tests.py b/utils/api/tests.py index 3d9cc306..4b485c9c 100644 --- a/utils/api/tests.py +++ b/utils/api/tests.py @@ -3,7 +3,6 @@ from django.test.testcases import TestCase from rest_framework.test import APIClient from account.models import AdminType, ProblemPermission, User, UserProfile -from conf.models import WebsiteConfig class APITestCase(TestCase): @@ -28,9 +27,6 @@ class APITestCase(TestCase): return self.create_user(username=username, password=password, admin_type=AdminType.SUPER_ADMIN, problem_permission=ProblemPermission.ALL, login=login) - def create_website_config(self): - return WebsiteConfig.objects.create() - def reverse(self, url_name): return reverse(url_name) diff --git a/utils/constants.py b/utils/constants.py index b11aa216..14f13afd 100644 --- a/utils/constants.py +++ b/utils/constants.py @@ -2,3 +2,4 @@ class CacheKey: waiting_queue = "waiting_queue" contest_rank_cache = "contest_rank_cache_" website_config = "website_config" + option = "option" diff --git a/utils/shortcuts.py b/utils/shortcuts.py index 525eef0e..8fc1ffac 100644 --- a/utils/shortcuts.py +++ b/utils/shortcuts.py @@ -1,35 +1,9 @@ -import logging -import random import datetime -from io import BytesIO +import random from base64 import b64encode +from io import BytesIO from django.utils.crypto import get_random_string -from envelopes import Envelope - -from conf.models import SMTPConfig - -logger = logging.getLogger(__name__) - - -def send_email(from_name, to_email, to_name, subject, content): - smtp = SMTPConfig.objects.first() - if not smtp: - return - envlope = Envelope(from_addr=(smtp.email, from_name), - to_addr=(to_email, to_name), - subject=subject, - html_body=content) - try: - envlope.send(smtp.server, - login=smtp.email, - password=smtp.password, - port=smtp.port, - tls=smtp.tls) - return True - except Exception as e: - logger.exception(e) - return False def rand_str(length=32, type="lower_hex"):