diff --git a/account/middleware.py b/account/middleware.py index 9141d534..b674346d 100644 --- a/account/middleware.py +++ b/account/middleware.py @@ -1,10 +1,5 @@ -import time -import pytz - -from django.contrib import auth -from django.utils import timezone -from django.utils.translation import ugettext as _ from django.db import connection +from django.utils.timezone import now from django.utils.deprecation import MiddlewareMixin from utils.api import JSONResponse @@ -14,14 +9,11 @@ class SessionRecordMiddleware(MiddlewareMixin): def process_request(self, request): if request.user.is_authenticated(): session = request.session - ip = request.META.get("HTTP_X_REAL_IP", "UNKNOWN IP") - user_agent = request.META.get("HTTP_USER_AGENT", "") - _ip = session.setdefault("ip", ip) - _user_agent = session.setdefault("user_agent", user_agent) - if ip != _ip or user_agent != _user_agent: - session.modified = True + session["user_agent"] = request.META.get("HTTP_USER_AGENT", "") + session["ip"] = request.META.get("HTTP_X_REAL_IP", "UNKNOWN IP") + session["last_activity"] = now() user_sessions = request.user.session_keys - if request.session.session_key not in user_sessions: + if session.session_key not in user_sessions: user_sessions.append(session.session_key) request.user.save() diff --git a/account/migrations/0001_initial.py b/account/migrations/0001_initial.py index a96776e0..e1e588ee 100644 --- a/account/migrations/0001_initial.py +++ b/account/migrations/0001_initial.py @@ -50,7 +50,7 @@ class Migration(migrations.Migration): fields=[ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), ('problems_status', jsonfield.fields.JSONField(default={})), - ('avatar', models.CharField(default=account.models._default_avatar, max_length=50)), + ('avatar', models.CharField(default="default.png", max_length=50)), ('blog', models.URLField(blank=True, null=True)), ('mood', models.CharField(blank=True, max_length=200, null=True)), ('accepted_problem_number', models.IntegerField(default=0)), diff --git a/account/models.py b/account/models.py index 2db0e55b..3ba07d9a 100644 --- a/account/models.py +++ b/account/models.py @@ -72,7 +72,7 @@ class UserProfile(models.Model): oi_problems_status = JSONField(default={}) real_name = models.CharField(max_length=32, blank=True, null=True) - avatar = models.CharField(max_length=256, default=f"{settings.IMAGE_UPLOAD_DIR}/default.png") + avatar = models.CharField(max_length=256, default=f"/{settings.IMAGE_UPLOAD_DIR}/default.png") blog = models.URLField(blank=True, null=True) mood = models.CharField(max_length=256, blank=True, null=True) github = models.CharField(max_length=64, blank=True, null=True) diff --git a/account/serializers.py b/account/serializers.py index 1b29170c..aa9675a3 100644 --- a/account/serializers.py +++ b/account/serializers.py @@ -26,7 +26,6 @@ class UserRegisterSerializer(serializers.Serializer): class UserChangePasswordSerializer(serializers.Serializer): old_password = serializers.CharField() new_password = serializers.CharField(min_length=6) - captcha = serializers.CharField() class UserSerializer(serializers.ModelSerializer): @@ -46,6 +45,7 @@ class UserProfileSerializer(serializers.ModelSerializer): class Meta: model = UserProfile + fields = "__all__" class UserInfoSerializer(serializers.ModelSerializer): diff --git a/account/tests.py b/account/tests.py index 7331a0ee..906ef28b 100644 --- a/account/tests.py +++ b/account/tests.py @@ -8,11 +8,9 @@ from otpauth import OtpAuth from utils.api.tests import APIClient, APITestCase from utils.shortcuts import rand_str -from utils.cache import default_cache -from utils.constants import CacheKey +from options.options import SysOptions from .models import AdminType, ProblemPermission, User -from conf.models import WebsiteConfig class PermissionDecoratorTest(APITestCase): @@ -157,13 +155,9 @@ class UserRegisterAPITest(CaptchaTest): self.data = {"username": "test_user", "password": "testuserpassword", "real_name": "real_name", "email": "test@qduoj.com", "captcha": self._set_captcha(self.client.session)} - # clea cache in redis - default_cache.delete(CacheKey.website_config) def test_website_config_limit(self): - website = WebsiteConfig.objects.create() - website.allow_register = False - website.save() + SysOptions.allow_register = False resp = self.client.post(self.register_url, data=self.data) self.assertDictEqual(resp.data, {"error": "error", "data": "Register have been disabled by admin"}) @@ -247,7 +241,6 @@ class TwoFactorAuthAPITest(APITestCase): def setUp(self): self.url = self.reverse("two_factor_auth_api") self.create_user("test", "test123") - self.create_website_config() def _get_tfa_code(self): user = User.objects.first() @@ -295,7 +288,6 @@ class ApplyResetPasswordAPITest(CaptchaTest): user.email = "test@oj.com" user.save() self.url = self.reverse("apply_reset_password_api") - self.create_website_config() self.data = {"email": "test@oj.com", "captcha": self._set_captcha(self.client.session)} def _refresh_captcha(self): diff --git a/account/urls/oj.py b/account/urls/oj.py index cc716830..aacbb59d 100644 --- a/account/urls/oj.py +++ b/account/urls/oj.py @@ -3,7 +3,7 @@ from django.conf.urls import url from ..views.oj import (ApplyResetPasswordAPI, ResetPasswordAPI, UserChangePasswordAPI, UserRegisterAPI, UserLoginAPI, UserLogoutAPI, UsernameOrEmailCheck, - SSOAPI, AvatarUploadAPI, TwoFactorAuthAPI, UserProfileAPI, + AvatarUploadAPI, TwoFactorAuthAPI, UserProfileAPI, UserRankAPI, CheckTFARequiredAPI, SessionManagementAPI) from utils.captcha.views import CaptchaAPIView diff --git a/account/views/oj.py b/account/views/oj.py index 1225556e..5f72156e 100644 --- a/account/views/oj.py +++ b/account/views/oj.py @@ -1,5 +1,4 @@ import os -import pickle from datetime import timedelta from importlib import import_module @@ -16,15 +15,14 @@ from utils.constants import ContestRuleType from options.options import SysOptions from utils.api import APIView, validate_serializer from utils.captcha import Captcha -from utils.shortcuts import rand_str, img2base64, timestamp2utcstr +from utils.shortcuts import rand_str, img2base64, datetime2str from ..decorators import login_required from ..models import User, UserProfile from ..serializers import (ApplyResetPasswordSerializer, ResetPasswordSerializer, UserChangePasswordSerializer, UserLoginSerializer, UserRegisterSerializer, UsernameOrEmailCheckSerializer, RankInfoSerializer) -from ..serializers import (SSOSerializer, TwoFactorAuthCodeSerializer, - UserProfileSerializer, +from ..serializers import (TwoFactorAuthCodeSerializer, UserProfileSerializer, EditUserProfileSerializer, AvatarUploadForm) from ..tasks import send_email_async @@ -81,7 +79,7 @@ class AvatarUploadAPI(APIView): img.write(chunk) user_profile = request.user.userprofile - user_profile.avatar = f"{settings.IMAGE_UPLOAD_DIR}/{name}" + user_profile.avatar = f"/{settings.IMAGE_UPLOAD_DIR}/{name}" user_profile.save() return self.success("Succeeded") @@ -327,7 +325,7 @@ class SessionManagementAPI(APIView): s["current_session"] = True s["ip"] = session["ip"] s["user_agent"] = session["user_agent"] - s["last_activity"] = timestamp2utcstr(session["last_activity"]) + s["last_activity"] = datetime2str(session["last_activity"]) s["session_key"] = key result.append(s) if modified: diff --git a/contest/models.py b/contest/models.py index d66d4007..10581ccb 100644 --- a/contest/models.py +++ b/contest/models.py @@ -1,8 +1,9 @@ +from utils.constants import ContestRuleType # noqa from django.db import models from django.utils.timezone import now from jsonfield import JSONField -from utils.constants import ContestStatus, ContestRuleType, ContestType +from utils.constants import ContestStatus, ContestType from account.models import User, AdminType from utils.models import RichTextField diff --git a/contest/views/oj.py b/contest/views/oj.py index c4fd4fc0..6811ef61 100644 --- a/contest/views/oj.py +++ b/contest/views/oj.py @@ -4,7 +4,7 @@ from utils.api import APIView, validate_serializer from utils.constants import CacheKey from account.decorators import login_required, check_contest_permission -from utils.constants import ContestRuleType, ContestType, ContestStatus +from utils.constants import ContestRuleType, ContestStatus from ..models import ContestAnnouncement, Contest, OIContestRank, ACMContestRank from ..serializers import ContestAnnouncementSerializer from ..serializers import ContestSerializer, ContestPasswordVerifySerializer diff --git a/judge/dispatcher.py b/judge/dispatcher.py index 597ed7d1..29b4475f 100644 --- a/judge/dispatcher.py +++ b/judge/dispatcher.py @@ -14,7 +14,7 @@ from judge.languages import languages from options.options import SysOptions from problem.models import Problem, ProblemRuleType from submission.models import JudgeStatus, Submission -from utils.cache import judge_cache, default_cache +from utils.cache import cache from utils.constants import CacheKey logger = logging.getLogger(__name__) @@ -22,31 +22,28 @@ logger = logging.getLogger(__name__) # 继续处理在队列中的问题 def process_pending_task(): - if judge_cache.llen(CacheKey.waiting_queue): + if cache.llen(CacheKey.waiting_queue): # 防止循环引入 from judge.tasks import judge_task - data = json.loads(judge_cache.rpop(CacheKey.waiting_queue).decode("utf-8")) + data = json.loads(cache.rpop(CacheKey.waiting_queue).decode("utf-8")) judge_task.delay(**data) class JudgeDispatcher(object): def __init__(self, submission_id, problem_id): self.token = hashlib.sha256(SysOptions.judge_server_token.encode("utf-8")).hexdigest() - self.redis_conn = judge_cache - self.submission = Submission.objects.get(pk=submission_id) + self.submission = Submission.objects.get(id=submission_id) self.contest_id = self.submission.contest_id if self.contest_id: - self.problem = Problem.objects.select_related("contest") \ - .get(id=problem_id, contest_id=self.contest_id) + self.problem = Problem.objects.select_related("contest").get(id=problem_id, contest_id=self.contest_id) self.contest = self.problem.contest else: self.problem = Problem.objects.get(id=problem_id) def _request(self, url, data=None): - kwargs = {"headers": {"X-Judge-Server-Token": self.token, - "Content-Type": "application/json"}} + kwargs = {"headers": {"X-Judge-Server-Token": self.token}} if data: - kwargs["data"] = json.dumps(data) + kwargs["json"] = data try: return requests.post(url, **kwargs).json() except Exception as e: @@ -55,7 +52,6 @@ class JudgeDispatcher(object): @staticmethod def choose_judge_server(): with transaction.atomic(): - # TODO: use more reasonable way servers = JudgeServer.objects.select_for_update().all().order_by("task_number") servers = [s for s in servers if s.status == "normal"] if servers: @@ -65,10 +61,10 @@ class JudgeDispatcher(object): return server @staticmethod - def release_judge_res(judge_server_id): + def release_judge_server(judge_server_id): with transaction.atomic(): # 使用原子操作, 同时因为use和release中间间隔了判题过程,需要重新查询一下 - server = JudgeServer.objects.select_for_update().get(id=judge_server_id) + server = JudgeServer.objects.get(id=judge_server_id) server.used_instance_number = F("task_number") - 1 server.save() @@ -94,7 +90,7 @@ class JudgeDispatcher(object): server = self.choose_judge_server() if not server: data = {"submission_id": self.submission.id, "problem_id": self.problem.id} - self.redis_conn.lpush(CacheKey.waiting_queue, json.dumps(data)) + cache.lpush(CacheKey.waiting_queue, json.dumps(data)) return sub_config = list(filter(lambda item: self.submission.language == item["name"], languages))[0] @@ -138,7 +134,7 @@ class JudgeDispatcher(object): else: self.submission.result = JudgeStatus.PARTIALLY_ACCEPTED self.submission.save() - self.release_judge_res(server.id) + self.release_judge_server(server.id) self.update_problem_status() if self.contest_id: @@ -223,7 +219,7 @@ class JudgeDispatcher(object): if self.contest_id and self.contest.status != ContestStatus.CONTEST_UNDERWAY: return if self.contest.real_time_rank: - default_cache.delete(CacheKey.contest_rank_cache + str(self.contest_id)) + cache.delete(CacheKey.contest_rank_cache + str(self.contest_id)) with transaction.atomic(): if self.contest.rule_type == ContestRuleType.ACM: acm_rank, _ = ACMContestRank.objects.select_for_update(). \ diff --git a/oj/local_settings.py b/oj/local_settings.py index cee68f25..bbe2398f 100644 --- a/oj/local_settings.py +++ b/oj/local_settings.py @@ -5,8 +5,12 @@ BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) DATABASES = { 'default': { - 'ENGINE': 'django.db.backends.sqlite3', - 'NAME': os.path.join(BASE_DIR, 'db.sqlite3'), + 'ENGINE': 'django.db.backends.postgresql_psycopg2', + 'HOST': '127.0.0.1', + 'PORT': 5433, + 'NAME': "onlinejudge", + 'USER': "onlinejudge", + 'PASSWORD': 'onlinejudge' } } diff --git a/oj/settings.py b/oj/settings.py index 672bc6f3..ac972c79 100644 --- a/oj/settings.py +++ b/oj/settings.py @@ -61,7 +61,6 @@ MIDDLEWARE_CLASSES = ( 'account.middleware.SessionRecordMiddleware', # 'account.middleware.LogSqlMiddleware', ) -SESSION_ENGINE = 'django.contrib.sessions.backends.cache' ROOT_URLCONF = 'oj.urls' TEMPLATES = [ @@ -166,41 +165,33 @@ LOGGING = { } -REST_FRAMEWORK = { - 'TEST_REQUEST_DEFAULT_FORMAT': 'json', - 'DEFAULT_RENDERER_CLASSES': ( - 'rest_framework.renderers.JSONRenderer', - ) -} +REDIS_URL = "redis://127.0.0.1:6379" -CACHE_JUDGE_QUEUE = "judge_queue" -CACHE_THROTTLING = "throttling" +def redis_config(db): + def make_key(key, key_prefix, version): + return key + + return { + "BACKEND": "utils.cache.MyRedisCache", + "LOCATION": f"{REDIS_URL}/{db}", + "TIMEOUT": None, + "KEY_PREFIX": "", + "KEY_FUNCTION": make_key + } CACHES = { - "default": { - "BACKEND": "django_redis.cache.RedisCache", - "LOCATION": "redis://127.0.0.1:6379/1", - "OPTIONS": { - "CLIENT_CLASS": "django_redis.client.DefaultClient", - } - }, - CACHE_JUDGE_QUEUE: { - "BACKEND": "django_redis.cache.RedisCache", - "LOCATION": "redis://127.0.0.1:6379/2", - "OPTIONS": { - "CLIENT_CLASS": "django_redis.client.DefaultClient", - } - }, - CACHE_THROTTLING: { - "BACKEND": "django_redis.cache.RedisCache", - "LOCATION": "redis://127.0.0.1:6379/3", - "OPTIONS": { - "CLIENT_CLASS": "django_redis.client.DefaultClient", - } - } + "default": redis_config(db=1) } + +CELERY_RESULT_BACKEND = CELERY_BROKER_URL = f"{REDIS_URL}/2" +CELERY_TASK_SOFT_TIME_LIMIT = CELERY_TASK_TIME_LIMIT = 180 + +SESSION_ENGINE = "django.contrib.sessions.backends.cache" +SESSION_CACHE_ALIAS = "default" + + # For celery REDIS_QUEUE = { "host": "127.0.0.1", diff --git a/options/options.py b/options/options.py index 51beb539..b2d76f16 100644 --- a/options/options.py +++ b/options/options.py @@ -113,15 +113,15 @@ class _SysOptionsMeta(type): @property def website_base_url(cls): return cls._get_option(OptionKeys.website_base_url) - + @website_base_url.setter def website_base_url(cls, value): cls._set_option(OptionKeys.website_base_url, value) - + @property def website_name(cls): return cls._get_option(OptionKeys.website_name) - + @website_name.setter def website_name(cls, value): cls._set_option(OptionKeys.website_name, value) @@ -173,7 +173,7 @@ class _SysOptionsMeta(type): @judge_server_token.setter def judge_server_token(cls, value): cls._set_option(OptionKeys.judge_server_token, value) - + class SysOptions(metaclass=_SysOptionsMeta): pass diff --git a/problem/serializers.py b/problem/serializers.py index d9e49195..4856b6d6 100644 --- a/problem/serializers.py +++ b/problem/serializers.py @@ -71,6 +71,7 @@ class CreateContestProblemSerializer(CreateOrEditProblemSerializer): class TagSerializer(serializers.ModelSerializer): class Meta: model = ProblemTag + fields = "__all__" class BaseProblemSerializer(serializers.ModelSerializer): @@ -88,6 +89,7 @@ class BaseProblemSerializer(serializers.ModelSerializer): class ProblemAdminSerializer(BaseProblemSerializer): class Meta: model = Problem + fields = "__all__" class ContestProblemAdminSerializer(BaseProblemSerializer): diff --git a/submission/views/oj.py b/submission/views/oj.py index 274418a7..e9672a0a 100644 --- a/submission/views/oj.py +++ b/submission/views/oj.py @@ -5,16 +5,16 @@ from problem.models import Problem, ProblemRuleType from contest.models import Contest, ContestStatus, ContestRuleType from utils.api import APIView, validate_serializer from utils.throttling import TokenBucket, BucketController +from utils.cache import cache from ..models import Submission from ..serializers import CreateSubmissionSerializer, SubmissionModelSerializer from ..serializers import SubmissionSafeSerializer, SubmissionListSerializer -from utils.cache import throttling_cache def _submit(response, user, problem_id, language, code, contest_id): # TODO: 预设默认值,需修改 controller = BucketController(user_id=user.id, - redis_conn=throttling_cache, + redis_conn=cache, default_capacity=30) bucket = TokenBucket(fill_rate=10, capacity=20, last_capacity=controller.last_capacity, diff --git a/utils/cache.py b/utils/cache.py index c77131f6..ed9059b1 100644 --- a/utils/cache.py +++ b/utils/cache.py @@ -1,6 +1,27 @@ -from django.conf import settings -from django_redis import get_redis_connection +from django.core.cache import cache, caches # noqa +from django.conf import settings # noqa -judge_cache = get_redis_connection(settings.CACHE_JUDGE_QUEUE) -throttling_cache = get_redis_connection(settings.CACHE_THROTTLING) -default_cache = get_redis_connection("default") +from django_redis.cache import RedisCache +from django_redis.client.default import DefaultClient + + +class MyRedisClient(DefaultClient): + def __getattr__(self, item): + client = self.get_client(write=True) + return getattr(client, item) + + def redis_incr(self, key, count=1): + """ + django 默认的 incr 在 key 不存在时候会抛异常 + """ + client = self.get_client(write=True) + return client.incr(key, count) + + +class MyRedisCache(RedisCache): + def __init__(self, server, params): + super().__init__(server, params) + self._client_cls = MyRedisClient + + def __getattr__(self, item): + return getattr(self.client, item)