Add problem transfer api

dmoj/settings.py

@@ -28,6 +28,8 @@
 DEBUG = True
 
 ALLOWED_HOSTS = []
+CORS_ALLOWED_ORIGINS = []
+CORS_URLS_REGEX = r'^/problem-export/'
 
 CSRF_FAILURE_VIEW = 'judge.views.widgets.csrf_failure'
 
@@ -117,6 +119,14 @@
 
 VNOJ_LONG_QUEUE_ALERT_THRESHOLD = 10
 
+# Transfer host
+VNOJ_PROBLEM_ENABLE_IMPORT = False
+VNOJ_PROBLEM_ENABLE_EXPORT = False
+VNOJ_PROBLEM_IMPORT_HOST = 'https://oj.vnoi.info'
+VNOJ_PROBLEM_IMPORT_SECRET = ''
+VNOJ_PROBLEM_IMPORT_JUDGE_PREFIX = 'vnoj/'
+VNOJ_PROBLEM_IMPORT_TIMEOUT = 5     # in seconds
+
 CELERY_TIMEZONE = 'Asia/Ho_Chi_Minh'
 
 # Some problems have a lot of testcases, and each testcase
@@ -326,6 +336,7 @@
                         'judge.ProblemGroup',
                         'judge.ProblemType',
                         'judge.License',
+                        'judge.ProblemExportKey',
                     ],
                 },
                 {
@@ -421,9 +432,11 @@
     'martor',
     'adminsortable2',
     'django_cleanup.apps.CleanupConfig',
+    'corsheaders',
 )
 
 MIDDLEWARE = (
+    'corsheaders.middleware.CorsMiddleware',
     'judge.middleware.ShortCircuitMiddleware',
     'django.middleware.common.CommonMiddleware',
     'django.contrib.sessions.middleware.SessionMiddleware',
@@ -496,6 +509,7 @@
                 'judge.template_context.site_theme',
                 'judge.template_context.misc_config',
                 'judge.template_context.math_setting',
+                'judge.template_context.site_setting',
                 'social_django.context_processors.backends',
                 'social_django.context_processors.login_redirect',
             ],
@@ -668,6 +682,7 @@
 BRIDGED_JUDGE_PROXIES = None
 BRIDGED_DJANGO_ADDRESS = [('localhost', 9998)]
 BRIDGED_DJANGO_CONNECT = None
+BRIDGED_MONITOR_UPDATE_URL = None
 
 # Event Server configuration
 EVENT_DAEMON_USE = False

dmoj/urls.py

@@ -21,6 +21,7 @@
 from judge.views.magazine import MagazinePage
 from judge.views.problem_data import ProblemDataView, ProblemSubmissionDiff, \
     problem_data_file, problem_init_view
+from judge.views.problem_transfer import ProblemExportSelect2View, ProblemExportView, ProblemImportView
 from judge.views.register import ActivationView, RegistrationView
 from judge.views.select2 import AssigneeSelect2View, CommentSelect2View, ContestSelect2View, \
     ContestUserSearchSelect2View, OrganizationSelect2View, OrganizationUserSelect2View, ProblemSelect2View, \
@@ -421,6 +422,12 @@ def paged_list_view(view, name):
     ])),
 
     path('magazine/', MagazinePage.as_view(), name='magazine'),
+
+    re_path('^problem-export/(?P<secret>[a-zA-Z0-9_-]{48})', include([
+        path('', ProblemExportView.as_view(), name='problem_export'),
+        path('/select', ProblemExportSelect2View.as_view(), name='problem_export_select2_ajax'),
+    ])),
+    path('problem-import', ProblemImportView.as_view(), name='problem_import'),
 ]
 
 favicon_paths = ['apple-touch-icon-180x180.png', 'apple-touch-icon-114x114.png', 'android-chrome-72x72.png',

judge/admin/__init__.py

@@ -7,7 +7,7 @@
 from judge.admin.contest import ContestAdmin, ContestParticipationAdmin, ContestTagAdmin
 from judge.admin.interface import BlogPostAdmin, FlatPageAdmin, LicenseAdmin, LogEntryAdmin, NavigationBarAdmin
 from judge.admin.organization import OrganizationAdmin, OrganizationRequestAdmin
-from judge.admin.problem import ProblemAdmin
+from judge.admin.problem import ProblemAdmin, ProblemExportKeyAdmin
 from judge.admin.profile import ProfileAdmin, UserAdmin
 from judge.admin.runtime import JudgeAdmin, LanguageAdmin
 from judge.admin.submission import SubmissionAdmin
@@ -16,7 +16,7 @@
 from judge.admin.ticket import TicketAdmin
 from judge.models import Badge, BlogPost, Comment, CommentLock, Contest, ContestParticipation, \
     ContestTag, Judge, Language, License, MiscConfig, NavigationBar, Organization, \
-    OrganizationRequest, Problem, ProblemGroup, ProblemType, Profile, Submission, Tag, \
+    OrganizationRequest, Problem, ProblemExportKey, ProblemGroup, ProblemType, Profile, Submission, Tag, \
     TagGroup, TagProblem, Ticket
 
 admin.site.register(BlogPost, BlogPostAdmin)
@@ -37,6 +37,7 @@
 admin.site.register(Organization, OrganizationAdmin)
 admin.site.register(OrganizationRequest, OrganizationRequestAdmin)
 admin.site.register(Problem, ProblemAdmin)
+admin.site.register(ProblemExportKey, ProblemExportKeyAdmin)
 admin.site.register(ProblemGroup, ProblemGroupAdmin)
 admin.site.register(ProblemType, ProblemTypeAdmin)
 admin.site.register(Profile, ProfileAdmin)

judge/admin/problem.py

@@ -234,3 +234,18 @@ def construct_change_message(self, request, form, *args, **kwargs):
         if form.cleaned_data.get('change_message'):
             return form.cleaned_data['change_message']
         return super(ProblemAdmin, self).construct_change_message(request, form, *args, **kwargs)
+
+
+class ProblemExportKeyAdmin(VersionAdmin):
+    fieldsets = (
+        (None, {'fields': ('name', 'remaining_uses')}),
+        (_('Description'), {'fields': ('description',)}),
+    )
+    list_display = ['name', 'remaining_uses']
+    search_fields = ('name', 'description')
+
+    def save_model(self, request, obj, form, change):
+        super().save_model(request, obj, form, change)
+        if not change:
+            secret = obj.generate_secret()
+            self.message_user(request, gettext('Generated secret: %s') % secret)

judge/balancer/balancer.py

@@ -64,7 +64,7 @@ def _try_judge(self):
                 self.judge_to_bridge[judge.name] = bridge_id
                 self.bridge_to_judge[bridge_id] = judge
 
-                packet['storage-namespace'] = self.config['bridges'][bridge_id].get('storage_namespace')
+                packet.setdefault('storage-namespace', self.config['bridges'][bridge_id].get('storage_namespace'))
                 judge.submit(packet)
 
     def free_judge(self, judge):

judge/bridge/daemon.py

@@ -18,20 +18,20 @@ def reset_judges():
     Judge.objects.update(online=False, ping=None, load=None)
 
 
-def judge_daemon(run_monitor=False, problem_storage_globs=None):
+def judge_daemon(config):
     reset_judges()
     Submission.objects.filter(status__in=Submission.IN_PROGRESS_GRADING_STATUS) \
         .update(status='IE', result='IE', error=None)
     judges = JudgeList()
 
     monitor = None
-    if run_monitor:
+    if 'run_monitor' in config:
         from judge.bridge.monitor import Monitor
-        monitor = Monitor(judges, problem_storage_globs or [])
+        monitor = Monitor(judges, **config)
 
     judge_server = Server(
         settings.BRIDGED_JUDGE_ADDRESS,
-        partial(JudgeHandler, judges=judges, ignore_problems_packet=run_monitor),
+        partial(JudgeHandler, judges=judges, ignore_problems_packet=('run_monitor' in config)),
     )
     django_server = Server(settings.BRIDGED_DJANGO_ADDRESS, partial(DjangoHandler, judges=judges))
 

judge/bridge/judge_handler.py

@@ -237,7 +237,7 @@ def submit(self, id, problem, language, source):
         data = self.get_related_submission_data(id)
         self._working = id
         self._no_response_job = threading.Timer(20, self._kill_if_no_response)
-        self.send({
+        request = {
             'name': 'submission-request',
             'submission-id': id,
             'problem-id': problem,
@@ -254,7 +254,14 @@ def submit(self, id, problem, language, source):
                 'file-only': data.file_only,
                 'file-size-limit': data.file_size_limit,
             },
-        })
+        }
+        if '/' in problem:
+            storage_namespace, problem = problem.split('/')
+            request.update({
+                'storage-namespace': storage_namespace,
+                'problem-id': problem,
+            })
+        self.send(request)
 
     def _kill_if_no_response(self):
         logger.error('Judge failed to acknowledge submission: %s: %s', self.name, self._working)

judge/bridge/monitor.py

@@ -3,9 +3,13 @@
 import os
 import threading
 import time
+from contextlib import closing
+from http.server import BaseHTTPRequestHandler, HTTPServer
 from pathlib import Path
+from urllib.request import urlopen
 
 from django import db
+from django.db.models import Q
 
 from judge.models import Problem
 
@@ -27,6 +31,27 @@
 logger = logging.getLogger('judge.monitor')
 
 
+class JudgeControlRequestHandler(BaseHTTPRequestHandler):
+    signal = None
+
+    def update_problems(self):
+        if self.signal is not None:
+            self.signal.set()
+
+    def do_POST(self):
+        if self.path == '/update/problems':
+            logger.info('Problem update requested')
+            self.update_problems()
+            self.send_response(200)
+            self.end_headers()
+            self.wfile.write(b'As you wish.')
+            return
+        self.send_error(404)
+
+    def do_GET(self):
+        self.send_error(404)
+
+
 def _ensure_connection():
     db.connection.close_if_unusable_or_obsolete()
 
@@ -49,48 +74,87 @@ class SendProblemsHandler(FileSystemEventHandler):
         EVENT_TYPE_CREATED,
     )
 
-    def __init__(self, signal):
+    def __init__(self, signal, propagation_signal):
         self.signal = signal
+        self.propagation_signal = propagation_signal
 
     def on_any_event(self, event):
         if event.event_type not in self.ALLOWED_EVENT_TYPES:
             return
         self.signal.set()
+        self.propagation_signal.set()
 
 
 class Monitor:
-    def __init__(self, judges, problem_globs):
+    def __init__(self, judges, **config):
         if not has_watchdog_installed:
             raise ImportError('watchdog is not installed')
 
         self.judges = judges
-        self.problem_globs = problem_globs
+        self.problem_globs = [(entry['storage_namespaces'], entry['problem_storage_globs'])
+                              for entry in config['run_monitor']]
 
         self.updater_exit = False
         self.updater_signal = threading.Event()
+        self.propagation_signal = threading.Event()
         self.updater = threading.Thread(target=self.updater_thread)
+        self.propagator = threading.Thread(target=self.propagate_update_signal)
+
+        self.update_pings = config.get('update_pings') or []
+
+        api_listen = config.get('api_listen')
+        if api_listen:
+            api_listen = (api_listen['host'], api_listen['port'])
 
-        self._handler = SendProblemsHandler(self.updater_signal)
+            class Handler(JudgeControlRequestHandler):
+                signal = self.updater_signal
+
+            self.api_server = HTTPServer(api_listen, Handler)
+        else:
+            self.api_server = None
+
+        self._handler = SendProblemsHandler(self.updater_signal, self.propagation_signal)
         self._observer = Observer()
 
-        for root in set(map(find_glob_root, problem_globs)):
+        for _, dir_glob in self.problem_globs:
+            root = find_glob_root(dir_glob)
             self._observer.schedule(self._handler, root, recursive=True)
             logger.info('Scheduled for monitoring: %s', root)
 
     def update_supported_problems(self):
         problems = []
-        for dir_glob in self.problem_globs:
+        for storage_namespace, dir_glob in self.problem_globs:
             for problem_config in glob.iglob(os.path.join(dir_glob, 'init.yml'), recursive=True):
                 if os.access(problem_config, os.R_OK):
                     problem_dir = os.path.dirname(problem_config)
                     problem = os.path.basename(problem_dir)
-                    problems.append(problem)
+                    if storage_namespace:
+                        problems.append(storage_namespace + '/' + problem)
+                    else:
+                        problems.append(problem)
         problems = set(problems)
 
         _ensure_connection()
-        problem_ids = list(Problem.objects.filter(code__in=list(problems)).values_list('id', flat=True))
+        problem_ids = list(Problem.objects.filter(Q(code__in=list(problems)) | Q(judge_code__in=list(problems)))
+                           .values_list('id', flat=True))
         self.judges.update_problems_all(problems, problem_ids)
 
+    def propagate_update_signal(self):
+        while True:
+            self.propagation_signal.wait()
+            self.propagation_signal.clear()
+            if self.updater_exit:
+                return
+
+            for url in self.update_pings:
+                logger.info('Pinging for problem update: %s', url)
+                try:
+                    with closing(urlopen(url, data=b'')) as f:
+                        f.read()
+                except Exception:
+                    logger.exception('Failed to ping for problem update: %s', url)
+            time.sleep(3)
+
     def updater_thread(self) -> None:
         while True:
             self.updater_signal.wait()
@@ -106,14 +170,21 @@ def updater_thread(self) -> None:
 
     def start(self):
         self.updater.start()
+        self.propagator.start()
         self.updater_signal.set()
         try:
+            if self.api_server:
+                thread = threading.Thread(target=self.api_server.serve_forever)
+                thread.start()
             self._observer.start()
         except OSError:
             logger.exception('Failed to start problem monitor.')
 
     def stop(self):
         self._observer.stop()
         self._observer.join(1)
+        if self.api_server:
+            self.api_server.shutdown()
         self.updater_exit = True
         self.updater_signal.set()
+        self.propagation_signal.set()

judge/judgeapi.py

@@ -94,7 +94,7 @@ def judge_submission(submission, rejudge=False, batch_rejudge=False, judge_id=No
         response = judge_request({
             'name': 'submission-request',
             'submission-id': submission.id,
-            'problem-id': submission.problem.code,
+            'problem-id': submission.problem.judge_code or submission.problem.code,
             'language': submission.language.key,
             'source': submission.source.source,
             'judge-id': judge_id,

judge/management/commands/runbridged.py

@@ -1,14 +1,17 @@
+import yaml
 from django.core.management.base import BaseCommand
 
 from judge.bridge.daemon import judge_daemon
 
 
 class Command(BaseCommand):
-    def add_arguments(self, parser):
-        parser.add_argument('--monitor', action='store_true', default=False,
-                            help='if specified, run a monitor to automatically update problems')
-        parser.add_argument('--problem-storage-globs', nargs='*', default=[],
-                            help='globs to monitor for problem updates')
+    def add_arguments(self, parser) -> None:
+        parser.add_argument('-c', '--config', type=str, help='file to load bridged configurations from')
 
     def handle(self, *args, **options):
-        judge_daemon(options['monitor'], options['problem_storage_globs'])
+        if options['config']:
+            with open(options['config'], 'r') as f:
+                config = yaml.safe_load(f)
+        else:
+            config = {}
+        judge_daemon(config)