Merge branch 'main' of github.com:IACR/latex-submit

webapp/__init__.py

@@ -22,6 +22,10 @@
 import secrets
 import string
 import sys
+import json
+from apscheduler.schedulers.background import BackgroundScheduler
+from apscheduler.executors import pool
+from apscheduler.triggers.interval import IntervalTrigger
 
 # Make sure we aren't running on an old python.
 assert sys.version_info >= (3, 7)
@@ -62,6 +66,11 @@ def get_pdf_url(paperid, version):
 login_manager = LoginManager()
 login_manager.login_view = 'auth.login'
 
+class Scheduler(BackgroundScheduler):
+    """A simple wrapper around apscheduler. Much simpler than flask_apscheduler."""
+    def init_app(self, app):
+        self.app = app
+
 class SQLAlchemy():
     """Wrapper for sqlalchemy scoped_session and engine, like flask-sqlalchemy."""
     def __init__(self, engine, session):
@@ -70,6 +79,9 @@ def __init__(self, engine, session):
 
 # DB wrapper is global so we can have easy access to engine and session.
 db = SQLAlchemy(None, None)
+# scheduler is only used to clean up old jobs in case it has config.DEMO_INSTANCE set to tru
+scheduler = Scheduler(executors={'default': pool.ThreadPoolExecutor(1)},
+                      timezone='America/Los_Angeles')
 
 # In case we want to try to force the pragma for sqlite3.
 # @event.listens_for(Engine, 'connect')
@@ -94,6 +106,20 @@ def shutdown_session(ex):
     # Create database tables if they don't already exist.
     Base.metadata.create_all(bind=db.engine)
     login_manager.init_app(app)
+    if config.DEMO_INSTANCE:
+        from .cleanup import cleanup_task
+        scheduler.init_app(app)
+        scheduler.start()
+        # Check every 15 minutes to clean up old papers.
+        trigger = IntervalTrigger(minutes=15)
+        scheduler.add_job(cleanup_task,
+                          trigger=trigger,
+                          args=[],
+                          id='cleanup_update',
+                          name='cleanup_update')
+        app.logger.warning([str(job) for job in scheduler.get_jobs()])
+    else:
+        app.logger.warning('Scheduler was not started')
     with app.app_context():
         from . import admin
         from . import routes

webapp/admin.py

@@ -173,7 +173,7 @@ def user():
                                                ts=timestamp,
                                                _external=True)}
             msg.body = app.jinja_env.get_template('admin/new_account.txt').render(maildata)
-            if app.config['TESTING']:
+            if app.config['DEBUG']:
                 print(msg.body)
             mail.send(msg)
             flash('User {} was created and they were notified.'.format(form.email.data))
@@ -276,7 +276,7 @@ def approve_final():
                 'paperid': paperid,
                 'copyedit_url': url_for('admin_file.copyedit_home', _external=True)}
     editor_msg.body = app.jinja_env.get_template('admin/copyedit_approved.txt').render(maildata)
-    if app.config['TESTING']:
+    if app.config['DEBUG']:
         print(editor_msg.body)
     mail.send(editor_msg)
     ############ send a message to the author.
@@ -289,7 +289,7 @@ def approve_final():
                              sender=app.config['EDITOR_EMAILS'],
                              recipients=[paper_status.email])
         author_msg.body = app.jinja_env.get_template('admin/author_finished.txt').render(maildata)
-        if app.config['TESTING']:
+        if app.config['DEBUG']:
             print(author_msg.body)
         mail.send(author_msg)
     except Exception as e:
@@ -505,7 +505,7 @@ def finish_copyedit():
                              sender=app.config['EDITOR_EMAILS'],
                              recipients=[paper_status.email])
         author_msg.body = app.jinja_env.get_template('admin/author_finished.txt').render(maildata)
-        if app.config['TESTING']:
+        if app.config['DEBUG']:
             print(author_msg.body)
         mail.send(author_msg)
         log_event(db, paperid, 'Copy edit was finished and author notified')
@@ -531,7 +531,7 @@ def finish_copyedit():
                 'final_url': url_for('home_bp.view_copyedit', paperid=paperid,auth=create_hmac([paperid, Version.COPYEDIT.value]),
                                      _external=True)}
     msg.body = app.jinja_env.get_template('admin/copyedit_finished.txt').render(maildata)
-    if app.config['TESTING']:
+    if app.config['DEBUG']:
         print(msg.body)
     mail.send(msg)
     log_event(db, paperid, 'Copy edit was finished and author notified')

webapp/auth.py

@@ -119,7 +119,7 @@ def _send_login_link(initiator, email):
                                        auth=create_hmac([email, str(ts)]),
                                        _external=True)}
     msg.body = app.jinja_env.get_template('recover_password.txt').render(maildata)
-    if app.config['TESTING']:
+    if app.config['DEBUG']:
         print(msg.body)
     mail.send(msg)
     return True

webapp/cleanup.py

@@ -0,0 +1,45 @@
+"""This is used only if config.DEMO_INSTANCE is True, and is
+used to remove old articles.
+"""
+
+from datetime import datetime, timedelta
+from pathlib import Path
+import shutil
+
+from sqlalchemy import create_engine, select
+from sqlalchemy.orm import Session
+from . import scheduler
+from .metadata.db_models import PaperStatus
+
+def cleanup_task():
+    with scheduler.app.app_context():
+        if scheduler.app.config['DEBUG']:
+            scheduler.app.logger.warning('skipping cleanup of existing papers')
+            return
+        if scheduler.app.config['DEMO_INSTANCE']:
+            scheduler.app.logger.warning('cleanup should not be configured')
+            return
+        scheduler.app.logger.warning('cleaning up papers')
+        submit_deadline = datetime.now() - timedelta(days=1)
+        copyedit_deadline = datetime.now() - timedelta(days=4)
+        engine = create_engine(scheduler.app.config['SQLALCHEMY_DATABASE_URI'])
+        deleted_ids = set()
+        with Session(engine) as session:
+            papers = session.execute(select(PaperStatus)).scalars().all()
+            for paper in papers:
+                if paper.status == PaperStatus.PENDING.value and paper.lastmodified < submit_deadline:
+                    deleted_ids.add(paper.paperid)
+                    session.delete(paper)
+                elif paper.lastmodified < copyedit_deadline:
+                    deleted_ids.add(paper.paperid)
+                    session.delete(paper)
+            session.commit()
+        engine.dispose(True)
+        for paperid in deleted_ids:
+            dir = Path(scheduler.app.config['DATA_DIR']) / Path(paperid)
+            try:
+                shutil.rmtree(dir)
+                scheduler.app.logger.warning('Deleted {}'.format(dir.name))
+            except Exception as e:
+                scheduler.app.logger.warning('Unable to delete the directory for the paper {}. ({})'.format(str(dir.absolute()), str(e)))
+

webapp/config.py

@@ -24,10 +24,11 @@ class Config:
     CROSSREF_PUBLISHER_EMAIL = '[email protected]'
     USERS = None
     TESTING = False
+    DEMO_INSTANCE = False
     WTF_CSRF_TIME_LIMIT = None
     EXPORT_PATH = '/tmp'
     FUNDING_SEARCH_URL = '/searchapi/search'
-    SUBMIT_BYPASS = 'testing'
+    SUBMIT_BYPASS = '' # now deprecated and replaced by DEMO_INSTANCE
     JOURNALS = [
         {
             'hotcrp_key': 'cic',
@@ -75,6 +76,7 @@ class DebugConfig(Config):
     FLASK_ENV = 'development'
     DEBUG = True
     TESTING = True
+    DEMO_INSTANCE = True
     # used by kevin.
     SQLALCHEMY_DATABASE_URI = 'mysql+pymysql://publication:st0remydata@localhost/publication'
     # SQLALCHEMY_DATABASE_URI = 'mysql+pymysql://publication:mystuff@localhost/publication'

webapp/requirements.txt

@@ -54,3 +54,4 @@ requests
 docker
 markdown
 pymemcache
+apscheduler

webapp/routes.py

@@ -51,20 +51,21 @@ def home():
 def show_submit_version():
     form = SubmitForm(request.args)
     if not form.paperid.data:
-        bypass = request.args.get('bypass', None)
-        if bypass != app.config['SUBMIT_BYPASS']: # just for testing.
+        if not app.config['DEMO_INSTANCE']:
             return redirect(url_for('home_bp.home'))
-        #TODO: remove this if. It's only for testing to supply a paperid when it doesn't come from internal.
         # In this case the submission doesn't come from hotcrp, so we make up some fields.
         random.seed()
         form.paperid.data = ''.join(random.choices(string.ascii_lowercase + string.digits, k=8))
         form.hotcrp.data = NO_HOTCRP
         form.hotcrp_id.data = NO_HOTCRP
         form.version.data = 'candidate'
-        form.accepted.data = '2022-09-30 17:49:20'
-        form.submitted.data = '2022-08-03 06:44:30'
+        now = datetime.datetime.now()
+        submitted = now - datetime.timedelta(days=10)
+        accepted = now - datetime.timedelta(days=5)
+        form.accepted.data = accepted.strftime('%Y-%m-%d %H:%M:%S')
+        form.submitted.data = submitted.strftime('%Y-%m-%d %H:%M:%S')
         form.journal.data = 'cic'
-        form.volume.data = '1'
+        form.volume.data = '9999'
         form.issue.data = '1'
         form.generate_auth()
     else:
@@ -509,7 +510,7 @@ def compile_for_copyedit():
     copyedit_url = url_for('admin_file.copyedit', paperid=paperid, _external=True)
     msg.body = 'A paper for CiC is being compiled for copy editing.\n\nYou can view it at {}'.format(copyedit_url)
     mail.send(msg)
-    if app.config['TESTING']:
+    if app.config['DEBUG']:
         print(msg.body)
     data = {'title': 'Compiling your LaTeX for copy editor',
             'status_url': status_url,
@@ -546,7 +547,7 @@ def final_review():
     final_review_url = url_for('admin_file.final_review', paperid=paperid, _external=True)
     msg.body = 'A paper for CiC needs final review.\n\nYou can view it at {}'.format(final_review_url)
     mail.send(msg)
-    if app.config['TESTING']:
+    if app.config['DEBUG']:
         print(msg.body)
     return render_template('message.html',
                            title='Your paper will be reviewed',
@@ -619,6 +620,9 @@ def view_copyedit(paperid, auth):
             try:
                 json_file = paper_path / Path('compilation.json')
                 comp = Compilation.model_validate_json(json_file.read_text(encoding='UTF-8'))
+            except Exception as e:
+                logging.error('Unable to read compilation:' + str(e))
+            try:
                 data['comp'] = comp
                 if comp.bibtex:
                     data['marked_bibtex'] = mark_bibtex(comp.bibtex)
@@ -630,7 +634,7 @@ def view_copyedit(paperid, auth):
                 else:
                     data['bibtex_log'] = ['No bibtex log']
             except Exception as e:
-                logging.error('Unable to parse compilation:' + str(e))
+                logging.error('Unable to ingest compilation:' + str(e))
                 return render_template('message.html',
                                        title='An error has occurred',
                                        error='An error has occurred reading data. Please contact the admin.')

webapp/templates/base.html

@@ -26,6 +26,10 @@ <h4 class="text-center">Is it made out of <a href="https://iacr.org/tinfoil.html
 	<a class="navbar-brand mx-md-5" href="https://iacr.org/"><img class="iacrlogo" src="/img/iacrlogo_small.png" alt="IACR Logo" style="max-width:6rem;"></a>
 	<div class="d-none d-md-inline navbar-brand">{{site_name}}</div>
 	<span class="d-inline d-md-none navbar-brand">{{site_shortname}}</span>
+          {% if config.DEMO_INSTANCE %}
+          <div class="navbar-brand mx-md-2"><span class="text-danger bg-warning p-2 rounded">TEST INSTANCE</span></div>
+          {% endif %}
+        </span>
 	<div class="collapse navbar-collapse" id="navbarSupportedContent">
 	  <ul class="navbar-nav mb-2 mb-lg-0 mx-auto">
             <li class="nav-item">

webapp/templates/index.html

@@ -9,8 +9,13 @@
 <div class="row mt-4">
   <div class="col-12 col-md-6">
     <div class="alert alert-danger">
+      {% if config.DEMO_INSTANCE %}
+      This is only a test server. If you are trying to submit a final version for the journal,
+      then you should start from your hotcrp instance.
+      {% else %}
       This site is currently used for submissions of <strong>final</strong> versions of
       papers for <a href="https://cic.iacr.org/">IACR Communications in Cryptology</a>.
+      {% endif %}
       If you are looking to submit a paper to the journal, see
       the <a href="https://cic.iacr.org/callforpapers">Call for Papers</a>.
     </div>
@@ -25,6 +30,17 @@
 </div>
 <div class="row mt-3">
   <div class="col-12 col-md-8 offset-md-2">
+    {% if config.DEMO_INSTANCE %}
+    <p class="alert alert-info">
+      This test server allows people to experiment with the infrastructure, but is not part of
+      the actual pipeline for the journal. Papers uploaded here are not private,
+      but they will be deleted every day so nothing is permanent. If you wish to have
+      access as an admin, notify {{config.COPYEDITOR_EMAILS}}.
+    </p>
+    <div class="text-center my-2">
+      <a class="btn btn-primary" href="{{url_for('home_bp.show_submit_version', bypass=config.SUBMIT_BYPASS)}}">Upload a paper</a>
+    </div>
+    {% endif %}
     <p>
       The goal of this site is to reduce the cost of open access publishing by
       <a href="http://ftp.tug.org/TUGboat/tb44-1/tb136bos-metadata.pdf">reducing the

webapp/templates/login.html

@@ -42,6 +42,7 @@ <h1 class="mt-4">Log In</h1>
   </div>
   {% endfor %}
   <p class="alert alert-info mt-4">
+    Accounts are only for administrators or copy editors.
     If you forgot your password, you can try to <a href="/recover">reset your password</a>
   </p>
   {#

webapp/templates/submit.html

@@ -149,6 +149,11 @@
         </div>
         {{ form.submit(class_='mt-5 btn btn-primary') }}
       </form>
+      {% if config.DEMO_INSTANCE %}
+      <div class="alert alert-warning mt-2">Nothing you submit here will be private, but papers that are submitted here
+        will be automatically deleted every few days. 
+      </div>
+      {% endif %}
     </div>
     <div class="tab-pane fade p-3" id="nav-help" role="tabpanel" aria-labelledby="nav-help-tab">
       <h3>How to prepare your paper</h3>