(SIO-2356) Make deployment easier

Create an easy way to deploy OIOIOI using the Docker technology as a backend.
Only the most basic settings should be exposed to the operator, all the other ones should be standardized.
Updating should also be noob-friendly, without the need to read diffs from UPGRADING.rst, etc.

In the light of these changes Vagrant support is also being removed.
Some improvements are also made to hopefully fix Selenium.

Change-Id: Ieb0b293dab8a7d36e6c7f3a0347c2a7a33341f77

.dockerignore

@@ -1,11 +1,14 @@
 Dockerfile*
 docker*
-.idea
+.dockerignore
+.git
 .github
+.idea
 *.swp
 venv
 logs
 deployment
+extra
 /test*
 *.rst
 /rst

Dockerfile

@@ -8,7 +8,6 @@ RUN dpkg --add-architecture i386 && \
         git \
         libpq-dev \
         postgresql-client \
-        rabbitmq-server \
         libdb-dev \
         fp-compiler fp-units-base fp-units-math \
         texlive-latex-base \
@@ -30,9 +29,7 @@ ARG oioioi_uid=1234
 #Bash as shell, setup folders, create oioioi user
 RUN rm /bin/sh && ln -s /bin/bash /bin/sh && \
     mkdir -pv /sio2/oioioi && \
-    mkdir -pv /sio2/logs && \
     mkdir -pv /sio2/sandboxes && \
-    chmod a+rw -R /sio2/logs && \
     useradd -U oioioi -m -d /home/oioioi/ -u $oioioi_uid && \
     echo "oioioi ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers && \
     chown -R oioioi:oioioi /sio2
@@ -41,76 +38,51 @@ RUN rm /bin/sh && ln -s /bin/bash /bin/sh && \
 RUN sed -i -e "s/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/" /etc/locale.gen && \
     locale-gen
 
-# Configure rabbitmq-server
-RUN echo "[{rabbit, [{tcp_listeners, [5672]}, {loopback_users, []}]}]." > /etc/rabbitmq/rabbitmq.config && \
-    echo "SERVER_ERL_ARGS=\"+K true +A 4 +P 1048576 -kernel\""          > /etc/rabbitmq/rabbitmq-env.conf
-
-# Configuring .bashrc
-RUN echo "source /sio2/venv/bin/activate" >> ~/.bashrc && \
-    echo "cd /sio2" >> ~/.bashrc && \
-    echo "export PATH=$PATH:~/.local/bin" >> ~/.bashrc
-
 # Installing python dependencies
 USER oioioi
 
-RUN easy_install --user distribute
-RUN pip install psycopg2-binary --user
-RUN pip install twisted --user
+ENV PATH $PATH:/home/oioioi/.local/bin/
+
+RUN pip install --user psycopg2-binary twisted uwsgi
 
 WORKDIR /sio2/oioioi
 
-COPY setup.py requirements.txt ./
+COPY --chown=oioioi:oioioi setup.py requirements.txt ./
 RUN pip install -r requirements.txt --user
 
-ADD --chown=oioioi:oioioi . /sio2/oioioi
-
-ENV PATH $PATH:/home/oioioi/.local/bin/
+COPY --chown=oioioi:oioioi . /sio2/oioioi
 
 RUN oioioi-create-config /sio2/deployment
 
 WORKDIR /sio2/deployment
 
 RUN sed -i -e \
-       "s/django.db.backends./django.db.backends.postgresql_psycopg2/g;\
+       "s/SERVER = 'django'/SERVER = 'uwsgi-http'/g;\
+        s/DEBUG = True/DEBUG = False/g;\
+        s/django.db.backends./django.db.backends.postgresql_psycopg2/g;\
         s/'NAME': ''/'NAME': 'oioioi'/g;\
         s/'USER': ''/'USER': 'oioioi'/g;\
         s/'HOST': '',/'HOST': 'db',/g;\
-        s/'PASSWORD': ''/'PASSWORD': ''/g;\
+        s/'PASSWORD': ''/'PASSWORD': 'password'/g;\
         s/#BROKER_URL/BROKER_URL/g;\
-        s/USE_UNSAFE_EXEC/#USE_UNSAFE_EXEC/g;\
-        s/#FILETRACKER_SERVER_ENABLED/FILETRACKER_SERVER_ENABLED/g;\
         s/#FILETRACKER_LISTEN_ADDR/FILETRACKER_LISTEN_ADDR/g;\
         s/#FILETRACKER_LISTEN_PORT/FILETRACKER_LISTEN_PORT/g;\
-        s/#FILETRACKER_LISTEN_URL/FILETRACKER_LISTEN_URL/g;\
-        s/#FILETRACKER_URL/FILETRACKER_URL/g;\
+        s|#FILETRACKER_URL = '.*'|FILETRACKER_URL = 'http://web:9999'|g;\
+        s/#RUN_SIOWORKERSD$/RUN_SIOWORKERSD/g;\
         s/#SIOWORKERS_LISTEN_ADDR/SIOWORKERS_LISTEN_ADDR/g;\
         s/#SIOWORKERS_LISTEN_PORT/SIOWORKERS_LISTEN_PORT/g;\
-        s/#RUN_SIOWORKERSD.*$/RUN_SIOWORKERSD = True/g;\
-        s/#USE_UNSAFE_EXEC = True/USE_UNSAFE_EXEC = True/g;\
+        s/RUN_LOCAL_WORKERS = True/RUN_LOCAL_WORKERS = False/g;\
         s/AVAILABLE_COMPILERS = SYSTEM_COMPILERS/#AVAILABLE_COMPILERS = SYSTEM_COMPILERS/g;\
         s/DEFAULT_COMPILERS = SYSTEM_DEFAULT_COMPILERS/#DEFAULT_COMPILERS = SYSTEM_DEFAULT_COMPILERS/g;\
+        s/USE_UNSAFE_EXEC = True/USE_UNSAFE_EXEC = False/g;\
+        s/#DEFAULT_SAFE_EXECUTION_MODE/DEFAULT_SAFE_EXECUTION_MODE/g;\
         s/#USE_UNSAFE_CHECKER = True/USE_UNSAFE_CHECKER = False/g;\
-        s/.*RUN_LOCAL_WORKERS = True/RUN_LOCAL_WORKERS = False/g;\
-        s/ALLOWED_HOSTS = \\[.*\\]/ALLOWED_HOSTS = \\['oioioi', '127.0.0.1', 'localhost', 'web'\\]/g"\
+        \$afrom basic_settings import *\nALLOWED_HOSTS = ALLOWED_HOSTS + \\['oioioi', '127.0.0.1', 'localhost', 'web'\\]" \
         settings.py && \
-    echo "SIOWORKERS_BACKEND = 'oioioi.sioworkers.backends.SioworkersdBackend'"     >> settings.py && \
-    echo "CELERY_RESULT_BACKEND = None"                                             >> settings.py && \
-    sed -i \
-        -e "s|twistd|/home/oioioi/.local/bin/twistd|g"\
-        -e "s|{{ PROJECT_DIR }}/logs|/sio2/logs|g"\
-        -e "s|command=filetracker-server|command=/home/oioioi/.local/bin/filetracker-server|g"\
-        supervisord.conf && \
-    mkdir -p /sio2/logs/{supervisor,runserver}
+    cp /sio2/oioioi/oioioi/selenium_settings.py selenium_settings.py && \
+    mkdir -p /sio2/deployment/logs/{supervisor,runserver}
 
 # Download sandboxes
-RUN ./manage.py supervisor > /dev/null --daemonize && \
+RUN ./manage.py supervisor > /dev/null --daemonize --nolaunch=uwsgi && \
     ./manage.py download_sandboxes -q -y -c /sio2/sandboxes && \
     ./manage.py supervisor stop all
-
-RUN sed -i -e "s|FILETRACKER_URL = '.*'|FILETRACKER_URL = 'http://web:9999'|g" settings.py && \
-    cp settings.py test_settings.py && \
-    sed -i -e "s/from oioioi.default_settings import \*/from oioioi.test_settings import \*\n\
-from oioioi import default_settings\nAVAILABLE_COMPILERS = default_settings.AVAILABLE_COMPILERS\n\
-DEFAULT_COMPILERS = default_settings.DEFAULT_COMPILERS\n/g" test_settings.py && \
-    cp test_settings.py selenium_settings.py && \
-    sed -i -e "s/from oioioi.test_settings/from oioioi.selenium_settings/g" selenium_settings.py

INSTALL.rst

@@ -68,15 +68,11 @@ with the normal user privileges. **This is not a safe configuration and the
 judging will run quite slowly.** It is to easily make OIOIOI up and running for
 testing purposes.
 
-Run the Django web server in one terminal::
-
-  ./manage.py runserver 0.0.0.0:8000
-
-and in the other the evaluation daemons::
+Run the Django web server and the evaluation daemons::
 
   ./manage.py supervisor
 
-The *supervisor* process monitors all processes needed by OIOIOI, except the
+The *supervisor* process monitors all processes needed by OIOIOI, including the
 web server. It has `many nice features`_.
 
 You can create an administrator account by running::
@@ -126,6 +122,9 @@ Production configuration
 
    * uwsgi (*pip install uwsgi*)
 
+#. Turn on UWSGI by setting *SERVER* to *uwsgi* in *settings.py* and restart
+   the supervisor.
+
 #. Install and configure web server. We recommend using nginx with uwsgi plugin
    (included in *nginx-full* Ubuntu package). An example configuration is
    automatically created as *nginx-site.conf*. Have a look there. What you
@@ -135,8 +134,6 @@ Production configuration
      ln -s ../sites-available/oioioi /etc/nginx/sites-enabled/
      service nginx reload
 
-   Once this is done, you no more need to run *manage.py runserver*.
-
    If you prefer deploying with Apache, an example configuration is created
    as *apache-site.conf*. You would need to install *apache2* and
    *libapache2-mod-uwsgi* packages.

README.rst

@@ -8,43 +8,14 @@ main component — the web interface.
 Installation
 ------------
 
-Vagrant (for development)
-~~~~~~~~~~~~~~~~~~~~~~~~~
-
-You can easily start development and run oioioi out of the box with `vagrant`_.
-Just enter the directory where Vagrantfile and this README are placed, and type::
-
-  vagrant up
-
-It will create an instance of virtual machine with web server and judges running.
-
-You can specify configuration in `vagrant.yml` (if you don't have such file,
-create it in the same directory as Vagrantfile).
-Supported configuration options (with example)::
-
-  port: 8001  # run oioioi on port 8001 instead of the default 8000
-  runserver_cmd: runserver_plus  # use manage.py runserver_plus instead of manage.py runserver
-
-.. _vagrant: https://www.vagrantup.com/docs/
-
-Don't forget to create the superuser. In order to do so,
-you should login into virtual machine created by Vagrant (default password is "vagrant")::
-
-  ssh 127.0.0.1 -p 2222 -l vagrant
-
-and run::
-
-  cd deployment
-  python manage.py createsuperuser
-
 Docker (for deployment)
 ~~~~~~~~~~~~~~~~~~~~~~~
 
 Additionally, there are available docker files to create images containing our services. We do not recommend this method of running OIOIOI. Please inspect Docker files and startup scripts before using in production.
 
 To run the infrastructure simply::
 
-  docker-compose up
+  "OIOIOI_CONFIGDIR=<config directory>" "OIOIOI_VERSION=<oioioi_version>" docker-compose up
 
 Make sure to change default superuser password. To do that:
    1. Login to the superuser with default credentials (username:admin, password:admin)
@@ -54,42 +25,11 @@ Make sure to change default superuser password. To do that:
 
 To start additional number of workers::
 
-  docker-compose scale worker=<number>
+  "OIOIOI_CONFIGDIR=<config directory>" "OIOIOI_VERSION=<oioioi_version>" docker-compose up --scale worker=<number>
 
 as described `in Docker docs`_.
 
-.. _in Docker docs: https://docs.docker.com/compose/reference/scale/
-
-Docker (for development)
-~~~~~~~~~~~~~~~~~~~~~~~
-
-It is possible to develop using docker images, but this we do not recommend it.
-Better use Vagrant or install OIOIOI manually, as described in the next section.
-
-Working directory should be the repository root.
-
-First prepare the image with::
-
-    OIOIOI_UID=$(id -u) docker-compose -f docker-compose-dev.yml build
-
-Create config files and logs folder on host::
-
-    id=$(docker create oioioi-dev)  #Create oioioi container
-    docker cp $id:/sio2/logs logs  #Copy initial logs folder from oioioi container
-    docker cp $id:/sio2/deployment deployment  #Copy initial deployment config from oioioi contanier
-    docker rm -v $id  #Remove unneeded container
-
-Then you can start oioioi with::
-
-    OIOIOI_UID=$(id -u) docker-compose -f docker-compose-dev.yml up
-
-to start the infrastructure in development mode. Current dirrectory with source
-code will be binded to /sio2/oioioi/ inside running container, and logs from
-services will be availible outside of the container in ./logs/.
-
-In both cases, oioioi web interface will be availible at localhost:8000, and the user
-admin with password admin will be created. If you are using docker installation
-in production encvironment remember to change the password.
+.. _in Docker docs: https://docs.docker.com/compose/reference/up/
 
 Manual installation
 ~~~~~~~~~~~~~~~~~~~

UPGRADING.rst

@@ -865,4 +865,22 @@ List of changes since the *CONFIG_VERSION* numbering was introduced:
         -
          [program:ipauthsyncd]
          command={{ PYTHON }} {{ PROJECT_DIR }}/manage.py ipauthsyncd
-         startretries=0
+         startretries=0
+
+#. * Changed the 'UWSGI_ENABLED' setting to a more general 'SERVER' setting.
+     To make sure that your typical production setup (UWSGI + reverse proxy)
+     keeps working, set this to 'uwsgi'.::
+
+        @@ -28,12 +27,20 @@ SITE_NAME = 'OIOIOI'
+         # including but not limited to the mail notifications.
+         PUBLIC_ROOT_URL = 'http://localhost'
+
+        -# Run uwsgi daemon. Shall be True, False or 'auto'.
+        -# 'auto' means daemon will be run iff DEBUG is disabled.
+        -UWSGI_ENABLED = 'auto'
+        +# The server to be run. Options are:
+        +# django - django's http server
+        +# uwsgi - uwsgi daemon
+        +# uwsgi-http - uwsgi deamon with builtin http server
+        +# None - nothing will be run
+        +SERVER = None