diff --git a/Dockerfile b/Dockerfile index 373e8e637..7323f9246 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,15 +1,55 @@ -FROM python:3.7 +# This is oioioi user linux uid. Setting it is useful in development. +# By default we use an unused uid of 1234. +# This is placed here to avoid redownloading package on uid change +ARG OIOIOI_UID=1234 + +FROM python:3.7 AS build ENV PYTHONUNBUFFERED 1 -RUN dpkg --add-architecture i386 && \ - apt-get update && \ - apt-get install -y \ +RUN apt-get update \ + && apt-get install --no-install-recommends -y \ + build-essential \ git \ libpq-dev \ + libdb5.3-dev + +ARG OIOIOI_UID + +RUN useradd -u ${OIOIOI_UID} -m oioioi \ + && mkdir -p /sio2/oioioi \ + && chown -R oioioi:oioioi /sio2 + +USER oioioi +WORKDIR /sio2 + +RUN pip install --user virtualenv \ + && /home/oioioi/.local/bin/virtualenv -p python3.7 venv + +RUN . /sio2/venv/bin/activate \ + && pip install psycopg2-binary==2.8.6 twisted librabbitmq uwsgi + +COPY --chown=oioioi:oioioi setup.py requirements.txt /sio2/oioioi/ + +WORKDIR /sio2/oioioi + +RUN . /sio2/venv/bin/activate \ + && pip install -r requirements.txt + +COPY --chown=oioioi:oioioi . /sio2/oioioi + +FROM python:3.7 AS base + +ENV PYTHONUNBUFFERED 1 + +RUN dpkg --add-architecture i386 \ + && apt-get update \ + && apt-get install -y \ + sudo \ + wget \ + locales \ + libdb5.3 \ postgresql-client \ - libdb-dev \ - fp-compiler fp-units-base fp-units-math \ texlive-latex-base \ texlive-lang-polish \ texlive-latex-extra \ @@ -17,83 +57,74 @@ RUN dpkg --add-architecture i386 && \ texlive-lang-european \ texlive-lang-czechslovak \ texlive-pstricks \ - ghostscript \ texlive-fonts-recommended \ - gcc-multilib \ - sudo \ + ghostscript \ + flite \ + sox \ libstdc++6:i386 \ zlib1g:i386 \ - locales \ - python3-pip && \ - apt-get clean + && rm -rf /var/lib/apt/lists/* -# This is oioioi user linux uid. Setting it is useful in development. -# By default we use an unused uid of 1234. -# This is placed here to avoid redownloading package on uid change -ARG oioioi_uid=1234 +ARG OIOIOI_UID -#Bash as shell, setup folders, create oioioi user -RUN rm /bin/sh && ln -s /bin/bash /bin/sh && \ - mkdir -pv /sio2/oioioi && \ - mkdir -pv /sio2/sandboxes && \ - useradd -U oioioi -m -d /home/oioioi/ -u $oioioi_uid && \ - echo "oioioi ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers && \ - chown -R oioioi:oioioi /sio2 +RUN useradd -u ${OIOIOI_UID} -m oioioi \ + && mkdir -p /sio2/oioioi \ + && chown -R oioioi:oioioi /sio2 \ + && echo "oioioi ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers # Modify locale -RUN sed -i -e "s/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/" /etc/locale.gen && \ - locale-gen +RUN sed -i -e "s/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/" /etc/locale.gen \ + && locale-gen -COPY ./entrypoint_checks.sh /entrypoint_checks.sh -RUN chmod +x /entrypoint_checks.sh && chown oioioi /entrypoint_checks.sh - -# Installing python dependencies USER oioioi -ENV PATH $PATH:/home/oioioi/.local/bin/ - -RUN pip3 install --user psycopg2-binary==2.8.6 twisted uwsgi +COPY --from=build --chown=oioioi:oioioi /sio2 /sio2 -WORKDIR /sio2/oioioi +WORKDIR /sio2 -COPY --chown=oioioi:oioioi setup.py requirements.txt ./ -RUN pip3 install -r requirements.txt --user -COPY --chown=oioioi:oioioi requirements_static.txt ./ -RUN pip3 install -r requirements_static.txt --user - -COPY --chown=oioioi:oioioi . /sio2/oioioi - -RUN oioioi-create-config /sio2/deployment +RUN . /sio2/venv/bin/activate \ + && oioioi-create-config --docker deployment WORKDIR /sio2/deployment RUN sed -i -e \ - "s/SERVER = 'django'/SERVER = 'uwsgi-http'/g;\ - s/DEBUG = True/DEBUG = False/g;\ - s/django.db.backends./django.db.backends.postgresql/g;\ - s/'NAME': ''/'NAME': 'oioioi'/g;\ - s/'USER': ''/'USER': 'oioioi'/g;\ - s/'HOST': '',/'HOST': 'db',/g;\ - s/'PASSWORD': ''/'PASSWORD': 'password'/g;\ - s/#BROKER_URL/BROKER_URL/g;\ - s/#FILETRACKER_LISTEN_ADDR/FILETRACKER_LISTEN_ADDR/g;\ + "s/#FILETRACKER_LISTEN_ADDR/FILETRACKER_LISTEN_ADDR/g;\ s/#FILETRACKER_LISTEN_PORT/FILETRACKER_LISTEN_PORT/g;\ - s|#FILETRACKER_URL = '.*'|FILETRACKER_URL = 'http://web:9999'|g;\ s/#RUN_SIOWORKERSD$/RUN_SIOWORKERSD/g;\ - s/#SIOWORKERS_LISTEN_ADDR/SIOWORKERS_LISTEN_ADDR/g;\ - s/#SIOWORKERS_LISTEN_PORT/SIOWORKERS_LISTEN_PORT/g;\ - s/RUN_LOCAL_WORKERS = True/RUN_LOCAL_WORKERS = False/g;\ s/AVAILABLE_COMPILERS = SYSTEM_COMPILERS/#AVAILABLE_COMPILERS = SYSTEM_COMPILERS/g;\ s/DEFAULT_COMPILERS = SYSTEM_DEFAULT_COMPILERS/#DEFAULT_COMPILERS = SYSTEM_DEFAULT_COMPILERS/g;\ - s/USE_UNSAFE_EXEC = True/USE_UNSAFE_EXEC = True/g;\ - s/#DEFAULT_SAFE_EXECUTION_MODE/#DEFAULT_SAFE_EXECUTION_MODE/g;\ - s/#USE_UNSAFE_CHECKER = True/#USE_UNSAFE_CHECKER = False/g;\ - \$afrom basic_settings import *\nALLOWED_HOSTS = ALLOWED_HOSTS + \\['oioioi', '127.0.0.1', 'localhost', 'web'\\]" \ + \$afrom basic_settings import *" \ settings.py && \ - cp /sio2/oioioi/oioioi/selenium_settings.py selenium_settings.py && \ mkdir -p /sio2/deployment/logs/{supervisor,runserver} -# Download sandboxes -RUN ./manage.py supervisor > /dev/null --daemonize --nolaunch=uwsgi && \ - ./manage.py download_sandboxes -q -y -c /sio2/sandboxes && \ - ./manage.py supervisor stop all +FROM base AS development + +USER root + +RUN apt-get update \ + && apt-get install -y \ + fp-compiler \ + fp-units-base \ + fp-units-math \ + gcc-multilib \ + && rm -rf /var/lib/apt/lists/* + +USER oioioi + +WORKDIR /sio2/oioioi + +COPY --chown=oioioi:oioioi requirements_static.txt /sio2/oioioi +RUN . /sio2/venv/bin/activate \ + && pip install -r requirements_static.txt + +RUN cp /sio2/oioioi/oioioi/selenium_settings.py /sio2/deployment/selenium_settings.py + +WORKDIR /sio2/deployment + +ENTRYPOINT [ "/sio2/oioioi/scripts/docker-entrypoint.sh" ] + +FROM base as production + +WORKDIR /sio2/deployment + +ENTRYPOINT [ "/sio2/oioioi/scripts/docker-entrypoint.sh" ] diff --git a/GUIDE.md b/GUIDE.md index 5394ddc19..ac9088d80 100644 --- a/GUIDE.md +++ b/GUIDE.md @@ -26,15 +26,15 @@ newgrp docker ``` All the commands below are being run in `oioioi` directory (main directory of the repository). -In order to use `easy_toolbox.py` alternative method, check python package requirements in `easy_toolbox.py`. +In order to use `easy_toolbox.py` alternative method, check python package requirements in `easy_toolbox.py`. -To build OIOIOI image run +To build OIOIOI image run ```bash OIOIOI_UID=$(id -u) docker-compose -f docker-compose-dev.yml -f extra/docker/docker-compose-dev-noserver.yml build ``` or ```bash -./easy_toolbox.py build +./easy_toolbox.py build ``` Set your containers up and running @@ -50,7 +50,7 @@ Wait some time for the migration to finish (no more than a few minutes). Run your web service ```bash -OIOIOI_UID=$(id -u) docker-compose -f docker-compose-dev.yml -f extra/docker/docker-compose-dev-noserver.yml exec web python3 manage.py runserver 0.0.0.0:8000 +OIOIOI_UID=$(id -u) docker-compose -f docker-compose-dev.yml -f extra/docker/docker-compose-dev-noserver.yml exec web ./manage.py runserver 0.0.0.0:8000 ``` or ```bash @@ -59,6 +59,23 @@ or Now visit `localhost:8000` and start exploring OIOIOI. +### Download sandboxes + +To properly judge submissions in an isolated environment, you need to download the execution sandboxes. +This is necessary if you're going to work on any part of the judging process. +To download them, run: + +```bash +./easy_toolbox.py download-sandboxes +``` +or +```bash +OIOIOI_UID=$(id -u) docker-compose -f docker-compose-dev.yml -f extra/docker/docker-compose-dev-noserver.yml exec web ./manage.py download_sandboxes +``` +and follow the prompts. + +This needs to be done only one time, unless you remove the filetracker database that stores them. + ## Run unit tests In order to run unit tests Docker installation is required. To do it just run @@ -75,9 +92,10 @@ In order to run Cypress tests a few more steps are required. - Clear the database - Create superuser (admin, admin) - Run the service +- Ensure that task execution sandboxes are downloaded (it is recommended to do this beforehand, see section [**Download sandboxes**](#download-sandboxes) above) - Run the tests -All of these steps can be done by running +All of these steps (aside from downloading sandboxes) can be done by running ```bash ./easy_toolbox.py flush-db && ./easy_toolbox.py add-superuser && ./easy_toolbox.py cypress-apply-settings && ./easy_toolbox.py run ``` @@ -103,12 +121,12 @@ As mentioned above GitHub Actions replaced most of the Hudson jobs. Right now th - Transifex translations uploader ## Deployment -No changes can be seen on `szkopul.edu.pl` without deployment. +No changes can be seen on `szkopul.edu.pl` without deployment. @twalen is responsible for conducting the deployments, and they are usually done during maintenance window (see `szkopul.edu.pl` main page for details). -During deployment translation files are being uploaded and downloaded. The translation manager is -[Transifex](https://www.transifex.com/sio2project/sio2project/dashboard/). When you add localized text +During deployment translation files are being uploaded and downloaded. The translation manager is +[Transifex](https://www.transifex.com/sio2project/sio2project/dashboard/). When you add localized text to OIOIOI it is uploaded to Transifex with GitHub Action and later, when translated, downloaded, compiled and saved to the codebase. (Both of these jobs need to be run manually). @@ -119,20 +137,20 @@ Ticketing was moved to GitHub Issues. ## FAQ ### When to use `./easy_toolbox.py build`? -Essentially, it is not easy to answer. +Essentially, it is not easy to answer. Firstly, let's understand what each of the commands does. -As you may already know, SIO2 development environment works in containerized infrastructure. -We have four containers up and running, required for SIO2 to work properly. -Database (`db`), RabbitMQ (`broker`), Worker (`worker`) and OIOIOI (`web`). -Postgres and RabbitMQ already have existing docker images on Docker Hub. -We only need to build (here comes the magic word) SIO dependent images. -`Dockerfile` defines what steps are to be done in order to create the environment. -Once you built the image, you can set the container up. -Remember - things like dependencies (`requirements[_static].txt`, `setup.py`) are downloaded during the built, -so if you changed something in those places you either need to build the image again, -or apply these changes by hand (e.g. do `pip install`). -If you have good internet connection and adequate CPU, it shouldn't be hard to build the image again, -especially that it is more stable approach. +As you may already know, SIO2 development environment works in containerized infrastructure. +We have four containers up and running, required for SIO2 to work properly. +Database (`db`), RabbitMQ (`broker`), Worker (`worker`) and OIOIOI (`web`). +Postgres and RabbitMQ already have existing docker images on Docker Hub. +We only need to build (here comes the magic word) SIO dependent images. +`Dockerfile` defines what steps are to be done in order to create the environment. +Once you built the image, you can set the container up. +Remember - things like dependencies (`requirements[_static].txt`, `setup.py`) are downloaded during the built, +so if you changed something in those places you either need to build the image again, +or apply these changes by hand (e.g. do `pip install`). +If you have good internet connection and adequate CPU, it shouldn't be hard to build the image again, +especially that it is more stable approach. ### When to use `./easy_toolbox.py up`? Whenever you don't have the containers up. @@ -151,8 +169,8 @@ If not, check the output of `docker ps -a` and `docker logs `. ### When to use `./easy_toolbox.py run`? Whenever you want the Django web service to be running. In development environment we use dev server, so it should catch all changes in the code. -You can have the server running all the time - just make sure, -that Django discovered your changes +You can have the server running all the time - just make sure, +that Django discovered your changes (the server will restart with appropriate message like "detected changes in file `xyz.py`"). ### What to do when I get permission denied error? diff --git a/README.rst b/README.rst index 292454c27..d7ed43dff 100644 --- a/README.rst +++ b/README.rst @@ -52,8 +52,46 @@ as described `in Docker docs`_. .. _in Docker docs: https://docs.docker.com/compose/reference/up/ +Docker image +============ + +Additionally, there is a Docker image provided at (TODO: update this with a link to the docker image, wherever it is hosted).:: + + docker run --rm + --network=sio2-network \ + --name=oioioi-web \ + -e "DATABASE_HOST=db" \ + -e "DATABASE_PORT=5432" \ + -e "DATABASE_NAME=oioioi" \ + -e "DATABASE_USER=oioioi" \ + -e "DATABASE_PASSWORD=password" \ + -e "RABBITMQ_HOST=broker" \ + -e "RABBITMQ_PORT=5672" \ + -e "RABBITMQ_USER=oioioi" \ + -e "RABBITMQ_PASSWORD=password" \ + -e "FILETRACKER_URL=http://web:9999" \ + -e "SIOWORKERS_LISTEN_URL=http://web:7890" \ + -e "ENABLE_SANDBOXED_COMPILERS=True" \ + -p 8000:8000 \ + + +The image is configured through various environment variables: + +* database configuration (`DATABASE_...`) is required for the image to work. +* the RabbitMQ broker (`RABBITMQ_...`) is optional, although running it is recommended in production workloads. +* the `FILETRACKER_URL` variable specifies the URL at which the Filetracker server is available to workers (and the OIOIOI instance, if Filetracker is on a separate host). +* the `SIOWORKERS_LISTEN_URL` variable specifies the URL at which the `sioworkersd` worker daemon is available. Usually this is the same machine the main OIOIOI instance is hosted at. +* the `ENABLE_SANDBOXED_COMPILERS` variable instructs the workers to use sanboxed compilers. Those compilers need to be installed manually before enabling this option using the `./manage.py download_sandboxes` command inside the container in the `/sio2/deployment` directory. + +Additionally, there are several directories inside the deployment directory that store persistent data, which should be mounted to an external volume: + +* `/sio2/deployment/media`, which stores the Filetracker server database, including the sandboxes and user submissions. +* `/sio2/deployment/logs`, which stores the logs generated by all the services. +* `/sio2/deployment/basic_setitngs.py`, which is a file that allows to override the default settings provided by OIOIOI. + + Docker (for development) -~~~~~~~~~~~~~~~~~~~~~~~ +~~~~~~~~~~~~~~~~~~~~~~~~ Make sure you installed docker properly. The easiest way to do this:: @@ -95,6 +133,27 @@ Additionally you can bind config files and logs folder to the host:: Remember to also uncomment the appropriate volume binding in the web service description in the docker-compose-dev.yml. +Setting up sandboxes and compilers +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +In production environments, OIOIOI relies on prebuilt sandboxes and compilers, which standardize and isolate the +enviroment the user submissions are judged in. They can be downloaded after the basic OIOIOI installation is set up, +by running the `./manage.py download_sandboxes` command in the deployment directory. + +Installing sandboxes without an internet connection +=================================================== + +If the machine you're installing OIOIOI on doesn't have an internet connection, you can download the sandboxes beforehand, +and instruct the `download_sandboxes` command to load the files from a local cache instead of the remote server::: + + # if in a docker container, this directory can be bind mounted from the host system + ./manage.py download_sandboxes -m file:///sio2/sandboxes/MANIFEST -c /sio2/sandboxes # example path, can be something else + +All the sandboxes are available at https://downloads.sio2project.mimuw.edu.pl/sandboxes/, and the list of sandboxes +downloaded by default using the command is available at https://downloads.sio2project.mimuw.edu.pl/sandboxes/MANIFEST +Those archives need to be placed directly in the directory that is later passed to the `-c` flag of the command, alongside +the `MANIFEST` and `LICENSE` files. + Running tests on Docker ~~~~~~~~~~~~~~~~~~~~~~~ @@ -105,7 +164,7 @@ you are connected to after using docker exec -it “web” /bin/bash. The defaul docker-compose -f docker-compose-dev.yml -f extra/docker/docker-compose-dev-noserver.yml exec "web" ../oioioi/test.sh oioioi/{name_of_the_app}/ Running static code analysis tools locally (requires Docker) -~~~~~~~~~~~~~~~~~~~~~~~ +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ The static code analysis tools currently in use for python code are black, isort, pep8 and pylint. All of them can be run locally using the `run_static.sh` shell script. @@ -127,7 +186,8 @@ To run one of the tools:: ./run_static.sh pep8 Script toolbox for Docker (development) -~~~~~~~~~~~~~~~~~~~~~~~~~ +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + Copy-pasting all Docker commands from GitHub can be tedious. Instead use a set of pre-prepared commands embedded into `easy_toolbox.py`. For help run `easy_toolbox.py -h`. Add custom commands by editing `RAW_COMMANDS` in the file. Script can be used with user-friendly CLI or by passing commands as arguments. @@ -141,7 +201,7 @@ Developer environment can be easily set up by running:: For system requirements check `easy_toolbox.py`. Manual installation (deprecated) -~~~~~~~~~~~~~~~~~~~ +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ See `INSTALL`_ for instructions. diff --git a/db_init.sh b/db_init.sh deleted file mode 100755 index e3b4b633f..000000000 --- a/db_init.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash -set -e -set -x - -/sio2/oioioi/wait-for-it.sh -t 60 "db:5432" - -./manage.py migrate 2>&1 | tee /sio2/deployment/logs/migrate.log - -if [[ -v CREATE_SUPERUSER ]]; then - ./manage.py loaddata ../oioioi/oioioi_selenium/data.json; -fi \ No newline at end of file diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index c2cfdc314..5ca2b6576 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -1,5 +1,4 @@ -version: '2.4' -# https://github.com/docker/cli/issues/1293 +version: '3.8' services: db: image: library/postgres:12.2 @@ -12,31 +11,43 @@ services: stop_grace_period: 1m web: image: sio2project/oioioi-dev - command: ["/sio2/oioioi/selenium_init.sh"] + command: ["/sio2/oioioi/scripts/oioioi-init.sh"] build: context: . dockerfile: Dockerfile + target: development args: - - "oioioi_uid=${OIOIOI_UID}" - extra_hosts: - - "web:127.0.0.1" + - "OIOIOI_UID=${OIOIOI_UID}" + environment: + DEBUG: "True" + DATABASE_HOST: "db" + DATABASE_PORT: "5432" + DATABASE_NAME: "oioioi" + DATABASE_USER: "oioioi" + DATABASE_PASSWORD: "password" + RABBITMQ_HOST: "broker" + RABBITMQ_PORT: "5672" + RABBITMQ_USER: "oioioi" + RABBITMQ_PASSWORD: "oioioi" + FILETRACKER_URL: "http://web:9999" + SIOWORKERS_LISTEN_URL: "http://web:7890" + ENABLE_SANDBOXED_COMPILERS: "True" + USE_UNSAFE_CHECKER: "True" ports: - "8000:8000" volumes: - .:/sio2/oioioi -# - ./deployment:/sio2/deployment + - oioioi-media-dev:/sio2/deployment/media stop_grace_period: 3m depends_on: - db - broker worker: - image: sio2project/oioioi-dev - command: ["/sio2/oioioi/worker_init.sh"] - volumes: - - .:/sio2/oioioi + image: ghcr.io/sio2project/sioworkers:latest stop_grace_period: 1m cap_add: - ALL + privileged: true depends_on: - db broker: @@ -47,3 +58,4 @@ services: stop_grace_period: 1m volumes: postgress-data-dev: + oioioi-media-dev: diff --git a/docker-compose-selenium.yml b/docker-compose-selenium.yml index 5d2d780d7..f5e3d4610 100644 --- a/docker-compose-selenium.yml +++ b/docker-compose-selenium.yml @@ -1,45 +1,13 @@ -version: '2.4' -# https://github.com/docker/cli/issues/1293 +version: '3.8' services: db: - image: library/postgres:12.2 - environment: - POSTGRES_USER: "oioioi" - POSTGRES_PASSWORD: "password" - POSTGRES_DB: "oioioi" - stop_grace_period: 1m + volumes: [] web: - image: sio2project/oioioi-selenium - command: ["/sio2/oioioi/selenium_init.sh"] - build: - context: . - dockerfile: Dockerfile - extra_hosts: - - "web:127.0.0.1" ports: - "8001:8000" environment: DJANGO_SETTINGS_MODULE: "selenium_settings" - stop_grace_period: 3m - depends_on: - - db - - broker - worker: - image: sio2project/oioioi-selenium - command: ["/sio2/oioioi/worker_init.sh"] - cap_add: - - ALL - environment: - DJANGO_SETTINGS_MODULE: "selenium_settings" - stop_grace_period: 1m - depends_on: - - db - broker: - image: library/rabbitmq:3.8 - environment: - RABBITMQ_DEFAULT_USER: oioioi - RABBITMQ_DEFAULT_PASS: oioioi - stop_grace_period: 1m + volumes: [] selenium-hub: image: selenium/hub:3.141.59-20200409 ports: @@ -52,4 +20,4 @@ services: volumes: - /dev/shm:/dev/shm depends_on: - - selenium-hub \ No newline at end of file + - selenium-hub diff --git a/docker-compose.yml b/docker-compose.yml index f7e322c14..af3cb4198 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,4 +1,4 @@ -version: '3' +version: '3.8' services: db: image: library/postgres:12.2 @@ -11,21 +11,32 @@ services: stop_grace_period: 1m web: image: sio2project/oioioi:$OIOIOI_VERSION - command: ["/sio2/oioioi/oioioi_init.sh"] + command: ["/sio2/oioioi/scripts/oioioi-init.sh"] + environment: + DATABASE_HOST: "db" + DATABASE_PORT: "5432" + DATABASE_NAME: "oioioi" + DATABASE_USER: "oioioi" + DATABASE_PASSWORD: "password" + FILETRACKER_URL: "http://web:9999" + SIOWORKERS_LISTEN_URL: "http://web:7890" + ENABLE_SANDBOXED_COMPILERS: "True" ports: - "8000:8000" volumes: - $OIOIOI_CONFIGDIR/basic_settings.py:/sio2/deployment/basic_settings.py + - oioioi_media:/sio2/deployment/media + - oioioi_logs:/sio2/deployment/logs stop_grace_period: 3m depends_on: - db - broker worker: - image: sio2project/oioioi:$OIOIOI_VERSION - command: ["/sio2/oioioi/worker_init.sh"] + image: ghcr.io/sio2project/sioworkers:latest stop_grace_period: 1m cap_add: - ALL + privileged: true depends_on: - db broker: @@ -36,3 +47,5 @@ services: stop_grace_period: 1m volumes: postgress-data: + oioioi-media: + oioioi-logs: diff --git a/easy_toolbox.py b/easy_toolbox.py index fea430646..45ed5715b 100755 --- a/easy_toolbox.py +++ b/easy_toolbox.py @@ -3,7 +3,7 @@ # pip requirements: # python ^3.6 # inquirer (only for GUI) -# +# # system: # docker # docker-compose @@ -27,19 +27,20 @@ ("build", "Build OIOIOI container from source.", "build", True), ("up", "Run all SIO2 containers", "up -d"), ("down", "Stop all SIO2 containers", "down", True), - ("run", "Run server", "{exec} web python3 manage.py runserver 0.0.0.0:8000"), + ("run", "Run server", "{exec} web ./manage.py runserver 0.0.0.0:8000"), + ("download-sandboxes", "Download sandboxes", "{exec} web ./manage.py download_sandboxes"), ("bash", "Open command prompt on web container.", "{exec} web bash"), ("bash-db", "Open command prompt on database container.", "{exec} db bash"), # This one CLEARS the database. Use wisely. - ("flush-db", "Clear database.", "{exec} web python manage.py flush --noinput", True), + ("flush-db", "Clear database.", "{exec} web ./manage.py flush --noinput", True), ("add-superuser", "Create admin_admin.", - "{exec} web python manage.py loaddata ../oioioi/oioioi_cypress/cypress/fixtures/admin_admin.json"), - ("test", "Run unit tests.", "{exec} web ../oioioi/test.sh"), - ("test-slow", "Run unit tests. (--runslow)", "{exec} web ../oioioi/test.sh --runslow"), + "{exec} web ./manage.py loaddata ../oioioi/oioioi/admin_admin_fixture.json"), + ("test", "Run unit tests.", "run web ../oioioi/test.sh"), + ("test-slow", "Run unit tests. (--runslow)", "run web ../oioioi/test.sh --runslow"), ("test-abc", "Run specific test file. (edit the toolbox)", - "{exec} web ../oioioi/test.sh -v oioioi/problems/tests/test_task_archive.py"), + "run web ../oioioi/test.sh -v oioioi/problems/tests/test_task_archive.py"), ("test-coverage", "Run coverage tests.", - "{exec} 'web' ../oioioi/test.sh oioioi/problems --cov-report term --cov-report xml:coverage.xml --cov=oioioi"), + "run 'web' ../oioioi/test.sh oioioi/problems --cov-report term --cov-report xml:coverage.xml --cov=oioioi"), ("cypress-apply-settings", "Apply settings for CyPress.", "{exec} web bash -c \"echo CAPTCHA_TEST_MODE=True >> settings.py\""), ] diff --git a/extra/docker/docker-compose-dev-noserver.yml b/extra/docker/docker-compose-dev-noserver.yml index f0bc48fcd..12b05477e 100644 --- a/extra/docker/docker-compose-dev-noserver.yml +++ b/extra/docker/docker-compose-dev-noserver.yml @@ -1,7 +1,5 @@ -version: '2.4' -# https://github.com/docker/cli/issues/1293 +version: '3.8' services: web: volumes: - ./extra/sample-configs/basic_settings_noserver.py:/sio2/deployment/basic_settings.py - diff --git a/extra/easy-install/oioioi.sh b/extra/easy-install/oioioi.sh index d6208997b..c4eb86525 100755 --- a/extra/easy-install/oioioi.sh +++ b/extra/easy-install/oioioi.sh @@ -146,7 +146,8 @@ case "$1" in echot "Downloading images..." docker_compose_fun pull echot "Initializing the database..." - docker_compose_fun run --rm -e CREATE_SUPERUSER=1 web /sio2/oioioi/db_init.sh + docker_compose_fun run --rm web ./manage.py migrate + docker_compose_fun run --rm web ./manage.py loaddata /sio2/oioioi/oioioi/admin_admin_fixture.json docker_compose_fun stop echot "Your admin password is \"admin\". Don't forget to change it!" ;; @@ -206,7 +207,7 @@ case "$1" in echot "Redownloading images..." docker_compose_fun pull echot "Reinitializing the database..." - docker_compose_fun run --rm web /sio2/oioioi/db_init.sh + docker_compose_fun run --rm web ./manage.py migrate docker_compose_fun stop echot "Updated from $OIOIOI_OLD_VERSION to $OIOIOI_VERSION." ;; @@ -215,4 +216,3 @@ case "$1" in echot "Invalid command!" ;; esac - diff --git a/oioioi_cypress/cypress/fixtures/admin_admin.json b/oioioi/admin_admin_fixture.json similarity index 100% rename from oioioi_cypress/cypress/fixtures/admin_admin.json rename to oioioi/admin_admin_fixture.json diff --git a/oioioi/deployment/create_config.py b/oioioi/deployment/create_config.py index c48dd37a4..232e688df 100644 --- a/oioioi/deployment/create_config.py +++ b/oioioi/deployment/create_config.py @@ -52,7 +52,7 @@ def generate_from_template(dir, filename, context, mode=None): os.chmod(dest, mode) -def generate_all(dir, verbose): +def generate_all(dir, verbose, docker=False): generate_from_template(dir, 'basic_settings.py', {}) generate_from_template( @@ -83,6 +83,9 @@ def generate_all(dir, verbose): virtual_env = os.environ.get('VIRTUAL_ENV', '') user = pwd.getpwuid(os.getuid())[0] + if docker: + generate_from_template(dir, 'docker_settings.py', {}) + manage_py = os.path.join(dir, 'manage.py') generate_from_template( dir, @@ -158,6 +161,12 @@ def main(): usage = "%(prog)s [options] dir" parser = ArgumentParser(usage=usage) parser.add_argument('-v', '--verbose', action='store_true', dest='verbose') + parser.add_argument( + '--docker', + action='store_true', + dest='docker', + help='generate docker container configuration' + ) parser.add_argument("dir", help="deployment folder to create") args = parser.parse_args() @@ -169,7 +178,7 @@ def main(): os.makedirs(absolute_dir) try: - generate_all(absolute_dir, args.verbose) + generate_all(absolute_dir, args.verbose, args.docker) except BaseException: shutil.rmtree(absolute_dir) raise diff --git a/oioioi/deployment/docker_settings.py.template b/oioioi/deployment/docker_settings.py.template new file mode 100644 index 000000000..0c2899edf --- /dev/null +++ b/oioioi/deployment/docker_settings.py.template @@ -0,0 +1,66 @@ +import os +from typing import Optional + +def _parse_bool(value: str) -> bool: + return value == "True" + + +DEBUG = _parse_bool(os.getenv("DEBUG", "False")) + +SERVER = "django" if DEBUG else "uwsgi-http" + +if DEBUG: + try: + from settings import ALLOWED_HOSTS as BASE_ALLOWED_HOSTS + except ImportError: + BASE_ALLOWED_HOSTS = [] + + ALLOWED_HOSTS = BASE_ALLOWED_HOSTS + ['oioioi', '127.0.0.1', 'localhost', 'web'] + +DATABASES = { + "default": { + "ENGINE": "django.db.backends.postgresql", + "NAME": os.getenv("DATABASE_NAME"), + "USER": os.getenv("DATABASE_USER"), + "PASSWORD": os.getenv("DATABASE_PASSWORD"), + "HOST": os.getenv("DATABASE_HOST"), + "PORT": os.getenv("DATABASE_PORT"), + "ATOMIC_REQUESTS": True, + } +} + +# RabbitMQ connection settings +host = os.getenv("RABBITMQ_HOST", None) +port = os.getenv("RABBITMQ_PORT", None) +user = os.getenv("RABBITMQ_USER", None) +password = os.getenv("RABBITMQ_PASSWORD", None) + +if ( + host is not None + and port is not None + and user is not None + and password is not None +): + BROKER_URL = f"amqp://{user}:{password}@{host}:{port}" + +del host, port, user, password + +FILETRACKER_URL = os.getenv("FILETRACKER_URL") + +SIOWORKERS_LISTEN_ADDR = "0.0.0.0" +SIOWORKERS_LISTEN_PORT = 7890 +SIOWORKERS_LISTEN_URL = os.getenv("SIOWORKERS_LISTEN_URL") + +RUN_LOCAL_WORKERS = False + +if not _parse_bool(os.getenv("ENABLE_SANDBOXED_COMPILERS", "False")): + from oioioi.default_settings import SYSTEM_COMPILERS, SYSTEM_DEFAULT_COMPILERS + + AVAILABLE_COMPILERS = SYSTEM_COMPILERS + DEFAULT_COMPILERS = SYSTEM_DEFAULT_COMPILERS + +USE_UNSAFE_EXEC = False + +DEFAULT_SAFE_EXECUTION_MODE = "sio2jail" + +USE_UNSAFE_CHECKER = os.getenv("USE_UNSAFE_CHECKER", DEBUG) diff --git a/oioioi/deployment/manage.py.template b/oioioi/deployment/manage.py.template index 2e3a328aa..5aff97587 100644 --- a/oioioi/deployment/manage.py.template +++ b/oioioi/deployment/manage.py.template @@ -6,7 +6,10 @@ if __name__ == "__main__": # Configure virtualenv, if present when running the configuration generator if '__VIRTUAL_ENV__' and 'VIRTUAL_ENV' not in os.environ: activate_this = os.path.join('__VIRTUAL_ENV__', 'bin', 'activate_this.py') - execfile(activate_this, dict(__file__=activate_this)) + exec( + compile(open(activate_this, 'rb').read(), 'activate_this', 'exec'), + dict(__file__=activate_this) + ) from oioioi.deployment.init import init_env init_env('__DIR__') diff --git a/oioioi/deployment/settings.py.template b/oioioi/deployment/settings.py.template index 0d142c34c..acc717981 100755 --- a/oioioi/deployment/settings.py.template +++ b/oioioi/deployment/settings.py.template @@ -491,3 +491,8 @@ ZEUS_INSTANCES = { # variable, then corresponding setting will be set automatically. # CAPTCHA_FLITE_PATH = '' # CAPTCHA_SOX_PATH = '' + +try: + from docker_settings import * +except ImportError: + pass diff --git a/oioioi/deployment/supervisord.conf.template b/oioioi/deployment/supervisord.conf.template index be745311b..5a22ce751 100644 --- a/oioioi/deployment/supervisord.conf.template +++ b/oioioi/deployment/supervisord.conf.template @@ -91,7 +91,7 @@ stderr_logfile={{ PROJECT_DIR }}/logs/receive_from_workers-err.log {% if settings.SIOWORKERS_BACKEND != 'oioioi.sioworkers.backends.SioworkersdBackend' %}exclude=true{% endif %} [program:sioworkersd] -command={{ PYTHON }} /home/oioioi/.local/bin//twistd -n -l- --pidfile={{ PROJECT_DIR }}/pidfiles/sioworkersd.pid sioworkersd --database={{ PROJECT_DIR }}/sioworkersd.db +command=twistd -n -l- --pidfile={{ PROJECT_DIR }}/pidfiles/sioworkersd.pid sioworkersd --database={{ PROJECT_DIR }}/sioworkersd.db startretries=0 redirect_stderr=false stdout_logfile={{ PROJECT_DIR }}/logs/sioworkersd.log diff --git a/oioioi/selenium_settings.py b/oioioi/selenium_settings.py index c4c583efb..ebedaf699 100644 --- a/oioioi/selenium_settings.py +++ b/oioioi/selenium_settings.py @@ -62,3 +62,8 @@ 'handlers': ['console'], 'level': 'INFO', } + +try: + from docker_settings import * +except ImportError: + pass diff --git a/oioioi_init.sh b/oioioi_init.sh deleted file mode 100755 index 7f7c08372..000000000 --- a/oioioi_init.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash -set -e -set -x - -/sio2/oioioi/wait-for-it.sh -t 60 "db:5432" - -exec ./manage.py supervisor --logfile=/sio2/deployment/logs/supervisor.log diff --git a/oioioi_selenium/README.rst b/oioioi_selenium/README.rst index 49bc5bbaa..2ee051e68 100644 --- a/oioioi_selenium/README.rst +++ b/oioioi_selenium/README.rst @@ -12,8 +12,8 @@ USELEFUL TIPS AND TRICKS 1. Do not run tests straight by invoking `test_selenium.sh` script. Instead build docker images without the -d flag in another terminal (just copy and paste the command from the script and remove -d) - [`docker-compose -f docker-compose-selenium.yml up`]. Why? - You can easily navigate through the logs there real-time + [`docker-compose -f docker-compose-dev.yml -f docker-compose-selenium.yml up`]. + Why? You can easily navigate through the logs there real-time (instance is visible at 8001 port). 2. Run the docker images and test from a different terminal (using pytest). There is one problem with this, though. @@ -115,4 +115,4 @@ to add it there! .. autoclass:: oioioi_selenium.__init__.OIOIOISeleniumTestCase :members: - :undoc-members: \ No newline at end of file + :undoc-members: diff --git a/requirements.txt b/requirements.txt index c6beb29ba..8d2c294d1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ # therefore they must be listed here. Moreover, they cannot be listed in # setup.py, as pip is not able to install them. http://github.com/Supervisor/supervisor/zipball/master#egg=supervisor==4.0.0.dev0 -http://github.com/badochov/djsupervisor/zipball/master#egg=djsupervisor==0.4.0 +http://github.com/sio2project/django-supervisor/zipball/master#egg=django-supervisor==0.4.0 http://github.com/sio2project/sioworkers/archive/refs/tags/v1.4.1.tar.gz -e . diff --git a/run_static.sh b/run_static.sh index 1a5b3caeb..a04de7d73 100755 --- a/run_static.sh +++ b/run_static.sh @@ -6,11 +6,11 @@ cmd="$1" export OIOIOI_UID=$(id -u) # the image has to be rebuilt for file changes to be visible -docker_run_built="docker run --rm --entrypoint=/entrypoint_checks.sh -t sio2project/oioioi-dev " +docker_run_built="docker run --rm -t sio2project/oioioi-dev /sio2/oioioi/scripts/static-checks.sh" # can be used without rebuilding the image docker_compose_alias="docker-compose -f docker-compose-dev.yml -f extra/docker/docker-compose-dev-noserver.yml " -docker_compose_exec="${docker_compose_alias} exec web /sio2/oioioi/entrypoint_checks.sh " +docker_compose_exec="${docker_compose_alias} run web /sio2/oioioi/scripts/static-checks.sh " # choose the way of running static on docker docker_run="$docker_compose_exec" diff --git a/scripts/docker-entrypoint.sh b/scripts/docker-entrypoint.sh new file mode 100755 index 000000000..d444c11bb --- /dev/null +++ b/scripts/docker-entrypoint.sh @@ -0,0 +1,10 @@ +#!/bin/bash +set -e + +. /sio2/venv/bin/activate + +if [ ! -z ${DATABASE_HOST+x} ] && [ ! -z ${DATABASE_PORT+x} ]; then + /sio2/oioioi/scripts/wait-for-it.sh -t 60 "${DATABASE_HOST}:${DATABASE_PORT}" +fi + +exec "$@" diff --git a/scripts/oioioi-init.sh b/scripts/oioioi-init.sh new file mode 100755 index 000000000..a450d22b2 --- /dev/null +++ b/scripts/oioioi-init.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -e +set -x + +./manage.py migrate 2>&1 | tee /sio2/deployment/logs/migrate.log + +# load the fixture at the first start of the container and only in a development environment +if [ ${DEBUG} = "True" ]; then + ./manage.py loaddata ../oioioi/oioioi/admin_admin_fixture.json +fi + +exec ./manage.py supervisor --logfile=/sio2/deployment/logs/supervisor.log diff --git a/entrypoint_checks.sh b/scripts/static-checks.sh similarity index 98% rename from entrypoint_checks.sh rename to scripts/static-checks.sh index 4d113bab2..c08444fbf 100755 --- a/entrypoint_checks.sh +++ b/scripts/static-checks.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash set -o pipefail diff --git a/wait-for-it.sh b/scripts/wait-for-it.sh similarity index 100% rename from wait-for-it.sh rename to scripts/wait-for-it.sh diff --git a/selenium_init.sh b/selenium_init.sh deleted file mode 100755 index 090e7a099..000000000 --- a/selenium_init.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash -set -e -set -x - -/sio2/oioioi/wait-for-it.sh -t 60 "db:5432" - -./manage.py migrate 2>&1 | tee /sio2/deployment/logs/migrate.log - -./manage.py loaddata ../oioioi/oioioi_selenium/data.json - -exec ./manage.py supervisor --logfile=/sio2/deployment/logs/supervisor.log diff --git a/test_cypress.sh b/test_cypress.sh index 68cb034c1..29370735c 100755 --- a/test_cypress.sh +++ b/test_cypress.sh @@ -4,15 +4,15 @@ set -e # Run this script with -g to run CyPress with GUI -# or without to run tests in terminal mode. +# or without to run tests in terminal mode. # cy:run runs tests in terminal mode # cy:open opens interactive gui -echo -e "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!" -echo -e " Remember to flush the database " -echo -e " and create superuser admin, admin. " -echo -e "Some tests may relay on an empty database" -echo -e "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!" +echo -e "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!" +echo -e " Remember to flush the database " +echo -e " and create superuser admin, admin. " +echo -e " Some tests may rely on an empty database " +echo -e "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!" gui='run' diff --git a/test_selenium.sh b/test_selenium.sh index 8b41c2921..a40e46676 100755 --- a/test_selenium.sh +++ b/test_selenium.sh @@ -1,7 +1,7 @@ #!/bin/bash docker_compose_fun() { - docker-compose -p oioioi-selenium -f docker-compose-selenium.yml "$@" + OIOIOI_UID=$(id -u) docker-compose -p oioioi-selenium -f docker-compose-dev.yml -f docker-compose-selenium.yml "$@" } cd "`dirname "$0"`" diff --git a/worker_init.sh b/worker_init.sh index 32911c01a..a4d4a8346 100755 --- a/worker_init.sh +++ b/worker_init.sh @@ -4,8 +4,8 @@ set -x sudo apt install -y proot -/sio2/oioioi/wait-for-it.sh -t 60 "db:5432" -/sio2/oioioi/wait-for-it.sh -t 0 "web:8000" +/sio2/oioioi/scripts/wait-for-it.sh -t 60 "${DATABASE_HOST}:${DATABASE_PORT}" +/sio2/oioioi/scripts/wait-for-it.sh -t 0 "web:8000" mkdir -pv /sio2/deployment/logs/database