From 3a5bb001ab6d8d3677555445e04de20b059c40ba Mon Sep 17 00:00:00 2001 From: Eugene Molotov Date: Wed, 21 Feb 2024 20:36:25 +0500 Subject: [PATCH] install: use packages from OS rather than pip Several reasons: 1. I am tired to of GitHub's notifications, that some package is vulnerable, please bump version. Last days I don't have much time to find out, that exactly problem is and should I consider to bump. For example GitHub suggested me to bump starlette to 0.36.2, The "patch" only forces to use non-vulnerable 'python-multipart' package [1]. In any case I did not have to bump starlette, since I don't use any `request.form()` method in QLLR. 2. Let's imagine, that yet another vulnerability affects QLLR. I bump some depency. So server admin, that installed QLLR on his VPS should also bump that dependency. But before that admin needs to know about it. At the moment of writing I don't have any newsletter to notify "PLEASE UPDATE ASAP". Also, if I had that newsletter, admin is free not to subscribe. 3. Some pip packages may require to upgrade OS package or OS itself. For example python3 -m pip install urllib3 won't work on debian stretch. For details in [2]. 4. After the incident, when some scumbag intentionally embeded malware code in his package [3] and other politic-related events in and around Russia and Belarus, there is trend in Russia to use dependencies or applications with Russian vendors. TODO: [1] https://github.com/encode/starlette/commit/13e5c26a27f4903924624736abd6131b2da80cc5 [2] https://github.com/urllib3/urllib3/issues/2168 [3] https://github.com/advisories/GHSA-97m3-w2cp-4xx6 This version The mean reason, on previous For instance there is a vulnerability in pythondependecy TODO 1. GitHub watches requerements TODO 2. Scumbags adding malicious code in pip packages TODO 3. urllib3 suddenly stops working TODO --- .github/workflows/docker.yml | 34 ++++++++++++++++++++++++++++++++++ Dockerfile.develop | 23 ----------------------- Dockerfile.develop.alt10 | 34 ++++++++++++++++++++++++++++++++++ Dockerfile.develop.bookworm | 34 ++++++++++++++++++++++++++++++++++ Dockerfile.develop.bullseye | 33 +++++++++++++++++++++++++++++++++ Dockerfile.develop.buster | 36 ++++++++++++++++++++++++++++++++++++ Dockerfile.develop.focal | 35 +++++++++++++++++++++++++++++++++++ Dockerfile.develop.jammy | 33 +++++++++++++++++++++++++++++++++ Dockerfile.develop.noble | 34 ++++++++++++++++++++++++++++++++++ Dockerfile.develop.sisyphus | 34 ++++++++++++++++++++++++++++++++++ Dockerfile.develop.trixie | 34 ++++++++++++++++++++++++++++++++++ Dockerfile.production | 16 +++++++++++++++- README.md | 2 +- docker-compose.yml | 9 ++++++--- docs/install.md | 22 +++++++++++++++++----- qllr/templating.py | 12 +++++++++++- requirements.txt | 16 +++++++--------- tests/test_export_ratings.py | 2 +- 18 files changed, 399 insertions(+), 44 deletions(-) create mode 100644 .github/workflows/docker.yml delete mode 100644 Dockerfile.develop create mode 100644 Dockerfile.develop.alt10 create mode 100644 Dockerfile.develop.bookworm create mode 100644 Dockerfile.develop.bullseye create mode 100644 Dockerfile.develop.buster create mode 100644 Dockerfile.develop.focal create mode 100644 Dockerfile.develop.jammy create mode 100644 Dockerfile.develop.noble create mode 100644 Dockerfile.develop.sisyphus create mode 100644 Dockerfile.develop.trixie diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml new file mode 100644 index 0000000..25fd9c2 --- /dev/null +++ b/.github/workflows/docker.yml @@ -0,0 +1,34 @@ +name: Docker + +on: + push: + branches: [ master, develop ] + pull_request: + branches: [ master, develop ] + +jobs: + test: + strategy: + matrix: + dockerfile-suffix: [ + "alt10", "sisyphus", + "buster", "bullseye", "bookworm", "trixie", + "focal", "jammy", "noble" + ] + + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Build image + uses: docker/build-push-action@v5 + with: + context: . + file: ./Dockerfile.develop.${{ matrix.dockerfile-suffix }} + push: false + load: true + tags: test:latest + - name: Test + run: docker run --rm test:latest diff --git a/Dockerfile.develop b/Dockerfile.develop deleted file mode 100644 index f405c10..0000000 --- a/Dockerfile.develop +++ /dev/null @@ -1,23 +0,0 @@ -FROM python:3.7-stretch - -RUN echo "deb https://apt.postgresql.org/pub/repos/apt/ stretch-pgdg main" > /etc/apt/sources.list.d/pgdg.list -RUN wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - - -RUN apt-get update - -RUN DEBIAN_FRONTEND=noninteractive apt-get install -y postgresql-9.6 - -ENV PATH="/usr/lib/postgresql/9.6/bin:${PATH}" -# take out coverage report from source directory -ENV COVERAGE_FILE="/tmp/qllr.coverage" - -COPY --chown=www-data . /opt/qllr - -WORKDIR /opt/qllr - -RUN python3 -m pip install -r requirements.txt -RUN python3 -m pip install -r requirements_dev.txt - -USER www-data - -CMD ["./scripts/test"] diff --git a/Dockerfile.develop.alt10 b/Dockerfile.develop.alt10 new file mode 100644 index 0000000..677592e --- /dev/null +++ b/Dockerfile.develop.alt10 @@ -0,0 +1,34 @@ +FROM alt:p10 + +RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \ + python3-module-async-timeout \ + python3-module-cachetools \ + python3-module-jinja2 \ + python3-module-pip \ + python3-module-psycopg2 \ + python3-module-requests \ + python3-module-starlette \ + python3-module-uvicorn \ + && find /var/lib/apt/lists/ -type f -delete + +RUN apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y \ + git \ + postgresql15-server \ + python3-modules-sqlite3 \ + && find /var/lib/apt/lists/ -type f -delete + +ENV PATH="/usr/lib/postgresql/15/bin:${PATH}" +# take out coverage report from source directory +ENV COVERAGE_FILE="/tmp/qllr.coverage" + +COPY --chown=apache . /opt/qllr + +WORKDIR /opt/qllr + +RUN python3 -m pip install --no-deps -r requirements.txt +RUN python3 -m pip install -r requirements_dev.txt + +USER apache + +CMD ["./scripts/test"] diff --git a/Dockerfile.develop.bookworm b/Dockerfile.develop.bookworm new file mode 100644 index 0000000..e4233f0 --- /dev/null +++ b/Dockerfile.develop.bookworm @@ -0,0 +1,34 @@ +FROM debian:bookworm + +RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \ + python3-asyncpg \ + python3-cachetools \ + python3-jinja2 \ + python3-pip \ + python3-psycopg2 \ + python3-requests \ + python3-starlette \ + python3-uvicorn \ + && rm -rf /var/lib/apt/lists/* + +RUN apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + git \ + postgresql-15 \ + && rm -rf /var/lib/apt/lists/* + +ENV PATH="/usr/lib/postgresql/15/bin:${PATH}" +# take out coverage report from source directory +ENV COVERAGE_FILE="/tmp/qllr.coverage" +ENV PIP_BREAK_SYSTEM_PACKAGES=1 + +COPY --chown=www-data . /opt/qllr + +WORKDIR /opt/qllr + +RUN python3 -m pip install -r requirements.txt +RUN python3 -m pip install -r requirements_dev.txt + +USER www-data + +CMD ["./scripts/test"] diff --git a/Dockerfile.develop.bullseye b/Dockerfile.develop.bullseye new file mode 100644 index 0000000..91704f0 --- /dev/null +++ b/Dockerfile.develop.bullseye @@ -0,0 +1,33 @@ +FROM debian:bullseye + +RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \ + python3-asyncpg \ + python3-cachetools \ + python3-jinja2 \ + python3-pip \ + python3-psycopg2 \ + python3-requests \ + python3-starlette \ + python3-uvicorn \ + && rm -rf /var/lib/apt/lists/* + +RUN apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + git \ + postgresql-13 \ + && rm -rf /var/lib/apt/lists/* + +ENV PATH="/usr/lib/postgresql/13/bin:${PATH}" +# take out coverage report from source directory +ENV COVERAGE_FILE="/tmp/qllr.coverage" + +COPY --chown=www-data . /opt/qllr + +WORKDIR /opt/qllr + +RUN python3 -m pip install -r requirements.txt +RUN python3 -m pip install -r requirements_dev.txt + +USER www-data + +CMD ["./scripts/test"] diff --git a/Dockerfile.develop.buster b/Dockerfile.develop.buster new file mode 100644 index 0000000..b6c629c --- /dev/null +++ b/Dockerfile.develop.buster @@ -0,0 +1,36 @@ +FROM debian:buster + +RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \ + python3-asgiref \ + python3-click \ + python3-cachetools \ + python3-h11 \ + python3-idna \ + python3-jinja2 \ + python3-pip \ + python3-psycopg2 \ + python3-requests \ + python3-sniffio \ + python3-typing-extensions \ + && rm -rf /var/lib/apt/lists/* + +RUN apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + git \ + postgresql-11 \ + && rm -rf /var/lib/apt/lists/* + +ENV PATH="/usr/lib/postgresql/11/bin:${PATH}" +# take out coverage report from source directory +ENV COVERAGE_FILE="/tmp/qllr.coverage" + +COPY --chown=www-data . /opt/qllr + +WORKDIR /opt/qllr + +RUN python3 -m pip install -r requirements.txt +RUN python3 -m pip install -r requirements_dev.txt + +USER www-data + +CMD ["./scripts/test"] diff --git a/Dockerfile.develop.focal b/Dockerfile.develop.focal new file mode 100644 index 0000000..9d0108c --- /dev/null +++ b/Dockerfile.develop.focal @@ -0,0 +1,35 @@ +FROM ubuntu:focal + +RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \ + python3-asyncpg \ + python3-cachetools \ + python3-idna \ + python3-jinja2 \ + python3-pip \ + python3-psycopg2 \ + python3-requests \ + python3-sniffio \ + python3-typing-extensions \ + python3-uvicorn \ + && rm -rf /var/lib/apt/lists/* + +RUN apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + git \ + postgresql-12 \ + && rm -rf /var/lib/apt/lists/* + +ENV PATH="/usr/lib/postgresql/12/bin:${PATH}" +# take out coverage report from source directory +ENV COVERAGE_FILE="/tmp/qllr.coverage" + +COPY --chown=www-data . /opt/qllr + +WORKDIR /opt/qllr + +RUN python3 -m pip install -r requirements.txt +RUN python3 -m pip install -r requirements_dev.txt + +USER www-data + +CMD ["./scripts/test"] diff --git a/Dockerfile.develop.jammy b/Dockerfile.develop.jammy new file mode 100644 index 0000000..3a90aa5 --- /dev/null +++ b/Dockerfile.develop.jammy @@ -0,0 +1,33 @@ +FROM ubuntu:jammy + +RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \ + python3-asyncpg \ + python3-cachetools \ + python3-jinja2 \ + python3-pip \ + python3-psycopg2 \ + python3-requests \ + python3-starlette \ + python3-uvicorn \ + && rm -rf /var/lib/apt/lists/* + +RUN apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + git \ + postgresql-14 \ + && rm -rf /var/lib/apt/lists/* + +ENV PATH="/usr/lib/postgresql/14/bin:${PATH}" +# take out coverage report from source directory +ENV COVERAGE_FILE="/tmp/qllr.coverage" + +COPY --chown=www-data . /opt/qllr + +WORKDIR /opt/qllr + +RUN python3 -m pip install -r requirements.txt +RUN python3 -m pip install -r requirements_dev.txt + +USER www-data + +CMD ["./scripts/test"] diff --git a/Dockerfile.develop.noble b/Dockerfile.develop.noble new file mode 100644 index 0000000..d9de119 --- /dev/null +++ b/Dockerfile.develop.noble @@ -0,0 +1,34 @@ +FROM ubuntu:noble + +RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \ + python3-asyncpg \ + python3-cachetools \ + python3-jinja2 \ + python3-pip \ + python3-psycopg2 \ + python3-requests \ + python3-starlette \ + python3-uvicorn \ + && rm -rf /var/lib/apt/lists/* + +RUN apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + git \ + postgresql-16 \ + && rm -rf /var/lib/apt/lists/* + +ENV PATH="/usr/lib/postgresql/16/bin:${PATH}" +# take out coverage report from source directory +ENV COVERAGE_FILE="/tmp/qllr.coverage" +ENV PIP_BREAK_SYSTEM_PACKAGES=1 + +COPY --chown=www-data . /opt/qllr + +WORKDIR /opt/qllr + +RUN python3 -m pip install -r requirements.txt +RUN python3 -m pip install -r requirements_dev.txt + +USER www-data + +CMD ["./scripts/test"] diff --git a/Dockerfile.develop.sisyphus b/Dockerfile.develop.sisyphus new file mode 100644 index 0000000..63899d5 --- /dev/null +++ b/Dockerfile.develop.sisyphus @@ -0,0 +1,34 @@ +FROM alt:sisyphus + +RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \ + python3-module-asyncpg \ + python3-module-cachetools \ + python3-module-jinja2 \ + python3-module-pip \ + python3-module-psycopg2 \ + python3-module-requests \ + python3-module-starlette \ + python3-module-uvicorn \ + && find /var/lib/apt/lists/ -type f -delete + +RUN apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y \ + git \ + postgresql15-server \ + python3-modules-sqlite3 \ + && find /var/lib/apt/lists/ -type f -delete + +ENV PATH="/usr/lib/postgresql/15/bin:${PATH}" +# take out coverage report from source directory +ENV COVERAGE_FILE="/tmp/qllr.coverage" + +COPY --chown=apache . /opt/qllr + +WORKDIR /opt/qllr + +RUN python3 -m pip install -r requirements.txt +RUN python3 -m pip install -r requirements_dev.txt + +USER apache + +CMD ["./scripts/test"] diff --git a/Dockerfile.develop.trixie b/Dockerfile.develop.trixie new file mode 100644 index 0000000..3021b79 --- /dev/null +++ b/Dockerfile.develop.trixie @@ -0,0 +1,34 @@ +FROM debian:trixie + +RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \ + python3-asyncpg \ + python3-cachetools \ + python3-jinja2 \ + python3-pip \ + python3-psycopg2 \ + python3-requests \ + python3-starlette \ + python3-uvicorn \ + && rm -rf /var/lib/apt/lists/* + +RUN apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + git \ + postgresql-16 \ + && rm -rf /var/lib/apt/lists/* + +ENV PATH="/usr/lib/postgresql/16/bin:${PATH}" +# take out coverage report from source directory +ENV COVERAGE_FILE="/tmp/qllr.coverage" +ENV PIP_BREAK_SYSTEM_PACKAGES=1 + +COPY --chown=www-data . /opt/qllr + +WORKDIR /opt/qllr + +RUN python3 -m pip install -r requirements.txt +RUN python3 -m pip install -r requirements_dev.txt + +USER www-data + +CMD ["./scripts/test"] diff --git a/Dockerfile.production b/Dockerfile.production index 6170bb4..4051c47 100644 --- a/Dockerfile.production +++ b/Dockerfile.production @@ -1,4 +1,18 @@ -FROM python:3.7-stretch +FROM debian:buster + +RUN apt-get update && apt-get install -y \ + python3-asgiref \ + python3-click \ + python3-cachetools \ + python3-h11 \ + python3-jinja2 \ + python3-idna \ + python3-pip \ + python3-psycopg2 \ + python3-requests \ + python3-sniffio \ + python3-typing-extensions \ + && rm -rf /var/lib/apt/lists/* COPY --chown=www-data . /opt/qllr diff --git a/README.md b/README.md index 7d347e7..aa79e41 100644 --- a/README.md +++ b/README.md @@ -44,7 +44,7 @@ For feeder: For development: ``` -docker build . -t em92/qllr-dev -f Dockerfile.develop +docker build . -t em92/qllr-dev -f Dockerfile.develop.buster ``` For production: diff --git a/docker-compose.yml b/docker-compose.yml index ed9166f..0f82580 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -5,9 +5,10 @@ services: context: . dockerfile: Dockerfile.production ports: - - "7081:7081" + - "127.0.0.1:7081:8000" environment: HOST: 0.0.0.0 + PORT: 8000 env_file: - .env depends_on: ["db"] @@ -21,9 +22,11 @@ services: volumes: - ./sql/init.sql:/docker-entrypoint-initdb.d/init.sql - test: + dev: build: context: . - dockerfile: Dockerfile.develop + dockerfile: Dockerfile.develop.buster volumes: - .:/opt/qllr + ports: + - "127.0.0.1:7081:8000" diff --git a/docs/install.md b/docs/install.md index 5340d99..d7c39f3 100644 --- a/docs/install.md +++ b/docs/install.md @@ -2,11 +2,22 @@ ## Installing and configuring QLLR ``` -# base apps -sudo apt-get install python3 python3-pip python3-venv postgresql git - -# psycopg2 build prerequires -sudo apt-get install python3-dev libpq-dev gcc +# dependecies +sudo apt-get install -y + git \ + postgresql + python3-asgiref \ + python3-click \ + python3-cachetools \ + python3-h11 \ + python3-idna \ + python3-jinja2 \ + python3-pip \ + python3-psycopg2 \ + python3-requests \ + python3-sniffio \ + python3-typing-extensions \ + python3-venv # install qllr git clone https://github.com/em92/quakelive-local-ratings @@ -16,6 +27,7 @@ cd ./quakelive-local-ratings python3 -m venv venv source venv/bin/activate +# install other dependencies python3 -m pip install -r requirements.txt ``` diff --git a/qllr/templating.py b/qllr/templating.py index 8162ca6..0266f49 100644 --- a/qllr/templating.py +++ b/qllr/templating.py @@ -1,9 +1,19 @@ import typing from urllib.parse import ParseResult, urlparse -from jinja2 import Undefined, contextfunction, escape +from jinja2 import Undefined from starlette.templating import Jinja2Templates +try: + from jinja2 import contextfunction +except ImportError: # pragma: nocover + from jinja2 import pass_context as contextfunction + +try: + from jinja2 import escape +except ImportError: # pragma: nocover + from markupsafe import escape + def render_ql_nickname(nickname): nickname = str(escape(nickname)) diff --git a/requirements.txt b/requirements.txt index 2fdf1a7..7be9416 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,13 +1,11 @@ -psycopg2==2.9.1; python_version < '3.11' -psycopg2==2.9.9; python_version >= '3.11' +psycopg2<3 trueskill==0.4.4 -starlette==0.27.0; -asyncpg==0.24.0; python_version < '3.11' -asyncpg==0.29.0; python_version >= '3.11' -uvicorn==0.14.0 +starlette +asyncpg +uvicorn requests -Jinja2==2.11.3 -cachetools==3.1.1 +Jinja2 +cachetools -MarkupSafe<2.1 # https://github.com/pallets/jinja/issues/1658 +MarkupSafe<2.1; python_version <= '3.7' # https://github.com/pallets/jinja/issues/1658 diff --git a/tests/test_export_ratings.py b/tests/test_export_ratings.py index 96e55a4..7e8b8b9 100644 --- a/tests/test_export_ratings.py +++ b/tests/test_export_ratings.py @@ -21,7 +21,7 @@ def test_ratings_ad_redirect_bad_format(service): def test_ratings_ad_csv(service): resp = service.get("/export_rating/ad.csv") - assert resp.encoding == "utf-8" + assert resp.encoding.lower() == "utf-8" assert resp.text == read_sample("exported_ratings_ad.csv")